feat: add diskann index (#369)
This commit is contained in:
@@ -60,6 +60,14 @@ jobs:
|
||||
sudo apt-get install -y clang libomp-dev
|
||||
shell: bash
|
||||
|
||||
- name: Install AIO
|
||||
if: runner.os == 'Linux' && runner.arch == 'X64'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libaio-dev
|
||||
shell: bash
|
||||
|
||||
- name: Print CPU info
|
||||
if: runner.os == 'Linux'
|
||||
run: lscpu
|
||||
@@ -97,7 +105,7 @@ jobs:
|
||||
pytest-xdist \
|
||||
scikit-build-core \
|
||||
setuptools_scm
|
||||
shell: bash
|
||||
shell: bash
|
||||
|
||||
- name: Build from source
|
||||
run: |
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
if: steps.changed_files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang-tidy=1:18.0-59~exp2 cmake ninja-build libomp-dev
|
||||
sudo apt-get install -y clang-tidy=1:18.0-59~exp2 cmake ninja-build libomp-dev libaio-dev
|
||||
|
||||
- name: Setup ccache
|
||||
if: steps.changed_files.outputs.any_changed == 'true'
|
||||
|
||||
@@ -40,6 +40,9 @@
|
||||
[submodule "thirdparty/RaBitQ-Library/RaBitQ-Library-0.1"]
|
||||
path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.1
|
||||
url = https://github.com/VectorDB-NTU/RaBitQ-Library.git
|
||||
[submodule "thirdparty/aio/libaio-0.3"]
|
||||
path = thirdparty/aio/libaio-0.3
|
||||
url = https://github.com/yugabyte/libaio.git
|
||||
[submodule "thirdparty/cppjieba/cppjieba-5.6.7"]
|
||||
path = thirdparty/cppjieba/cppjieba-5.6.7
|
||||
url = https://github.com/yanyiwu/cppjieba.git
|
||||
|
||||
@@ -116,6 +116,17 @@ else()
|
||||
endif()
|
||||
message(STATUS "RABITQ_ARCH_FLAG: ${RABITQ_ARCH_FLAG}")
|
||||
|
||||
# DiskAnn support (Linux x86_64 only, requires libaio)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386" AND NOT ANDROID AND NOT IOS)
|
||||
set(DISKANN_SUPPORTED ON)
|
||||
add_definitions(-DDISKANN_SUPPORTED=1)
|
||||
else()
|
||||
set(DISKANN_SUPPORTED OFF)
|
||||
add_definitions(-DDISKANN_SUPPORTED=0)
|
||||
message(STATUS "DiskAnn support disabled - only supported on Linux x86_64")
|
||||
endif()
|
||||
message(STATUS "DISKANN_SUPPORTED: ${DISKANN_SUPPORTED}")
|
||||
|
||||
option(USE_OSS_MIRROR "Use OSS mirror for faster third-party downloads" OFF)
|
||||
if(DEFINED ENV{USE_OSS_MIRROR} AND NOT "$ENV{USE_OSS_MIRROR}" STREQUAL "")
|
||||
set(USE_OSS_MIRROR "$ENV{USE_OSS_MIRROR}" CACHE BOOL "Use OSS mirror for faster third-party downloads" FORCE)
|
||||
@@ -155,6 +166,17 @@ if(BUILD_PYTHON_BINDINGS)
|
||||
message(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
|
||||
install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
|
||||
|
||||
# DiskAnn ships as a runtime-loaded shared module
|
||||
# (libzvec_diskann_plugin.so) that is brought online implicitly the
|
||||
# first time a DiskAnn index is created — users never call any load
|
||||
# function. The Python extension resolves the module next to _zvec.so
|
||||
# (see the $ORIGIN rpath in src/binding/python/CMakeLists.txt); the
|
||||
# module must therefore be installed alongside _zvec.so in the wheel.
|
||||
# The target exists only on platforms where DiskAnn is buildable
|
||||
# (currently Linux x86_64 with libaio).
|
||||
if(TARGET core_knn_diskann)
|
||||
install(TARGETS core_knn_diskann LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
|
||||
endif()
|
||||
# Bundle cppjieba's dictionary files so the `jieba` FTS tokenizer works
|
||||
# out of the box. python/zvec/__init__.py resolves this directory via
|
||||
# importlib.resources and registers it with set_default_jieba_dict_dir().
|
||||
|
||||
@@ -1,9 +1,60 @@
|
||||
import pytest
|
||||
import logging
|
||||
import platform
|
||||
|
||||
DISKANN_SUPPORTED = platform.system() == "Linux" and platform.machine() in (
|
||||
"x86_64",
|
||||
"AMD64",
|
||||
"i686",
|
||||
"i386",
|
||||
)
|
||||
|
||||
from typing import Any, Generator
|
||||
from zvec.typing import DataType, StatusCode, MetricType, QuantizeType
|
||||
import zvec
|
||||
|
||||
|
||||
# Cache the DiskAnn plugin preload status so we pay the load cost once per
|
||||
# test session. The plugin normally auto-loads on first DiskAnn use, but we
|
||||
# preload it explicitly here so a missing libaio / misplaced plugin .so
|
||||
# surfaces as a clear pytest skip instead of a confusing
|
||||
# "Create vector column indexer failed" deep inside the collection code path.
|
||||
_DISKANN_PRELOAD_REASON: str | None = None
|
||||
_DISKANN_PRELOAD_DONE: bool = False
|
||||
|
||||
|
||||
def _ensure_diskann_runtime_or_reason() -> str | None:
|
||||
"""Preload the DiskAnn plugin and return None on success or a human-readable
|
||||
skip reason on failure. Idempotent across calls."""
|
||||
global _DISKANN_PRELOAD_DONE, _DISKANN_PRELOAD_REASON
|
||||
if _DISKANN_PRELOAD_DONE:
|
||||
return _DISKANN_PRELOAD_REASON
|
||||
_DISKANN_PRELOAD_DONE = True
|
||||
|
||||
if not DISKANN_SUPPORTED:
|
||||
_DISKANN_PRELOAD_REASON = "DiskAnn only supported on Linux x86_64"
|
||||
return _DISKANN_PRELOAD_REASON
|
||||
|
||||
if not zvec.is_libaio_available():
|
||||
_DISKANN_PRELOAD_REASON = (
|
||||
"libaio is not available on this host; DiskAnn cannot run. "
|
||||
"Install libaio1 (or libaio1t64 on Ubuntu 24.04+) and retry."
|
||||
)
|
||||
return _DISKANN_PRELOAD_REASON
|
||||
|
||||
status = zvec.load_diskann_plugin()
|
||||
if status != zvec.DISKANN_PLUGIN_OK:
|
||||
_DISKANN_PRELOAD_REASON = (
|
||||
f"Failed to load DiskAnn plugin (status={status}); "
|
||||
"check that libzvec_diskann_plugin.so is installed alongside "
|
||||
"_zvec.so in the Python site-packages directory."
|
||||
)
|
||||
return _DISKANN_PRELOAD_REASON
|
||||
|
||||
_DISKANN_PRELOAD_REASON = None
|
||||
return None
|
||||
|
||||
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
InvertIndexParam,
|
||||
@@ -97,6 +148,15 @@ def full_schema_new(request) -> CollectionSchema:
|
||||
else:
|
||||
nullable, has_index, vector_index = True, False, HnswIndexParam()
|
||||
|
||||
# Skip DiskAnn tests on unsupported platforms or when the runtime cannot
|
||||
# be brought up (missing libaio, plugin .so not installed, etc.).
|
||||
from zvec.model.param import DiskAnnIndexParam
|
||||
|
||||
if isinstance(vector_index, DiskAnnIndexParam):
|
||||
skip_reason = _ensure_diskann_runtime_or_reason()
|
||||
if skip_reason is not None:
|
||||
pytest.skip(skip_reason)
|
||||
|
||||
scalar_index_param = None
|
||||
vector_index_param = None
|
||||
if has_index:
|
||||
|
||||
@@ -22,8 +22,10 @@ from zvec.model.param import (
|
||||
HnswIndexParam,
|
||||
FlatIndexParam,
|
||||
IVFIndexParam,
|
||||
DiskAnnIndexParam,
|
||||
HnswQueryParam,
|
||||
IVFQueryParam,
|
||||
DiskAnnQueryParam,
|
||||
)
|
||||
|
||||
from zvec.model.schema import FieldSchema, VectorSchema
|
||||
@@ -179,10 +181,24 @@ def get_ground_truth_map(collection, test_docs, query_vectors_map, metric_type,
|
||||
for field_name, query_vectors in query_vectors_map.items():
|
||||
ground_truth_map[field_name] = {}
|
||||
|
||||
# Support per-field metric type: metric_type can be a dict mapping
|
||||
# field_name -> MetricType, or a single MetricType applied to all fields.
|
||||
if isinstance(metric_type, dict):
|
||||
field_metric = metric_type.get(field_name, MetricType.IP)
|
||||
else:
|
||||
field_metric = metric_type
|
||||
|
||||
for i, query_vector in enumerate(query_vectors):
|
||||
# Get the ground truth for this query
|
||||
relevant_doc_ids_scores = get_ground_truth_for_vector_query(
|
||||
collection, query_vector, field_name, test_docs, i, metric_type, k, True
|
||||
collection,
|
||||
query_vector,
|
||||
field_name,
|
||||
test_docs,
|
||||
i,
|
||||
field_metric,
|
||||
k,
|
||||
True,
|
||||
)
|
||||
ground_truth_map[field_name][i] = relevant_doc_ids_scores
|
||||
|
||||
@@ -292,6 +308,7 @@ class TestRecall:
|
||||
[
|
||||
(True, True, HnswIndexParam()),
|
||||
(False, True, IVFIndexParam()),
|
||||
(False, True, DiskAnnIndexParam()),
|
||||
(False, True, FlatIndexParam()), # ——ok
|
||||
(
|
||||
True,
|
||||
@@ -371,6 +388,19 @@ class TestRecall:
|
||||
use_soar=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
True,
|
||||
True,
|
||||
DiskAnnIndexParam(
|
||||
metric_type=MetricType.IP,
|
||||
max_degree=32,
|
||||
),
|
||||
),
|
||||
(
|
||||
True,
|
||||
True,
|
||||
DiskAnnIndexParam(metric_type=MetricType.L2, max_degree=32),
|
||||
),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@@ -388,10 +418,16 @@ class TestRecall:
|
||||
):
|
||||
full_schema_params = request.getfixturevalue("full_schema_new")
|
||||
|
||||
# Build per-field metric type map so ground truth uses each field's
|
||||
# actual index metric (fields may fall back to HnswIndexParam/IP).
|
||||
field_metric_map = {}
|
||||
for vector_para in full_schema_params.vectors:
|
||||
if vector_para.name == "vector_fp32_field":
|
||||
metric_type = vector_para.index_param.metric_type
|
||||
break
|
||||
if vector_para.index_param is not None:
|
||||
field_metric_map[vector_para.name] = vector_para.index_param.metric_type
|
||||
else:
|
||||
field_metric_map[vector_para.name] = MetricType.IP
|
||||
|
||||
metric_type = field_metric_map.get("vector_fp32_field", MetricType.IP)
|
||||
|
||||
multiple_docs = [
|
||||
generate_doc_recall(i, full_collection_new.schema) for i in range(doc_num)
|
||||
@@ -438,9 +474,13 @@ class TestRecall:
|
||||
multiple_docs[i].vectors[field_name] for i in range(query_num)
|
||||
]
|
||||
|
||||
# Get ground truth mapping
|
||||
# Get ground truth mapping (pass per-field metric map)
|
||||
ground_truth_map = get_ground_truth_map(
|
||||
full_collection_new, multiple_docs, query_vectors_map, metric_type, top_k
|
||||
full_collection_new,
|
||||
multiple_docs,
|
||||
query_vectors_map,
|
||||
field_metric_map,
|
||||
top_k,
|
||||
)
|
||||
|
||||
# Validate ground truth mapping structure
|
||||
@@ -479,8 +519,8 @@ class TestRecall:
|
||||
|
||||
print("(recall_at_k_stats:\n")
|
||||
print(recall_at_k_stats)
|
||||
print("metric_type:")
|
||||
print(metric_type)
|
||||
print("field_metric_map:")
|
||||
print(field_metric_map)
|
||||
# Print Recall@K statistics
|
||||
print(f"Recall@{top_k} using Ground Truth:")
|
||||
for field_name, stats in recall_at_k_stats.items():
|
||||
@@ -552,7 +592,21 @@ class TestRecall:
|
||||
use_soar=True,
|
||||
),
|
||||
),
|
||||
# (True, True, IVFIndexParam(metric_type=MetricType.COSINE, n_list=150, n_iters=15, use_soar=False, )),
|
||||
(
|
||||
True,
|
||||
True,
|
||||
DiskAnnIndexParam(metric_type=MetricType.IP, max_degree=32),
|
||||
),
|
||||
(
|
||||
True,
|
||||
True,
|
||||
DiskAnnIndexParam(metric_type=MetricType.L2, max_degree=32),
|
||||
),
|
||||
(
|
||||
True,
|
||||
True,
|
||||
DiskAnnIndexParam(metric_type=MetricType.COSINE, max_degree=32),
|
||||
),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@@ -571,10 +625,16 @@ class TestRecall:
|
||||
):
|
||||
full_schema_params = request.getfixturevalue("full_schema_new")
|
||||
|
||||
# Build per-field metric type map so ground truth uses each field's
|
||||
# actual index metric (fields may fall back to HnswIndexParam/IP).
|
||||
field_metric_map = {}
|
||||
for vector_para in full_schema_params.vectors:
|
||||
if vector_para.name == "vector_fp32_field":
|
||||
metric_type = vector_para.index_param.metric_type
|
||||
break
|
||||
if vector_para.index_param is not None:
|
||||
field_metric_map[vector_para.name] = vector_para.index_param.metric_type
|
||||
else:
|
||||
field_metric_map[vector_para.name] = MetricType.IP
|
||||
|
||||
metric_type = field_metric_map.get("vector_fp32_field", MetricType.IP)
|
||||
|
||||
multiple_docs = [
|
||||
generate_doc_recall(i, full_collection_new.schema) for i in range(doc_num)
|
||||
@@ -621,9 +681,13 @@ class TestRecall:
|
||||
multiple_docs[i].vectors[field_name] for i in range(query_num)
|
||||
]
|
||||
|
||||
# Get ground truth mapping
|
||||
# Get ground truth mapping (pass per-field metric map)
|
||||
ground_truth_map = get_ground_truth_map(
|
||||
full_collection_new, multiple_docs, query_vectors_map, metric_type, top_k
|
||||
full_collection_new,
|
||||
multiple_docs,
|
||||
query_vectors_map,
|
||||
field_metric_map,
|
||||
top_k,
|
||||
)
|
||||
|
||||
# Validate ground truth mapping structure
|
||||
@@ -662,8 +726,8 @@ class TestRecall:
|
||||
|
||||
print("(recall_at_k_stats:\n")
|
||||
print(recall_at_k_stats)
|
||||
print("metric_type:")
|
||||
print(metric_type)
|
||||
print("field_metric_map:")
|
||||
print(field_metric_map)
|
||||
# Print Recall@K statistics
|
||||
print(f"Recall@{top_k} using Ground Truth:")
|
||||
for field_name, stats in recall_at_k_stats.items():
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""End-to-end collection tests for the DiskAnn index.
|
||||
|
||||
Mirrors ``test_collection_hnsw_rabitq.py`` but targets the DiskAnn plugin.
|
||||
|
||||
Two platform-level prerequisites are enforced at module import time:
|
||||
|
||||
1. DiskAnn is currently built only for Linux x86_64 — other platforms are
|
||||
skipped wholesale.
|
||||
2. The DiskAnn backend lives in a *runtime-loaded* plugin
|
||||
(``libzvec_diskann_plugin.so``). It must be loaded with ``RTLD_GLOBAL |
|
||||
RTLD_NOW`` BEFORE ``import zvec`` so that the plugin's ``IndexFactory``
|
||||
singleton is unified with the one inside ``_zvec.so``. After ``import
|
||||
zvec`` we must also call ``zvec.load_diskann_plugin()`` exactly once.
|
||||
|
||||
If either prerequisite fails the whole module is skipped so the rest of the
|
||||
test-suite is not affected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Platform gating (must happen BEFORE we touch zvec).
|
||||
# --------------------------------------------------------------------------- #
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (sys.platform == "linux" and platform.machine() in ("x86_64", "AMD64")),
|
||||
reason="DiskAnn plugin is only supported on Linux x86_64",
|
||||
)
|
||||
|
||||
# Promote all symbols in subsequently-loaded DSOs to the global namespace and
|
||||
# resolve relocations eagerly. This is REQUIRED so the DiskAnn plugin can see
|
||||
# the ``IndexFactory`` singleton that lives in ``_zvec.so`` and vice versa.
|
||||
# See: DiskAnn RTLD_GLOBAL + RTLD_NOW Requirement.
|
||||
if sys.platform == "linux":
|
||||
sys.setdlopenflags(sys.getdlopenflags() | os.RTLD_GLOBAL | os.RTLD_NOW)
|
||||
|
||||
import zvec # noqa: E402
|
||||
|
||||
from zvec import ( # noqa: E402
|
||||
Collection,
|
||||
CollectionOption,
|
||||
DataType,
|
||||
DiskAnnIndexParam,
|
||||
DiskAnnQueryParam,
|
||||
Doc,
|
||||
FieldSchema,
|
||||
MetricType,
|
||||
Query,
|
||||
VectorSchema,
|
||||
)
|
||||
from zvec.typing import QuantizeType # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def diskann_collection_schema():
|
||||
"""Create a collection schema with a DiskAnn index."""
|
||||
return zvec.CollectionSchema(
|
||||
name="test_diskann_collection",
|
||||
fields=[
|
||||
FieldSchema("id", DataType.INT64, nullable=False),
|
||||
FieldSchema("name", DataType.STRING, nullable=False),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"embedding",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=DiskAnnIndexParam(
|
||||
metric_type=MetricType.L2,
|
||||
max_degree=64,
|
||||
list_size=100,
|
||||
pq_chunk_num=0,
|
||||
quantize_type=QuantizeType.UNDEFINED,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def collection_option():
|
||||
"""Create collection options."""
|
||||
return CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def single_doc():
|
||||
"""Create a single document for testing."""
|
||||
return Doc(
|
||||
id="0",
|
||||
fields={"id": 0, "name": "test_doc_0"},
|
||||
vectors={"embedding": [0.1 + i * 0.01 for i in range(128)]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multiple_docs():
|
||||
"""Create multiple documents for testing."""
|
||||
return [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"test_doc_{i}"},
|
||||
vectors={"embedding": [i * 0.1 + j * 0.01 for j in range(128)]},
|
||||
)
|
||||
for i in range(1, 101)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def diskann_collection(
|
||||
tmp_path_factory, diskann_collection_schema, collection_option
|
||||
) -> Collection:
|
||||
"""
|
||||
Function-scoped fixture: creates and opens a collection with DiskAnn index.
|
||||
"""
|
||||
temp_dir = tmp_path_factory.mktemp("zvec_diskann")
|
||||
collection_path = temp_dir / "test_diskann_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=diskann_collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open DiskAnn collection"
|
||||
assert coll.path == str(collection_path)
|
||||
assert coll.schema.name == diskann_collection_schema.name
|
||||
|
||||
try:
|
||||
yield coll
|
||||
finally:
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collection_with_single_doc(
|
||||
diskann_collection: Collection, single_doc: Doc
|
||||
) -> Collection:
|
||||
"""Setup: insert single doc into collection."""
|
||||
assert diskann_collection.stats.doc_count == 0
|
||||
result = diskann_collection.insert(single_doc)
|
||||
assert bool(result)
|
||||
assert result.ok()
|
||||
assert diskann_collection.stats.doc_count == 1
|
||||
|
||||
yield diskann_collection
|
||||
|
||||
# Teardown: delete single doc
|
||||
diskann_collection.delete(single_doc.id)
|
||||
assert diskann_collection.stats.doc_count == 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collection_with_multiple_docs(
|
||||
diskann_collection: Collection, multiple_docs: list[Doc]
|
||||
) -> Collection:
|
||||
"""Setup: insert multiple docs into collection."""
|
||||
assert diskann_collection.stats.doc_count == 0
|
||||
result = diskann_collection.insert(multiple_docs)
|
||||
assert len(result) == len(multiple_docs)
|
||||
for item in result:
|
||||
assert item.ok()
|
||||
assert diskann_collection.stats.doc_count == len(multiple_docs)
|
||||
|
||||
yield diskann_collection
|
||||
|
||||
# Teardown: delete multiple docs
|
||||
diskann_collection.delete([doc.id for doc in multiple_docs])
|
||||
|
||||
|
||||
# ==================== Tests ====================
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionCreation:
|
||||
"""Test DiskAnn collection creation and schema validation."""
|
||||
|
||||
def test_collection_creation(
|
||||
self, diskann_collection: Collection, diskann_collection_schema
|
||||
):
|
||||
"""Test that collection is created with correct schema."""
|
||||
assert diskann_collection is not None
|
||||
assert diskann_collection.schema.name == diskann_collection_schema.name
|
||||
assert len(diskann_collection.schema.fields) == len(
|
||||
diskann_collection_schema.fields
|
||||
)
|
||||
assert len(diskann_collection.schema.vectors) == len(
|
||||
diskann_collection_schema.vectors
|
||||
)
|
||||
|
||||
def test_vector_schema_validation(self, diskann_collection: Collection):
|
||||
"""Test that vector schema has correct DiskAnn configuration."""
|
||||
vector_schema = diskann_collection.schema.vector("embedding")
|
||||
assert vector_schema is not None
|
||||
assert vector_schema.name == "embedding"
|
||||
assert vector_schema.data_type == DataType.VECTOR_FP32
|
||||
assert vector_schema.dimension == 128
|
||||
|
||||
index_param = vector_schema.index_param
|
||||
assert index_param is not None
|
||||
assert index_param.metric_type == MetricType.L2
|
||||
assert index_param.max_degree == 64
|
||||
assert index_param.list_size == 100
|
||||
assert index_param.pq_chunk_num == 0
|
||||
|
||||
def test_collection_stats(self, diskann_collection: Collection):
|
||||
"""Test initial collection statistics."""
|
||||
stats = diskann_collection.stats
|
||||
assert stats is not None
|
||||
assert stats.doc_count == 0
|
||||
assert len(stats.index_completeness) == 1
|
||||
assert stats.index_completeness["embedding"] == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionInsert:
|
||||
"""Test document insertion into DiskAnn collection."""
|
||||
|
||||
def test_insert_single_doc(self, diskann_collection: Collection, single_doc: Doc):
|
||||
"""Test inserting a single document."""
|
||||
result = diskann_collection.insert(single_doc)
|
||||
assert bool(result)
|
||||
assert result.ok()
|
||||
|
||||
stats = diskann_collection.stats
|
||||
assert stats is not None
|
||||
assert stats.doc_count == 1
|
||||
|
||||
def test_insert_multiple_docs(
|
||||
self, diskann_collection: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test inserting multiple documents."""
|
||||
result = diskann_collection.insert(multiple_docs)
|
||||
assert len(result) == len(multiple_docs)
|
||||
for item in result:
|
||||
assert item.ok()
|
||||
|
||||
stats = diskann_collection.stats
|
||||
assert stats is not None
|
||||
assert stats.doc_count == len(multiple_docs)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionFetch:
|
||||
"""Test document fetching from DiskAnn collection."""
|
||||
|
||||
def test_fetch_single_doc(
|
||||
self, collection_with_single_doc: Collection, single_doc: Doc
|
||||
):
|
||||
"""Test fetching a single document by ID."""
|
||||
result = collection_with_single_doc.fetch(ids=[single_doc.id])
|
||||
assert bool(result)
|
||||
assert single_doc.id in result.keys()
|
||||
|
||||
doc = result[single_doc.id]
|
||||
assert doc is not None
|
||||
assert doc.id == single_doc.id
|
||||
assert doc.field("id") == single_doc.field("id")
|
||||
assert doc.field("name") == single_doc.field("name")
|
||||
|
||||
def test_fetch_multiple_docs(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test fetching multiple documents by IDs."""
|
||||
ids = [doc.id for doc in multiple_docs[:10]]
|
||||
result = collection_with_multiple_docs.fetch(ids=ids)
|
||||
assert bool(result)
|
||||
assert len(result) == len(ids)
|
||||
|
||||
for doc_id in ids:
|
||||
assert doc_id in result
|
||||
doc = result[doc_id]
|
||||
assert doc is not None
|
||||
assert doc.id == doc_id
|
||||
|
||||
def test_fetch_nonexistent_doc(self, collection_with_single_doc: Collection):
|
||||
"""Test fetching a non-existent document."""
|
||||
result = collection_with_single_doc.fetch(ids=["nonexistent_id"])
|
||||
assert len(result) == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionQuery:
|
||||
"""Test vector search queries on DiskAnn collection."""
|
||||
|
||||
def test_query_by_vector(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying by vector with DiskAnn index."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
result = collection_with_multiple_docs.query(queries=query, topk=10)
|
||||
assert len(result) > 0
|
||||
assert len(result) <= 10
|
||||
|
||||
# First result should be the query document itself (or very close)
|
||||
first_doc = result[0]
|
||||
assert first_doc is not None
|
||||
assert first_doc.id is not None
|
||||
|
||||
def test_query_by_id(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying by document ID with DiskAnn index."""
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
id=multiple_docs[0].id,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
result = collection_with_multiple_docs.query(queries=query, topk=10)
|
||||
assert len(result) > 0
|
||||
assert len(result) <= 10
|
||||
|
||||
def test_query_with_different_list_size(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying with different list_size parameter values."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
|
||||
# Test with list_size=50
|
||||
query_small = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=50),
|
||||
)
|
||||
result_small = collection_with_multiple_docs.query(queries=query_small, topk=10)
|
||||
assert len(result_small) > 0
|
||||
|
||||
# Test with list_size=200
|
||||
query_large = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=200),
|
||||
)
|
||||
result_large = collection_with_multiple_docs.query(queries=query_large, topk=10)
|
||||
assert len(result_large) > 0
|
||||
|
||||
def test_query_with_topk(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying with different topk values."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
# Test topk=5
|
||||
result_5 = collection_with_multiple_docs.query(queries=query, topk=5)
|
||||
assert len(result_5) <= 5
|
||||
|
||||
# Test topk=20
|
||||
result_20 = collection_with_multiple_docs.query(queries=query, topk=20)
|
||||
assert len(result_20) <= 20
|
||||
|
||||
def test_query_with_filter(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying with filter conditions."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
# Query with id filter
|
||||
result = collection_with_multiple_docs.query(
|
||||
queries=query, topk=10, filter="id < 50"
|
||||
)
|
||||
assert len(result) > 0
|
||||
for doc in result:
|
||||
assert doc.field("id") < 50
|
||||
|
||||
def test_query_with_output_fields(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying with specific output fields."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
result = collection_with_multiple_docs.query(
|
||||
queries=query, topk=10, output_fields=["id", "name"]
|
||||
)
|
||||
assert len(result) > 0
|
||||
|
||||
first_doc = result[0]
|
||||
assert "id" in first_doc.field_names()
|
||||
assert "name" in first_doc.field_names()
|
||||
|
||||
def test_query_with_include_vector(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test querying with vector data included in results."""
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
|
||||
result = collection_with_multiple_docs.query(
|
||||
queries=query, topk=10, include_vector=True
|
||||
)
|
||||
assert len(result) > 0
|
||||
|
||||
first_doc = result[0]
|
||||
assert first_doc.vector("embedding") is not None
|
||||
assert len(first_doc.vector("embedding")) == 128
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionUpdate:
|
||||
"""Test document update in DiskAnn collection."""
|
||||
|
||||
def test_update_doc_fields(
|
||||
self, collection_with_single_doc: Collection, single_doc: Doc
|
||||
):
|
||||
"""Test updating document fields."""
|
||||
updated_doc = Doc(
|
||||
id=single_doc.id,
|
||||
fields={"id": single_doc.field("id"), "name": "updated_name"},
|
||||
)
|
||||
|
||||
result = collection_with_single_doc.update(updated_doc)
|
||||
assert bool(result)
|
||||
assert result.ok()
|
||||
|
||||
# Verify update
|
||||
fetched = collection_with_single_doc.fetch(ids=[single_doc.id])
|
||||
assert single_doc.id in fetched
|
||||
doc = fetched[single_doc.id]
|
||||
assert doc.field("name") == "updated_name"
|
||||
|
||||
def test_update_doc_vector(
|
||||
self, collection_with_single_doc: Collection, single_doc: Doc
|
||||
):
|
||||
"""Test updating document vector."""
|
||||
new_vector = [0.5 + i * 0.01 for i in range(128)]
|
||||
updated_doc = Doc(
|
||||
id=single_doc.id,
|
||||
vectors={"embedding": new_vector},
|
||||
)
|
||||
|
||||
result = collection_with_single_doc.update(updated_doc)
|
||||
assert bool(result)
|
||||
assert result.ok()
|
||||
|
||||
# Verify update
|
||||
fetched = collection_with_single_doc.fetch(
|
||||
ids=[single_doc.id],
|
||||
)
|
||||
assert single_doc.id in fetched
|
||||
doc = fetched[single_doc.id]
|
||||
assert doc.vector("embedding") is not None
|
||||
embedding = doc.vector("embedding")
|
||||
assert len(embedding) == 128
|
||||
# Verify vector values are approximately equal (float comparison)
|
||||
for i in range(128):
|
||||
assert math.isclose(embedding[i], new_vector[i], rel_tol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionDelete:
|
||||
"""Test document deletion from DiskAnn collection."""
|
||||
|
||||
def test_delete_single_doc(
|
||||
self, collection_with_single_doc: Collection, single_doc: Doc
|
||||
):
|
||||
"""Test deleting a single document."""
|
||||
result = collection_with_single_doc.delete(single_doc.id)
|
||||
assert bool(result)
|
||||
assert result.ok()
|
||||
|
||||
stats = collection_with_single_doc.stats
|
||||
assert stats.doc_count == 0
|
||||
|
||||
def test_delete_multiple_docs(
|
||||
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
|
||||
):
|
||||
"""Test deleting multiple documents."""
|
||||
ids_to_delete = [doc.id for doc in multiple_docs[:10]]
|
||||
result = collection_with_multiple_docs.delete(ids_to_delete)
|
||||
assert len(result) == len(ids_to_delete)
|
||||
for item in result:
|
||||
assert item.ok()
|
||||
|
||||
stats = collection_with_multiple_docs.stats
|
||||
assert stats.doc_count == len(multiple_docs) - len(ids_to_delete)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diskann_collection")
|
||||
class TestDiskAnnCollectionOptimizeAndReopen:
|
||||
"""Test collection optimize and reopen functionality."""
|
||||
|
||||
def test_optimize_close_reopen_and_query(
|
||||
self,
|
||||
tmp_path_factory,
|
||||
diskann_collection_schema,
|
||||
collection_option,
|
||||
multiple_docs: list[Doc],
|
||||
):
|
||||
"""Test inserting 100 docs, optimize, close, reopen and query."""
|
||||
# Create collection and insert 100 documents
|
||||
temp_dir = tmp_path_factory.mktemp("zvec_diskann_optimize")
|
||||
collection_path = temp_dir / "test_optimize_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=diskann_collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None
|
||||
assert coll.stats.doc_count == 0
|
||||
|
||||
# Insert 100 documents
|
||||
result = coll.insert(multiple_docs)
|
||||
assert len(result) == len(multiple_docs)
|
||||
for item in result:
|
||||
assert item.ok()
|
||||
assert coll.stats.doc_count == len(multiple_docs)
|
||||
|
||||
# Call optimize
|
||||
from zvec import OptimizeOption
|
||||
|
||||
coll.optimize(option=OptimizeOption())
|
||||
|
||||
# Verify data is still accessible after optimize
|
||||
query_vector = multiple_docs[0].vector("embedding")
|
||||
query = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
result_before_close = coll.query(queries=query, topk=10)
|
||||
assert len(result_before_close) > 0
|
||||
|
||||
# Close collection (destroy will close it)
|
||||
collection_path_str = str(collection_path)
|
||||
del coll
|
||||
|
||||
# Reopen collection
|
||||
reopened_coll = zvec.open(path=collection_path_str, option=collection_option)
|
||||
assert reopened_coll is not None
|
||||
assert reopened_coll.stats.doc_count == len(multiple_docs)
|
||||
|
||||
# Execute query on reopened collection
|
||||
query_after_reopen = Query(
|
||||
field_name="embedding",
|
||||
vector=query_vector,
|
||||
param=DiskAnnQueryParam(list_size=100),
|
||||
)
|
||||
result_after_reopen = reopened_coll.query(queries=query_after_reopen, topk=10)
|
||||
assert len(result_after_reopen) > 0
|
||||
assert len(result_after_reopen) <= 10
|
||||
|
||||
# Verify query results are valid
|
||||
first_doc = result_after_reopen[0]
|
||||
assert first_doc is not None
|
||||
assert first_doc.id is not None
|
||||
assert first_doc.field("id") is not None
|
||||
assert first_doc.field("name") is not None
|
||||
|
||||
# Cleanup
|
||||
reopened_coll.destroy()
|
||||
@@ -43,6 +43,21 @@ except Exception:
|
||||
# Public API — grouped by category
|
||||
# ==============================
|
||||
|
||||
# —— DiskAnn runtime plugin ——
|
||||
# Re-export the plugin management entry points defined by the C++ extension.
|
||||
# DiskAnn normally auto-loads on first use; these APIs let tests and
|
||||
# diagnostic tools preload the plugin and get a clear error if libaio is
|
||||
# missing or the plugin shared object cannot be located.
|
||||
from _zvec import (
|
||||
DISKANN_PLUGIN_DLOPEN_FAILED,
|
||||
DISKANN_PLUGIN_LIBAIO_MISSING,
|
||||
DISKANN_PLUGIN_OK,
|
||||
DISKANN_PLUGIN_UNSUPPORTED_PLATFORM,
|
||||
is_diskann_plugin_loaded,
|
||||
is_libaio_available,
|
||||
load_diskann_plugin,
|
||||
)
|
||||
|
||||
from . import model as model
|
||||
|
||||
# —— Extensions ——
|
||||
@@ -79,6 +94,8 @@ from .model.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
DiskAnnIndexParam,
|
||||
DiskAnnQueryParam,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
FtsQueryParam,
|
||||
@@ -144,6 +161,8 @@ __all__ = [
|
||||
"HnswRabitqIndexParam",
|
||||
"FlatIndexParam",
|
||||
"IVFIndexParam",
|
||||
"DiskAnnIndexParam",
|
||||
"DiskAnnQueryParam",
|
||||
"CollectionOption",
|
||||
"IndexOption",
|
||||
"OptimizeOption",
|
||||
@@ -182,6 +201,14 @@ __all__ = [
|
||||
"StatusCode",
|
||||
# Tools
|
||||
"require_module",
|
||||
# DiskAnn plugin
|
||||
"load_diskann_plugin",
|
||||
"is_diskann_plugin_loaded",
|
||||
"is_libaio_available",
|
||||
"DISKANN_PLUGIN_OK",
|
||||
"DISKANN_PLUGIN_UNSUPPORTED_PLATFORM",
|
||||
"DISKANN_PLUGIN_LIBAIO_MISSING",
|
||||
"DISKANN_PLUGIN_DLOPEN_FAILED",
|
||||
]
|
||||
|
||||
# ==============================
|
||||
|
||||
@@ -16,6 +16,8 @@ from .model.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
DiskAnnIndexParam,
|
||||
DiskAnnQueryParam,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
@@ -52,6 +54,8 @@ __all__: list = [
|
||||
"CollectionStats",
|
||||
"DataType",
|
||||
"DenseEmbeddingFunction",
|
||||
"DiskAnnIndexParam",
|
||||
"DiskAnnQueryParam",
|
||||
"Doc",
|
||||
"DocList",
|
||||
"FieldSchema",
|
||||
|
||||
@@ -17,6 +17,8 @@ from _zvec.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
DiskAnnIndexParam,
|
||||
DiskAnnQueryParam,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
FtsQueryParam,
|
||||
@@ -37,6 +39,8 @@ __all__ = [
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"CollectionOption",
|
||||
"DiskAnnIndexParam",
|
||||
"DiskAnnQueryParam",
|
||||
"FlatIndexParam",
|
||||
"FtsIndexParam",
|
||||
"FtsQueryParam",
|
||||
|
||||
@@ -93,6 +93,7 @@ class Kmc2CentroidsGenerator {
|
||||
auto *centroids = owner->mutable_centroids();
|
||||
|
||||
std::mt19937 mt((std::random_device())());
|
||||
|
||||
std::uniform_real_distribution<float> dist(0.0, 1.0);
|
||||
|
||||
ContainerType benches(cache.dimension());
|
||||
@@ -1051,4 +1052,4 @@ using NibbleInnerProductKmeans =
|
||||
LloydCluster<T, TPool, TContext, NibbleVectorArray<T>>;
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
@@ -361,4 +361,4 @@ class LloydCluster {
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
@@ -83,6 +83,11 @@ set_target_properties(zvec_c_api PROPERTIES
|
||||
VISIBILITY_INLINES_HIDDEN ON
|
||||
)
|
||||
|
||||
# On Windows, define ZVEC_BUILD_SHARED so that c_api.h uses __declspec(dllexport)
|
||||
if(MSVC OR WIN32)
|
||||
target_compile_definitions(zvec_c_api PRIVATE ZVEC_BUILD_SHARED)
|
||||
endif()
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# Static linking of C++ standard library is handled in platform-specific sections
|
||||
|
||||
@@ -19,6 +19,33 @@ set(SRC_LISTS
|
||||
|
||||
pybind11_add_module(_zvec ${SRC_LISTS})
|
||||
|
||||
# pybind11_add_module() defaults to CXX_VISIBILITY_PRESET=hidden +
|
||||
# VISIBILITY_INLINES_HIDDEN=ON, which hides the compiler-generated helper
|
||||
# symbols attached to inline functions (guard variables for static locals,
|
||||
# vtables, typeinfo ...). The DiskAnn runtime plugin has its own copy of
|
||||
# Factory<IndexStreamer>::Instance()'s guard variable; if _zvec.so's copy
|
||||
# is hidden, the two guards are separate and the factory constructor runs
|
||||
# twice during plugin load, wiping out the registrations that happened
|
||||
# during _zvec.so import. We switch to default visibility here and rely on
|
||||
# the version script (exports.map / exports.mac) to keep the dynamic
|
||||
# symbol table small by exporting only zvec::* and PyInit_*.
|
||||
set_target_properties(_zvec PROPERTIES
|
||||
CXX_VISIBILITY_PRESET default
|
||||
C_VISIBILITY_PRESET default
|
||||
VISIBILITY_INLINES_HIDDEN OFF)
|
||||
|
||||
# Ensure any change to the linker version script (exports.map) triggers a
|
||||
# re-link of _zvec.so. target_link_options() alone is a command-line flag
|
||||
# and does not register the script file as a build dependency, so stale
|
||||
# wheels would silently keep the old export set otherwise.
|
||||
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
set_property(TARGET _zvec APPEND PROPERTY LINK_DEPENDS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/exports.map")
|
||||
elseif (APPLE)
|
||||
set_property(TARGET _zvec APPEND PROPERTY LINK_DEPENDS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/exports.mac")
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
-Wl,--whole-archive
|
||||
@@ -34,12 +61,29 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
$<TARGET_FILE:core_metric_static>
|
||||
$<TARGET_FILE:core_utility_static>
|
||||
$<TARGET_FILE:core_quantizer_static>
|
||||
$<TARGET_FILE:core_plugin>
|
||||
-Wl,--no-whole-archive
|
||||
zvec_db
|
||||
${CMAKE_DL_LIBS}
|
||||
)
|
||||
target_link_options(_zvec PRIVATE
|
||||
"LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map"
|
||||
)
|
||||
# DiskAnn is x86-only (it depends on libaio, unavailable on ARM64), so the
|
||||
# runtime plugin (libzvec_diskann_plugin.so) is only produced on non-ARM
|
||||
# builds. It is shipped as a runtime-loaded shared module and brought up
|
||||
# implicitly the first time a DiskAnn index is created — users never need to
|
||||
# call any load function. If libaio is missing at runtime the auto-load
|
||||
# fails cleanly and the error is surfaced only when DiskAnn is actually
|
||||
# used; other index types (HNSW/IVF/Flat/Vamana) remain fully functional.
|
||||
# The .so must therefore be discoverable next to the extension module,
|
||||
# hence the $ORIGIN rpath below.
|
||||
if (NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|arm")
|
||||
set_target_properties(_zvec PROPERTIES
|
||||
BUILD_RPATH "$ORIGIN"
|
||||
INSTALL_RPATH "$ORIGIN"
|
||||
)
|
||||
endif()
|
||||
elseif (APPLE)
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_static>
|
||||
@@ -54,6 +98,7 @@ elseif (APPLE)
|
||||
-Wl,-force_load,$<TARGET_FILE:core_metric_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_utility_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_quantizer_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_plugin>
|
||||
zvec_db
|
||||
)
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
@@ -72,6 +117,7 @@ elseif (MSVC)
|
||||
core_metric_static
|
||||
core_utility_static
|
||||
core_quantizer_static
|
||||
core_plugin
|
||||
)
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
${_zvec_whole_archive_libs}
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <zvec/plugin/diskann_plugin.h>
|
||||
#include "python_collection.h"
|
||||
#include "python_config.h"
|
||||
#include "python_doc.h"
|
||||
@@ -21,6 +23,40 @@
|
||||
#include "python_type.h"
|
||||
|
||||
namespace zvec {
|
||||
|
||||
namespace {
|
||||
|
||||
// Expose DiskAnn plugin management to Python. The DiskAnn runtime normally
|
||||
// auto-loads on first use, but tests (and diagnostic tooling) need a way to
|
||||
// force a load up-front and get actionable errors when libaio is missing or
|
||||
// the plugin shared object cannot be located.
|
||||
void InitializeDiskAnnPluginBindings(pybind11::module_ &m) {
|
||||
m.def(
|
||||
"load_diskann_plugin",
|
||||
[](const std::string &path) { return ::zvec::LoadDiskAnnPlugin(path); },
|
||||
pybind11::arg("path") = std::string(),
|
||||
"Load the DiskAnn runtime plugin. Returns 0 on success or a negative "
|
||||
"DiskAnnPluginStatus code on failure (unsupported platform, libaio "
|
||||
"missing, or dlopen failure).");
|
||||
m.def("is_diskann_plugin_loaded", &::zvec::IsDiskAnnPluginLoaded,
|
||||
"Return True if the DiskAnn runtime plugin is currently loaded.");
|
||||
m.def("is_libaio_available", &::zvec::IsLibAioAvailable,
|
||||
"Return True if libaio is resolvable on this host (required by the "
|
||||
"DiskAnn runtime).");
|
||||
|
||||
// Status constants so callers can compare against well-known codes without
|
||||
// hard-coding integers.
|
||||
m.attr("DISKANN_PLUGIN_OK") = static_cast<int>(::zvec::kDiskAnnPluginOk);
|
||||
m.attr("DISKANN_PLUGIN_UNSUPPORTED_PLATFORM") =
|
||||
static_cast<int>(::zvec::kDiskAnnPluginUnsupportedPlatform);
|
||||
m.attr("DISKANN_PLUGIN_LIBAIO_MISSING") =
|
||||
static_cast<int>(::zvec::kDiskAnnPluginLibAioMissing);
|
||||
m.attr("DISKANN_PLUGIN_DLOPEN_FAILED") =
|
||||
static_cast<int>(::zvec::kDiskAnnPluginDlopenFailed);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PYBIND11_MODULE(_zvec, m) {
|
||||
m.doc() = "Zvec core module";
|
||||
|
||||
@@ -31,5 +67,6 @@ PYBIND11_MODULE(_zvec, m) {
|
||||
ZVecPyConfig::Initialize(m);
|
||||
ZVecPyDoc::Initialize(m);
|
||||
ZVecPyCollection::Initialize(m);
|
||||
InitializeDiskAnnPluginBindings(m);
|
||||
}
|
||||
} // namespace zvec
|
||||
|
||||
@@ -1,6 +1,34 @@
|
||||
{
|
||||
global:
|
||||
PyInit_*; # export only python related functions
|
||||
# Python module entry point(s).
|
||||
PyInit_*;
|
||||
|
||||
# Expose the full zvec C++ namespace so the DiskAnn runtime plugin
|
||||
# (libzvec_diskann_plugin.so), which is dlopen()ed with
|
||||
# RTLD_NOW | RTLD_GLOBAL after the interpreter has loaded _zvec.so,
|
||||
# can resolve its undefined references against this module. Without
|
||||
# this, the plugin fails to load with errors like
|
||||
# undefined symbol: _ZN4zvec6ailego6Logger10LEVEL_INFOE
|
||||
# because the default version script hides every internal symbol.
|
||||
extern "C++" {
|
||||
"zvec::*";
|
||||
zvec::*;
|
||||
# Also export the compiler-generated helper symbols that live
|
||||
# alongside symbols in the zvec namespace (guard variables for
|
||||
# static locals, vtables, typeinfo, VTT, construction vtables,
|
||||
# thunks). Without these, the DiskAnn plugin and _zvec.so each
|
||||
# get their own copy of e.g. the guard for Factory<T>::Instance's
|
||||
# static local, which causes the factory constructor to run
|
||||
# twice - wiping out registrations done during _zvec.so load.
|
||||
"guard variable for zvec::*";
|
||||
"vtable for zvec::*";
|
||||
"VTT for zvec::*";
|
||||
"typeinfo for zvec::*";
|
||||
"typeinfo name for zvec::*";
|
||||
"construction vtable for zvec::*";
|
||||
"non-virtual thunk to zvec::*";
|
||||
"virtual thunk to zvec::*";
|
||||
};
|
||||
local:
|
||||
*;
|
||||
};
|
||||
|
||||
@@ -34,6 +34,8 @@ static std::string index_type_to_string(const IndexType type) {
|
||||
return "HNSW";
|
||||
case IndexType::HNSW_RABITQ:
|
||||
return "HNSW_RABITQ";
|
||||
case IndexType::DISKANN:
|
||||
return "DISKANN";
|
||||
case IndexType::VAMANA:
|
||||
return "VAMANA";
|
||||
case IndexType::FTS:
|
||||
@@ -876,6 +878,112 @@ Args:
|
||||
t[0].cast<MetricType>(), t[1].cast<int>(), t[2].cast<int>(),
|
||||
t[3].cast<bool>(), t[4].cast<QuantizeType>());
|
||||
}));
|
||||
|
||||
// DiskAnnIndexParams
|
||||
py::class_<DiskAnnIndexParams, VectorIndexParams,
|
||||
std::shared_ptr<DiskAnnIndexParams>>
|
||||
diskann_params(m, "DiskAnnIndexParam", R"pbdoc(
|
||||
Parameters for configuring an DiskAnn index.
|
||||
|
||||
DiskAnn stores compressed vector in memory and high-definition vector on disk. At query time,
|
||||
only compressed vector will be loaded into memory. By this way, search memory at runtime is diminished.
|
||||
|
||||
Attributes:
|
||||
metric_type (MetricType): Distance metric used for similarity computation.
|
||||
Default is ``MetricType.IP`` (inner product).
|
||||
max_degree (int): Maximum out-degree of each node in the Vamana graph.
|
||||
Larger values improve recall at the cost of build time and index size.
|
||||
Clamped to the range [1, 100]. Default is 100.
|
||||
list_size (int): Candidate list size used during graph construction.
|
||||
Larger values improve graph quality and recall at the cost of build time.
|
||||
Clamped to the range [10, 100]. Default is 50.
|
||||
pq_chunk_num (int): Number of PQ chunks used for product-quantizing the
|
||||
in-memory compressed vectors. ``0`` means auto-pick based on dimension.
|
||||
Clamped to the range [1, 1024]. Default is 0.
|
||||
quantize_type (QuantizeType): Optional quantization type for vector
|
||||
compression (e.g., FP16, INT8). Default is ``QuantizeType.UNDEFINED``.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import MetricType, QuantizeType
|
||||
>>> params = DiskAnnIndexParam(
|
||||
... metric_type=MetricType.COSINE,
|
||||
... max_degree=100,
|
||||
... list_size=50,
|
||||
... pq_chunk_num=8,
|
||||
... quantize_type=QuantizeType.FP16
|
||||
... )
|
||||
>>> print(params.max_degree)
|
||||
100
|
||||
)pbdoc");
|
||||
diskann_params
|
||||
.def(py::init<MetricType, int, int, int, QuantizeType>(),
|
||||
py::arg("metric_type") = MetricType::IP, py::arg("max_degree") = 100,
|
||||
py::arg("list_size") = 100, py::arg("pq_chunk_num") = 0,
|
||||
py::arg("quantize_type") = QuantizeType::UNDEFINED,
|
||||
R"pbdoc(
|
||||
Constructs an DiskAnnIndexParams instance.
|
||||
|
||||
Args:
|
||||
metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
|
||||
max_degree (int, optional): Maximum out-degree of each node in the Vamana
|
||||
graph. Clamped to [1, 100]. Defaults to 100.
|
||||
list_size (int, optional): Candidate list size used during graph
|
||||
construction. Clamped to [10, 100]. Defaults to 50.
|
||||
pq_chunk_num (int, optional): Number of PQ chunks for product
|
||||
quantization. ``0`` means auto-pick based on dimension.
|
||||
Clamped to [1, 1024]. Defaults to 0.
|
||||
quantize_type (QuantizeType, optional): Vector quantization type.
|
||||
Defaults to QuantizeType.UNDEFINED.
|
||||
)pbdoc")
|
||||
.def_property_readonly("max_degree", &DiskAnnIndexParams::max_degree,
|
||||
"int: max node degree.")
|
||||
.def_property_readonly("list_size", &DiskAnnIndexParams::list_size,
|
||||
"int: list size of graph construction")
|
||||
.def_property_readonly(
|
||||
"pq_chunk_num",
|
||||
[](const DiskAnnIndexParams &self) -> int {
|
||||
return self.pq_chunk_num();
|
||||
},
|
||||
"int: chunk num of production quantization.")
|
||||
.def(
|
||||
"to_dict",
|
||||
[](const DiskAnnIndexParams &self) -> py::dict {
|
||||
py::dict dict;
|
||||
dict["type"] = index_type_to_string(self.type());
|
||||
dict["metric_type"] = metric_type_to_string(self.metric_type());
|
||||
dict["max_degree"] = self.max_degree();
|
||||
dict["list_size"] = self.list_size();
|
||||
dict["pq_chunk_num"] = self.pq_chunk_num();
|
||||
dict["quantize_type"] =
|
||||
quantize_type_to_string(self.quantize_type());
|
||||
return dict;
|
||||
},
|
||||
"Convert to dictionary with all fields")
|
||||
.def(
|
||||
"__repr__",
|
||||
[](const DiskAnnIndexParams &self) {
|
||||
return "{"
|
||||
"\"metric_type\":" +
|
||||
metric_type_to_string(self.metric_type()) +
|
||||
", \"max_degree\":" + std::to_string(self.max_degree()) +
|
||||
", \"list_size\":" + std::to_string(self.list_size()) +
|
||||
", \"pq_chunk_num\":" + std::to_string(self.pq_chunk_num()) +
|
||||
", \"quantize_type\":" +
|
||||
quantize_type_to_string(self.quantize_type()) + "}";
|
||||
})
|
||||
.def(py::pickle(
|
||||
[](const DiskAnnIndexParams &self) {
|
||||
return py::make_tuple(self.metric_type(), self.max_degree(),
|
||||
self.list_size(), self.pq_chunk_num(),
|
||||
self.quantize_type());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 5)
|
||||
throw std::runtime_error("Invalid state for DiskAnnIndexParams");
|
||||
return std::make_shared<DiskAnnIndexParams>(
|
||||
t[0].cast<MetricType>(), t[1].cast<int>(), t[2].cast<int>(),
|
||||
t[3].cast<int>(), t[4].cast<QuantizeType>());
|
||||
}));
|
||||
}
|
||||
|
||||
void ZVecPyParams::bind_query_params(py::module_ &m) {
|
||||
@@ -1118,6 +1226,54 @@ Args:
|
||||
return obj;
|
||||
}));
|
||||
|
||||
// binding diskann query params
|
||||
py::class_<DiskAnnQueryParams, QueryParams,
|
||||
std::shared_ptr<DiskAnnQueryParams>>
|
||||
diskann_params(m, "DiskAnnQueryParam", R"pbdoc(
|
||||
Query parameters for DiskAnn index.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.DISKANN``.
|
||||
list_size (int): Beam-search candidate list size used at query time.
|
||||
Higher values improve recall but increase latency. Default is 10.
|
||||
|
||||
Examples:
|
||||
>>> params = DiskAnnQueryParam(list_size=20)
|
||||
>>> print(params.list_size)
|
||||
20
|
||||
)pbdoc");
|
||||
diskann_params
|
||||
.def(py::init<int>(), py::arg("list_size") = 300, R"pbdoc(
|
||||
Constructs an DiskAnnQueryParams instance.
|
||||
|
||||
Args:
|
||||
list_size (int, optional): Beam-search candidate list size during
|
||||
graph search. Higher values improve recall at the cost of latency.
|
||||
Defaults to 300.
|
||||
)pbdoc")
|
||||
.def_property_readonly(
|
||||
"list_size",
|
||||
[](const DiskAnnQueryParams &self) -> int {
|
||||
return self.list_size();
|
||||
},
|
||||
"int: Beam-search candidate list size during DiskAnn query.")
|
||||
.def("__repr__",
|
||||
[](const DiskAnnQueryParams &self) -> std::string {
|
||||
return "{"
|
||||
"\"type\":" +
|
||||
index_type_to_string(self.type()) +
|
||||
", \"list_size\":" + std::to_string(self.list_size()) + "}";
|
||||
})
|
||||
.def(py::pickle(
|
||||
[](const DiskAnnQueryParams &self) {
|
||||
return py::make_tuple(self.list_size());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 1)
|
||||
throw std::runtime_error("Invalid state for DiskAnnQueryParams");
|
||||
return std::make_shared<DiskAnnQueryParams>(t[0].cast<int>());
|
||||
}));
|
||||
|
||||
// binding vamana query params
|
||||
py::class_<VamanaQueryParams, QueryParams, std::shared_ptr<VamanaQueryParams>>
|
||||
vamana_query_params(m, "VamanaQueryParam", R"pbdoc(
|
||||
|
||||
+18
-1
@@ -28,6 +28,7 @@ cc_directory(quantizer)
|
||||
cc_directory(utility)
|
||||
cc_directory(interface)
|
||||
cc_directory(mixed_reducer)
|
||||
cc_directory(plugin)
|
||||
|
||||
git_version(GIT_SRCS_VER ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
file(GLOB_RECURSE ALL_CORE_SRCS *.cc *.c *.h)
|
||||
@@ -39,10 +40,26 @@ if(NOT RABITQ_SUPPORTED)
|
||||
list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/algorithm/hnsw_rabitq/.*")
|
||||
endif()
|
||||
|
||||
# Always exclude algorithm/diskann implementation files from zvec_core.
|
||||
# The DiskAnn algorithm is provided by the separate core_knn_diskann library
|
||||
# (real on Linux x86_64, stub on other platforms). Including them here causes
|
||||
# duplicate symbols and missing -laio when test binaries link both zvec_core
|
||||
# (via zvec_db) and core_knn_diskann.
|
||||
list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/algorithm/diskann/.*")
|
||||
if(NOT DISKANN_SUPPORTED)
|
||||
list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/interface/indexes/diskann_index\\.cc")
|
||||
endif()
|
||||
|
||||
set(ZVEC_CORE_LIBS zvec_ailego zvec_turbo sparsehash magic_enum rabitqlib)
|
||||
# The plugin loader uses dlopen/dlsym, so link libdl on Linux.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
list(APPEND ZVEC_CORE_LIBS ${CMAKE_DL_LIBS})
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME zvec_core STATIC STRICT PACKED
|
||||
SRCS ${ALL_CORE_SRCS}
|
||||
LIBS zvec_ailego zvec_turbo sparsehash magic_enum rabitqlib
|
||||
LIBS ${ZVEC_CORE_LIBS}
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core
|
||||
VERSION "${GIT_SRCS_VER}"
|
||||
)
|
||||
|
||||
@@ -8,6 +8,42 @@ cc_directory(ivf)
|
||||
cc_directory(hnsw)
|
||||
cc_directory(hnsw_sparse)
|
||||
cc_directory(vamana)
|
||||
|
||||
if(DISKANN_SUPPORTED)
|
||||
message(STATUS "build diskann")
|
||||
cc_directory(diskann)
|
||||
else()
|
||||
message(STATUS "not build diskann")
|
||||
# Empty stub library for unsupported platforms
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
"// Stub implementation for unsupported platforms\n"
|
||||
"// DiskAnn only supports Linux x86_64\n"
|
||||
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
|
||||
)
|
||||
|
||||
if(MSVC)
|
||||
# MSVC: STATIC-only stub to avoid creating an empty DLL with no exports
|
||||
# (MSVC linker fails to produce an import library when there are zero exports)
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
STATIC STRICT ALWAYS_LINK
|
||||
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
else()
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(RABITQ_SUPPORTED)
|
||||
message(STATUS "BUILD RABITQ")
|
||||
cc_directory(hnsw_rabitq)
|
||||
|
||||
@@ -19,189 +19,195 @@ namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! General
|
||||
static const std::string GENERAL_CLUSTER_COUNT =
|
||||
"proxima.general.cluster.count";
|
||||
static const std::string GENERAL_CLUSTER_COUNT = "zvec.general.cluster.count";
|
||||
static const std::string GENERAL_THREAD_COUNT =
|
||||
"proxima.general.cluster.thread_count";
|
||||
"zvec.general.cluster.thread_count";
|
||||
|
||||
//! Optimize K-means
|
||||
static const std::string OPTKMEANS_CLUSTER_COUNT =
|
||||
"proxima.optkmeans.cluster.count";
|
||||
"zvec.optkmeans.cluster.count";
|
||||
static const std::string OPTKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.optkmeans.cluster.max_iterations";
|
||||
"zvec.optkmeans.cluster.max_iterations";
|
||||
static const std::string OPTKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.optkmeans.cluster.epsilon";
|
||||
"zvec.optkmeans.cluster.epsilon";
|
||||
static const std::string OPTKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.optkmeans.cluster.shard_factor";
|
||||
"zvec.optkmeans.cluster.shard_factor";
|
||||
static const std::string OPTKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.optkmeans.cluster.purge_empty";
|
||||
"zvec.optkmeans.cluster.purge_empty";
|
||||
static const std::string OPTKMEANS_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.optkmeans.cluster.markov_chain_length";
|
||||
"zvec.optkmeans.cluster.markov_chain_length";
|
||||
static const std::string OPTKMEANS_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.optkmeans.cluster.assumption_free";
|
||||
"zvec.optkmeans.cluster.assumption_free";
|
||||
|
||||
//! K-means
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "proxima.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "zvec.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON =
|
||||
"proxima.kmeans.cluster.epsilon";
|
||||
"zvec.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON = "zvec.kmeans.cluster.epsilon";
|
||||
static const std::string KMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmeans.cluster.max_iterations";
|
||||
"zvec.kmeans.cluster.max_iterations";
|
||||
static const std::string KMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "proxima.kmeans.cluster.batch";
|
||||
"zvec.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "zvec.kmeans.cluster.batch";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmeans.cluster.seeker_class";
|
||||
"zvec.kmeans.cluster.seeker_class";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmeans.cluster.seeker_params";
|
||||
"zvec.kmeans.cluster.seeker_params";
|
||||
|
||||
//! Mini Batch K-means
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.count";
|
||||
"zvec.minibatchkmeans.cluster.count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.minibatchkmeans.cluster.shard_factor";
|
||||
"zvec.minibatchkmeans.cluster.shard_factor";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.minibatchkmeans.cluster.epsilon";
|
||||
"zvec.minibatchkmeans.cluster.epsilon";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.minibatchkmeans.cluster.max_iterations";
|
||||
"zvec.minibatchkmeans.cluster.max_iterations";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.minibatchkmeans.cluster.purge_empty";
|
||||
"zvec.minibatchkmeans.cluster.purge_empty";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_TRY_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.try_count";
|
||||
"zvec.minibatchkmeans.cluster.try_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_BATCH_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.batch_count";
|
||||
"zvec.minibatchkmeans.cluster.batch_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_class";
|
||||
"zvec.minibatchkmeans.cluster.seeker_class";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_params";
|
||||
"zvec.minibatchkmeans.cluster.seeker_params";
|
||||
|
||||
//! K-means++
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT =
|
||||
"proxima.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT = "zvec.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS =
|
||||
"proxima.kmeanspp.cluster.class";
|
||||
"zvec.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS = "zvec.kmeanspp.cluster.class";
|
||||
static const std::string KMEANSPP_CLUSTER_PARAMS =
|
||||
"proxima.kmeanspp.cluster.params";
|
||||
"zvec.kmeanspp.cluster.params";
|
||||
|
||||
//! K-MC2
|
||||
static const std::string KMC2_CLUSTER_COUNT = "proxima.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_COUNT = "zvec.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmc2.cluster.shard_factor";
|
||||
"zvec.kmc2.cluster.shard_factor";
|
||||
static const std::string KMC2_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.kmc2.cluster.markov_chain_length";
|
||||
"zvec.kmc2.cluster.markov_chain_length";
|
||||
static const std::string KMC2_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "proxima.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "proxima.kmc2.cluster.params";
|
||||
"zvec.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "zvec.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "zvec.kmc2.cluster.params";
|
||||
|
||||
//! Bisecting K-means
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT =
|
||||
"proxima.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT = "zvec.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_INIT_COUNT =
|
||||
"proxima.bikmeans.cluster.init_count";
|
||||
"zvec.bikmeans.cluster.init_count";
|
||||
static const std::string BIKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.bikmeans.cluster.purge_empty";
|
||||
"zvec.bikmeans.cluster.purge_empty";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_CLASS =
|
||||
"proxima.bikmeans.cluster.first_class";
|
||||
"zvec.bikmeans.cluster.first_class";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_CLASS =
|
||||
"proxima.bikmeans.cluster.second_class";
|
||||
"zvec.bikmeans.cluster.second_class";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.bikmeans.cluster.first_params";
|
||||
"zvec.bikmeans.cluster.first_params";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.bikmeans.cluster.second_params";
|
||||
"zvec.bikmeans.cluster.second_params";
|
||||
|
||||
//! K-medoids
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT =
|
||||
"proxima.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT = "zvec.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmedoids.cluster.shard_factor";
|
||||
"zvec.kmedoids.cluster.shard_factor";
|
||||
static const std::string KMEDOIDS_CLUSTER_EPSILON =
|
||||
"proxima.kmedoids.cluster.epsilon";
|
||||
"zvec.kmedoids.cluster.epsilon";
|
||||
static const std::string KMEDOIDS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmedoids.cluster.max_iterations";
|
||||
"zvec.kmedoids.cluster.max_iterations";
|
||||
static const std::string KMEDOIDS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmedoids.cluster.purge_empty";
|
||||
"zvec.kmedoids.cluster.purge_empty";
|
||||
static const std::string KMEDOIDS_CLUSTER_BENCH_RATIO =
|
||||
"proxima.kmedoids.cluster.bench_ratio";
|
||||
"zvec.kmedoids.cluster.bench_ratio";
|
||||
static const std::string KMEDOIDS_CLUSTER_ONLY_MEANS =
|
||||
"proxima.kmedoids.cluster.only_means";
|
||||
"zvec.kmedoids.cluster.only_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_WITHOUT_MEANS =
|
||||
"proxima.kmedoids.cluster.without_means";
|
||||
"zvec.kmedoids.cluster.without_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmedoids.cluster.seeker_class";
|
||||
"zvec.kmedoids.cluster.seeker_class";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmedoids.cluster.seeker_params";
|
||||
"zvec.kmedoids.cluster.seeker_params";
|
||||
|
||||
//! Stratified
|
||||
static const std::string STRATIFIED_CLUSTER_COUNT =
|
||||
"proxima.stratified.cluster.count";
|
||||
"zvec.stratified.cluster.count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_CLASS =
|
||||
"proxima.stratified.cluster.first_class";
|
||||
"zvec.stratified.cluster.first_class";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_CLASS =
|
||||
"proxima.stratified.cluster.second_class";
|
||||
"zvec.stratified.cluster.second_class";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_COUNT =
|
||||
"proxima.stratified.cluster.first_count";
|
||||
"zvec.stratified.cluster.first_count";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_COUNT =
|
||||
"proxima.stratified.cluster.second_count";
|
||||
"zvec.stratified.cluster.second_count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.stratified.cluster.first_params";
|
||||
"zvec.stratified.cluster.first_params";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.stratified.cluster.second_params";
|
||||
"zvec.stratified.cluster.second_params";
|
||||
static const std::string STRATIFIED_CLUSTER_AUTO_TUNING =
|
||||
"proxima.stratified.cluster.auto_tuning";
|
||||
"zvec.stratified.cluster.auto_tuning";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_POOL_COUNT =
|
||||
"proxima.stratified.cluster.second_pool_count";
|
||||
"zvec.stratified.cluster.second_pool_count";
|
||||
|
||||
//! Gap Statistics
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN =
|
||||
"proxima.gapstats.cluster_estimater.k_min";
|
||||
"zvec.gapstats.cluster_estimater.k_min";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX =
|
||||
"proxima.gapstats.cluster_estimater.k_max";
|
||||
"zvec.gapstats.cluster_estimater.k_max";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_min_step";
|
||||
"zvec.gapstats.cluster_estimater.k_min_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_max_step";
|
||||
"zvec.gapstats.cluster_estimater.k_max_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_TRY_COUNT =
|
||||
"proxima.gapstats.cluster_estimater.try_count";
|
||||
"zvec.gapstats.cluster_estimater.try_count";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_SHARD_FACTOR =
|
||||
"proxima.gapstats.cluster_estimater.shard_factor";
|
||||
"zvec.gapstats.cluster_estimater.shard_factor";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_ENABLE_MC2 =
|
||||
"proxima.gapstats.cluster_estimater.enable_mc2";
|
||||
"zvec.gapstats.cluster_estimater.enable_mc2";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.gapstats.cluster_estimater.markov_chain_length";
|
||||
"zvec.gapstats.cluster_estimater.markov_chain_length";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_CLUSTER_CLASS =
|
||||
"proxima.gapstats.cluster_estimater.cluster_class";
|
||||
"zvec.gapstats.cluster_estimater.cluster_class";
|
||||
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.cluster.trainer.sample_count";
|
||||
"zvec.cluster.trainer.sample_count";
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.cluster.trainer.sample_ratio";
|
||||
"zvec.cluster.trainer.sample_ratio";
|
||||
static const std::string CLUSTER_TRAINER_THREAD_COUNT =
|
||||
"proxima.cluster.trainer.thread_count";
|
||||
"zvec.cluster.trainer.thread_count";
|
||||
static const std::string CLUSTER_TRAINER_FILE_NAME =
|
||||
"proxima.cluster.trainer.file_name";
|
||||
"zvec.cluster.trainer.file_name";
|
||||
static const std::string CLUSTER_TRAINER_CLASS_NAME =
|
||||
"proxima.cluster.trainer.class_name";
|
||||
"zvec.cluster.trainer.class_name";
|
||||
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.stratified.trainer.sample_count";
|
||||
"zvec.stratified.trainer.sample_count";
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.stratified.trainer.sample_ratio";
|
||||
"zvec.stratified.trainer.sample_ratio";
|
||||
static const std::string STRATIFIED_TRAINER_THREAD_COUNT =
|
||||
"proxima.stratified.trainer.thread_count";
|
||||
"zvec.stratified.trainer.thread_count";
|
||||
static const std::string STRATIFIED_TRAINER_FILE_NAME =
|
||||
"proxima.stratified.trainer.file_name";
|
||||
"zvec.stratified.trainer.file_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLASS_NAME =
|
||||
"proxima.stratified.trainer.class_name";
|
||||
"zvec.stratified.trainer.class_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLUSTER_COUNT =
|
||||
"proxima.stratified.trainer.cluster_count";
|
||||
"zvec.stratified.trainer.cluster_count";
|
||||
static const std::string STRATIFIED_TRAINER_AUTOAUNE =
|
||||
"proxima.stratified.trainer.autotune";
|
||||
"zvec.stratified.trainer.autotune";
|
||||
static const std::string STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX =
|
||||
"proxima.stratified.trainer.cluster_params_in_level_";
|
||||
"zvec.stratified.trainer.cluster_params_in_level_";
|
||||
|
||||
static const std::string MULTI_CHUNK_CLUSTER_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_CHUNK_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.chunk_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_THREAD_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.thread_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_EPSILON =
|
||||
"zvec.cluster.multi_chunk_cluster.epsilon";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MAX_ITERATIONS =
|
||||
"zvec.cluster.multi_chunk_cluster.max_iterations";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"zvec.cluster.multi_chunk_cluster.markov_chain_length";
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "multi_chunk_cluster.h"
|
||||
#include <chrono>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
bool MultiChunkClusterAlgorithm::is_valid(void) const {
|
||||
return features_ && features_->count();
|
||||
}
|
||||
|
||||
bool MultiChunkClusterAlgorithm::check_centroids(
|
||||
const IndexCluster::CentroidList & /*cents*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::init_centroids(
|
||||
size_t count, IndexCluster::CentroidList *out) {
|
||||
// Just resize, because the get random centroid step is done by cluster_once
|
||||
out->resize(count);
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::reset(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::cleanup(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::suggest(uint32_t k) {
|
||||
cluster_count_ = k;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::update(const ailego::Params ¶ms) {
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
int MultiChunkClusterAlgorithm::update_params(const ailego::Params ¶ms) {
|
||||
params.get(GENERAL_THREAD_COUNT, &thread_count_);
|
||||
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
|
||||
|
||||
params.get(MULTI_CHUNK_CLUSTER_THREAD_COUNT, &thread_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, &chunk_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_EPSILON, &epsilon_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, &max_iterations_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH, &markov_chain_length_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init_distance_func() {
|
||||
IndexMetric::Pointer metric_{};
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Create metric %s failed.", meta_.metric_name().c_str());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
int ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed wit ret %d.", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
distance_func_ = metric_->distance_matrix(1, 1);
|
||||
if (!distance_func_) {
|
||||
LOG_ERROR("DistanceMatrix function is nullptr.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::do_chunk() {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
size_t large_chunk_count = meta_.dimension() % chunk_count_;
|
||||
size_t base_chunk_dim_ = meta_.dimension() / chunk_count_;
|
||||
|
||||
chunk_dims_.clear();
|
||||
|
||||
for (size_t i = 0; i < chunk_count_; ++i) {
|
||||
if (i < large_chunk_count) {
|
||||
chunk_dims_.push_back(base_chunk_dim_ + 1);
|
||||
} else {
|
||||
chunk_dims_.push_back(base_chunk_dim_);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_dim_offsets_.clear();
|
||||
chunk_dim_offsets_.push_back(0);
|
||||
for (size_t i = 1; i < chunk_count_; ++i) {
|
||||
chunk_dim_offsets_.push_back(chunk_dim_offsets_[i - 1] +
|
||||
chunk_dims_[i - 1]);
|
||||
}
|
||||
chunk_dim_offsets_.push_back(meta_.dimension());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
|
||||
int ret = update_params(params);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = do_chunk();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = init_distance_func();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::mount(IndexFeatures::Pointer features) {
|
||||
if (!features) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!features->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
auto data_type = meta_.data_type();
|
||||
if (data_type != IndexMeta::DataType::DT_FP32 &&
|
||||
data_type != IndexMeta::DataType::DT_FP16) {
|
||||
LOG_ERROR("Unsupported meta type %u", data_type);
|
||||
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
features_ = std::move(features);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! cluster
|
||||
int MultiChunkClusterAlgorithm::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
cents.clear();
|
||||
cents.resize(chunk_count_ * cluster_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_cluster, i,
|
||||
threads->count(), ¢s, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish Chunk Count %zu, Finished Percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / chunk_count_);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkClusterAlgorithm::classify(
|
||||
IndexThreads::Pointer /*threads*/, IndexCluster::CentroidList & /*cents*/) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkClusterAlgorithm::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t features_count = features_->count();
|
||||
out->resize(features_count * chunk_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_label, i,
|
||||
threads->count(), cents, out, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish label cnt %zu, finished percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / features_count);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cluster
|
||||
int MultiChunkCluster::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->cluster(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkCluster::classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->classify(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkCluster::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
return algorithm_->label(std::move(threads), cents, out);
|
||||
}
|
||||
|
||||
//! Update Cluster
|
||||
int MultiChunkCluster::update(const ailego::Params ¶ms) {
|
||||
return algorithm_->update(params);
|
||||
}
|
||||
|
||||
//! Reset Cluster
|
||||
int MultiChunkCluster::reset(void) {
|
||||
return algorithm_->reset();
|
||||
}
|
||||
|
||||
//! Cleanup Cluster
|
||||
int MultiChunkCluster::cleanup(void) {
|
||||
return algorithm_->cleanup();
|
||||
}
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void MultiChunkCluster::suggest(uint32_t k) {
|
||||
algorithm_->suggest(k);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::mount(IndexFeatures::Pointer feats) {
|
||||
return algorithm_->mount(feats);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
IndexMeta new_meta(meta.data_type(), meta.dimension());
|
||||
|
||||
if (meta.metric_name() == "Cosine") {
|
||||
new_meta.set_dimension(meta.dimension() - 1);
|
||||
new_meta.set_metric("InnerProduct", 0, ailego::Params());
|
||||
}
|
||||
|
||||
auto data_type = new_meta.data_type();
|
||||
|
||||
if (new_meta.metric_name() == "InnerProduct") {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(
|
||||
new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
algorithm_->init(new_meta, params);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,466 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/algorithm/kmeans.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/core/framework/index_cluster.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_factory.h>
|
||||
#include "cluster_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! MultiChunkClusterAlgorithm
|
||||
class MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
typedef std::shared_ptr<MultiChunkClusterAlgorithm> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
virtual ~MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return chunk_dims_;
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return chunk_dim_offsets_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Check Centroids
|
||||
bool check_centroids(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Test if it is valid
|
||||
bool is_valid() const;
|
||||
|
||||
//! Do chunk
|
||||
int do_chunk();
|
||||
|
||||
//! Update parameters
|
||||
int update_params(const ailego::Params ¶ms);
|
||||
|
||||
int init_distance_func();
|
||||
|
||||
//! cluster thread
|
||||
virtual void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! label thread
|
||||
virtual void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! Initialize Centroids
|
||||
void init_centroids(size_t count, IndexCluster::CentroidList *out);
|
||||
|
||||
private:
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
|
||||
protected:
|
||||
uint32_t cluster_count_{0u};
|
||||
uint32_t thread_count_{0u};
|
||||
uint32_t chunk_count_{0u};
|
||||
uint32_t max_iterations_{20u};
|
||||
bool assumption_free_{false};
|
||||
uint32_t markov_chain_length_{32};
|
||||
double epsilon_{std::numeric_limits<float>::epsilon()};
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
|
||||
IndexMeta meta_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
|
||||
std::vector<uint32_t> chunk_dims_;
|
||||
std::vector<uint32_t> chunk_dim_offsets_;
|
||||
|
||||
IndexMetric::MatrixDistance distance_func_{nullptr};
|
||||
};
|
||||
|
||||
/*! Numerical cluster algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalAlgorithm : public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalKmeans<T, IndexThreads> algorithm(cluster_count_,
|
||||
chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<ailego::NumericalKmeans<T, IndexThreads>,
|
||||
IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%u) Failed to cluster.", i + 1);
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = static_cast<uint32_t>(sel_index);
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
/*! Inner Product Cluster Algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalInnerProductAlgorithm
|
||||
: public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t chunk_step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads> algorithm(
|
||||
cluster_count_, chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads>, IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%zu) Failed to cluster.", (size_t)(i + 1));
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = sel_index;
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
class MultiChunkCluster {
|
||||
public:
|
||||
std::shared_ptr<MultiChunkCluster> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkCluster() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkCluster() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return algorithm_->chunk_dims();
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return algorithm_->chunk_dim_offsets();
|
||||
}
|
||||
|
||||
protected:
|
||||
MultiChunkClusterAlgorithm::Pointer algorithm_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,57 @@
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
|
||||
|
||||
file(GLOB_RECURSE ALL_SRCS *.cc *.c)
|
||||
|
||||
# The DiskAnn plugin is loaded at runtime via zvec::LoadDiskAnnPlugin() with
|
||||
# RTLD_GLOBAL, so its undefined references to internal zvec symbols
|
||||
# (core_framework, core_knn_cluster, zvec_ailego, ...) are resolved at load
|
||||
# time against the hosting binary (_zvec.so for the Python extension, the
|
||||
# test executable for gtest, or libzvec_core for tools).
|
||||
#
|
||||
# As a consequence the plugin .so must NOT carry NEEDED entries for those
|
||||
# libs: otherwise the dynamic loader would try to resolve them from disk
|
||||
# before invoking dlopen's RTLD_GLOBAL symbol sharing, and the Python wheel
|
||||
# (which only ships _zvec.so + the plugin) would fail to load.
|
||||
#
|
||||
# We therefore link the plugin only against its system-level dependency
|
||||
# (libaio) and rely on the global include dirs plus -Wl,--unresolved-symbols
|
||||
# =ignore-all to let the linker build a shared library with unresolved
|
||||
# references to internal APIs.
|
||||
set(CORE_KNN_DISKANN_LIBS "")
|
||||
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386")
|
||||
list(APPEND CORE_KNN_DISKANN_LIBS aio)
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
SHARED STRICT
|
||||
SRCS *.cc
|
||||
LIBS ${CORE_KNN_DISKANN_LIBS}
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
||||
# Internal zvec libs are referenced only for header availability; the actual
|
||||
# symbol resolution happens at dlopen time. Expose their public include dirs
|
||||
# without adding them as link dependencies.
|
||||
foreach(_dep zvec_ailego core_framework core_knn_cluster)
|
||||
if(TARGET ${_dep})
|
||||
target_include_directories(core_knn_diskann PRIVATE
|
||||
$<TARGET_PROPERTY:${_dep},INTERFACE_INCLUDE_DIRECTORIES>)
|
||||
add_dependencies(core_knn_diskann ${_dep})
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
# Allow the plugin to have unresolved symbols that will be satisfied at
|
||||
# dlopen(RTLD_NOW | RTLD_GLOBAL) time by the hosting binary.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
target_link_options(core_knn_diskann PRIVATE
|
||||
"LINKER:--unresolved-symbols=ignore-all")
|
||||
endif()
|
||||
|
||||
# Rename the artifact to libzvec_diskann_plugin.so so it is clearly identified
|
||||
# as an optional plugin that users load at runtime via zvec::LoadDiskAnnPlugin().
|
||||
set_target_properties(core_knn_diskann PROPERTIES
|
||||
OUTPUT_NAME zvec_diskann_plugin
|
||||
)
|
||||
@@ -0,0 +1,324 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_algorithm.h"
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnAlgorithm::DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree)
|
||||
: entity_(entity), max_degree_(max_degree), lock_pool_(kLockCnt) {}
|
||||
|
||||
std::vector<diskann_id_t> DiskAnnAlgorithm::get_init_ids(DiskAnnContext *ctx) {
|
||||
const auto &entity = ctx->get_entity();
|
||||
|
||||
std::vector<diskann_id_t> init_ids;
|
||||
|
||||
init_ids.emplace_back(entity.medoid());
|
||||
|
||||
return init_ids;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::add_node(diskann_id_t id, DiskAnnContext *ctx) {
|
||||
const void *vec = entity_.get_vector(id);
|
||||
|
||||
ctx->reset_query(vec);
|
||||
|
||||
std::vector<diskann_id_t> pruned_list;
|
||||
|
||||
int ret = search_neighbor_and_prune(id, pruned_list, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
{
|
||||
auto lock = lock_for(id);
|
||||
entity_.set_neighbors(id, pruned_list);
|
||||
}
|
||||
|
||||
return inter_insert(id, pruned_list, ctx);
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::prune_node(diskann_id_t id, DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
auto neighbors = entity_.get_neighbors(id);
|
||||
|
||||
if (neighbors.first > max_degree_) {
|
||||
std::set<diskann_id_t> dummy_visited;
|
||||
std::vector<Neighbor> dummy_pool(0);
|
||||
std::vector<diskann_id_t> new_out_neighbors;
|
||||
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
diskann_id_t node_id = (neighbors.second)[i];
|
||||
|
||||
auto itr = dummy_visited.find(node_id);
|
||||
if (itr == dummy_visited.end() && node_id != id) {
|
||||
float dist = dc.dist(id, node_id);
|
||||
|
||||
dummy_pool.emplace_back(Neighbor(node_id, dist));
|
||||
dummy_visited.insert(node_id);
|
||||
}
|
||||
}
|
||||
|
||||
prune_neighbors(id, dummy_pool, new_out_neighbors, ctx);
|
||||
|
||||
{
|
||||
auto lock = lock_for(id);
|
||||
entity_.set_neighbors(id, new_out_neighbors);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::inter_insert(diskann_id_t id,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
for (auto &des : pruned_list) {
|
||||
std::vector<diskann_id_t> new_neighbors;
|
||||
bool need_prune = false;
|
||||
|
||||
{
|
||||
auto lock = lock_for(des);
|
||||
|
||||
auto neighbors = entity_.get_neighbors(des);
|
||||
|
||||
bool found = false;
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
if ((neighbors.second)[i] == id) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
if (neighbors.first <
|
||||
static_cast<uint64_t>(DiskAnnEntity::kDefaultGraphSlackFactor *
|
||||
max_degree_)) {
|
||||
entity_.add_neighbor(des, id);
|
||||
need_prune = false;
|
||||
} else {
|
||||
new_neighbors.resize(neighbors.first + 1);
|
||||
memcpy(&new_neighbors[0], neighbors.second,
|
||||
sizeof(diskann_id_t) * neighbors.first);
|
||||
|
||||
new_neighbors[neighbors.first] = id;
|
||||
|
||||
need_prune = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (need_prune) {
|
||||
std::set<diskann_id_t> new_visited;
|
||||
std::vector<Neighbor> new_pool(0);
|
||||
|
||||
size_t reserve_size = static_cast<size_t>(std::ceil(
|
||||
1.05 * DiskAnnEntity::kDefaultGraphSlackFactor * max_degree_));
|
||||
|
||||
new_pool.reserve(reserve_size);
|
||||
|
||||
for (auto node_id : new_neighbors) {
|
||||
if (new_visited.find(node_id) == new_visited.end() && node_id != des) {
|
||||
float dist = dc.dist(des, node_id);
|
||||
new_pool.emplace_back(Neighbor(node_id, dist));
|
||||
new_visited.insert(node_id);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<diskann_id_t> new_pruned_neighbors;
|
||||
prune_neighbors(des, new_pool, new_pruned_neighbors, ctx);
|
||||
|
||||
{
|
||||
auto lock = lock_for(des);
|
||||
entity_.set_neighbors(des, new_pruned_neighbors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::iterate_to_fixed_point(
|
||||
const std::vector<diskann_id_t> &init_ids, DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
std::vector<Neighbor> &expanded_nodes = ctx->expanded_nodes();
|
||||
NeighborPriorityQueue &best_list_nodes = ctx->best_list_nodes();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
|
||||
best_list_nodes.reserve(ctx->list_size());
|
||||
|
||||
for (auto id : init_ids) {
|
||||
const void *vec = entity_.get_vector(id);
|
||||
|
||||
float distance = dc.dist(vec);
|
||||
|
||||
Neighbor nn = Neighbor(id, distance);
|
||||
best_list_nodes.insert(nn);
|
||||
}
|
||||
|
||||
while (best_list_nodes.has_unexpanded_node()) {
|
||||
auto neighbor = best_list_nodes.closest_unexpanded();
|
||||
auto node_id = neighbor.id;
|
||||
|
||||
expanded_nodes.emplace_back(neighbor);
|
||||
|
||||
std::vector<diskann_id_t> id_scratch;
|
||||
{
|
||||
auto lock = lock_for(node_id);
|
||||
auto neighbors = entity_.get_neighbors(node_id);
|
||||
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
diskann_id_t neighbor_id = (neighbors.second)[i];
|
||||
|
||||
if (!visit.visited(neighbor_id)) {
|
||||
id_scratch.push_back(neighbor_id);
|
||||
|
||||
visit.set_visited(neighbor_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < id_scratch.size(); ++i) {
|
||||
diskann_id_t id = id_scratch[i];
|
||||
|
||||
const void *vec = entity_.get_vector(id);
|
||||
float dist = dc.dist(vec);
|
||||
|
||||
best_list_nodes.insert(Neighbor(id, dist));
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &result,
|
||||
DiskAnnContext *ctx) {
|
||||
if (pool.size() == 0) return 0;
|
||||
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
ailego_assert(std::is_sorted(pool.begin(), pool.end()));
|
||||
ailego_assert(result.size() == 0);
|
||||
|
||||
if (pool.size() > max_candidate_size_) {
|
||||
pool.resize(max_candidate_size_);
|
||||
}
|
||||
|
||||
std::vector<float> &occlude_factor = ctx->occlude_factor();
|
||||
|
||||
occlude_factor.clear();
|
||||
occlude_factor.insert(occlude_factor.end(), pool.size(), 0.0f);
|
||||
|
||||
float cur_alpha = 1;
|
||||
while (cur_alpha <= alpha_ && result.size() < max_degree_) {
|
||||
for (auto iter = pool.begin();
|
||||
result.size() < max_degree_ && iter != pool.end(); ++iter) {
|
||||
if (occlude_factor[iter - pool.begin()] > cur_alpha) {
|
||||
continue;
|
||||
}
|
||||
|
||||
occlude_factor[iter - pool.begin()] = std::numeric_limits<float>::max();
|
||||
|
||||
if (iter->id != id) {
|
||||
result.push_back(iter->id);
|
||||
}
|
||||
|
||||
for (auto iter2 = iter + 1; iter2 != pool.end(); iter2++) {
|
||||
auto t = iter2 - pool.begin();
|
||||
if (occlude_factor[t] > alpha_) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float djk = dc.dist(iter2->id, iter->id);
|
||||
|
||||
occlude_factor[t] =
|
||||
(djk == 0) ? std::numeric_limits<float>::max()
|
||||
: std::max(occlude_factor[t], iter2->distance / djk);
|
||||
}
|
||||
}
|
||||
cur_alpha *= 1.2f;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::prune_neighbors(diskann_id_t id,
|
||||
std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
if (pool.size() == 0) {
|
||||
pruned_list.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::sort(pool.begin(), pool.end());
|
||||
|
||||
pruned_list.clear();
|
||||
pruned_list.reserve(max_degree_);
|
||||
|
||||
occlude_list(id, pool, pruned_list, ctx);
|
||||
|
||||
ailego_assert(pruned_list.size() <= max_degree_);
|
||||
|
||||
if (saturate_graph_ && alpha_ > 1) {
|
||||
for (const auto &node : pool) {
|
||||
if (pruned_list.size() >= max_degree_) {
|
||||
break;
|
||||
}
|
||||
|
||||
if ((std::find(pruned_list.begin(), pruned_list.end(), node.id) ==
|
||||
pruned_list.end()) &&
|
||||
node.id != id) {
|
||||
pruned_list.push_back(node.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::search_neighbor_and_prune(
|
||||
diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
const std::vector<diskann_id_t> init_ids = get_init_ids(ctx);
|
||||
|
||||
int ret = iterate_to_fixed_point(init_ids, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto &pool = ctx->expanded_nodes();
|
||||
pool.erase(std::remove_if(pool.begin(), pool.end(),
|
||||
[id](const auto &node) { return node.id == id; }),
|
||||
pool.end());
|
||||
|
||||
ret = prune_neighbors(id, pool, pruned_list, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include <zvec/core/framework/index_meta.h>
|
||||
#include "diskann_context.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnAlgorithm {
|
||||
public:
|
||||
typedef std::unique_ptr<DiskAnnAlgorithm> UPointer;
|
||||
|
||||
public:
|
||||
DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree);
|
||||
|
||||
public:
|
||||
int add_node(diskann_id_t id, DiskAnnContext *ctx);
|
||||
int prune_node(diskann_id_t id, DiskAnnContext *ctx);
|
||||
|
||||
private:
|
||||
int search_neighbor_and_prune(diskann_id_t id,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int iterate_to_fixed_point(const std::vector<diskann_id_t> &init_ids,
|
||||
DiskAnnContext *ctx);
|
||||
int prune_neighbors(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int inter_insert(diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &result, DiskAnnContext *ctx);
|
||||
|
||||
std::vector<diskann_id_t> get_init_ids(DiskAnnContext *ctx);
|
||||
|
||||
private:
|
||||
static constexpr uint32_t kLockCnt{1U << 16};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
DiskAnnEntity &entity_;
|
||||
|
||||
uint32_t max_degree_{DiskAnnEntity::kDefaultMaxDegree};
|
||||
uint32_t max_candidate_size_{DiskAnnEntity::kDefaultMaxOcclusionSize};
|
||||
|
||||
std::vector<std::mutex> lock_pool_{};
|
||||
|
||||
//! Acquire the striped lock guarding the given id (RAII).
|
||||
[[nodiscard]] std::unique_lock<std::mutex> lock_for(diskann_id_t id) {
|
||||
return std::unique_lock<std::mutex>(lock_pool_[id & kLockMask]);
|
||||
}
|
||||
|
||||
float alpha_{DiskAnnEntity::kDefaultAlpha};
|
||||
bool saturate_graph_{true};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,630 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_builder.h"
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <ailego/math/euclidean_distance_matrix.h>
|
||||
#include <ailego/math/normalizer.h>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/interface/index_factory.h>
|
||||
#include "algorithm/cluster/vector_mean.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnBuilder::init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
LOG_INFO("Begin DiskAnnBuilder::init");
|
||||
|
||||
params.get(PARAM_DISKANN_BUILDER_MAX_DEGREE, &max_degree_);
|
||||
params.get(PARAM_DISKANN_BUILDER_LIST_SIZE, &list_size_);
|
||||
params.get(PARAM_DISKANN_BUILDER_THREAD_COUNT, &build_thread_count_);
|
||||
|
||||
if (build_thread_count_ == 0) {
|
||||
build_thread_count_ = std::thread::hardware_concurrency();
|
||||
}
|
||||
|
||||
if (build_thread_count_ > std::thread::hardware_concurrency()) {
|
||||
LOG_WARN("Build thread count [%s] greater than cpu cores %u",
|
||||
PARAM_DISKANN_BUILDER_THREAD_COUNT.c_str(),
|
||||
std::thread::hardware_concurrency());
|
||||
}
|
||||
|
||||
uint32_t max_pq_chunk_num{0};
|
||||
if (params.get(PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM, &max_pq_chunk_num)) {
|
||||
if (max_pq_chunk_num > meta.dimension()) {
|
||||
LOG_ERROR(
|
||||
"PQ Chunk Num larger than dimension, PQ Chunk Num: %d, Dimension: %d",
|
||||
max_pq_chunk_num, meta.dimension());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
max_pq_chunk_num_ = max_pq_chunk_num;
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_MEMORY_LIMIT)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_MEMORY_LIMIT, &memory_limit_);
|
||||
if (memory_limit_ <= 0) {
|
||||
LOG_ERROR("Invalid memory limit: %lf", memory_limit_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
memory_limit_set_ = true;
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT,
|
||||
&max_train_sample_count_);
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO, &train_sample_ratio_);
|
||||
}
|
||||
|
||||
raw_meta_ = meta;
|
||||
|
||||
build_meta_ = meta;
|
||||
if (meta.metric_name() == "InnerProduct") {
|
||||
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
|
||||
} else if (meta.metric_name() == "Cosine") {
|
||||
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
|
||||
|
||||
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
|
||||
build_meta_.set_dimension(meta.dimension() - 1);
|
||||
} else {
|
||||
build_meta_.set_dimension(meta.dimension() - 2);
|
||||
}
|
||||
}
|
||||
|
||||
metric_ = IndexFactory::CreateMetric(build_meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s",
|
||||
build_meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int ret = metric_->init(build_meta_, build_meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMeasure init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
raw_meta_.set_builder("DiskAnnBuilder", DiskAnnEntity::kRevision, params);
|
||||
|
||||
ret = entity_.init(meta, max_degree_, list_size_, memory_limit_,
|
||||
build_thread_count_);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
algo_ =
|
||||
DiskAnnAlgorithm::UPointer(new DiskAnnAlgorithm(entity_, max_degree_));
|
||||
|
||||
trainer_ =
|
||||
DiskAnnPqTrainer::UPointer(new DiskAnnPqTrainer(max_train_sample_count_));
|
||||
|
||||
state_ = BUILD_STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::cleanup(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::calculate_entry_point() {
|
||||
size_t dimension = build_meta_.dimension();
|
||||
|
||||
if (build_meta_.data_type() != IndexMeta::DataType::DT_FP32 &&
|
||||
build_meta_.data_type() != IndexMeta::DataType::DT_FP16) {
|
||||
LOG_ERROR("Data type not supported");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::vector<float> centroid_fp32;
|
||||
std::vector<ailego::Float16> centroid_fp16;
|
||||
|
||||
switch (build_meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
centroid_fp32.resize(dimension);
|
||||
NumericalVectorMean<float> accumulator(dimension);
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
accumulator.plus(entity_.get_vector(id), dimension * sizeof(float));
|
||||
}
|
||||
accumulator.mean(centroid_fp32.data(), dimension * sizeof(float));
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
centroid_fp16.resize(dimension);
|
||||
NumericalVectorMean<ailego::Float16> accumulator(dimension);
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
accumulator.plus(entity_.get_vector(id),
|
||||
dimension * sizeof(ailego::Float16));
|
||||
}
|
||||
accumulator.mean(centroid_fp16.data(),
|
||||
dimension * sizeof(ailego::Float16));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
// compute all to one distance
|
||||
diskann_id_t medoid_id = kInvalidId;
|
||||
float min_dist = std::numeric_limits<float>::max();
|
||||
|
||||
switch (build_meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
const float *data_ptr =
|
||||
reinterpret_cast<const float *>(entity_.get_vector(id));
|
||||
|
||||
float dist = 0.0f;
|
||||
ailego::SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
centroid_fp32.data(), data_ptr, dimension, &dist);
|
||||
|
||||
if (dist < min_dist) {
|
||||
min_dist = dist;
|
||||
medoid_id = id;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
const ailego::Float16 *data_ptr =
|
||||
reinterpret_cast<const ailego::Float16 *>(entity_.get_vector(id));
|
||||
|
||||
float dist = 0.0f;
|
||||
ailego::SquaredEuclideanDistanceMatrix<ailego::Float16, 1, 1>::Compute(
|
||||
centroid_fp16.data(), data_ptr, dimension, &dist);
|
||||
|
||||
if (dist < min_dist) {
|
||||
min_dist = dist;
|
||||
medoid_id = id;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
(*entity_.mutable_medoid()) = medoid_id;
|
||||
|
||||
LOG_INFO("Medoid Calculation Done. ID: %zu", (size_t)medoid_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::calculate_pq_chunk_num() {
|
||||
size_t doc_cnt = holder_->count();
|
||||
if (doc_cnt == 0) {
|
||||
LOG_ERROR("Invalid Input. Empty Vecs.");
|
||||
|
||||
return IndexError_InvalidLength;
|
||||
}
|
||||
|
||||
if (memory_limit_set_) {
|
||||
size_t memory_limit_bytes = get_memory_in_bytes(memory_limit_);
|
||||
size_t pq_chunk_num = std::floor(memory_limit_bytes / doc_cnt);
|
||||
if (pq_chunk_num <= 0) {
|
||||
LOG_ERROR("Insufficient memory limit for vec, memory: %zu, vec num: %zu",
|
||||
memory_limit_bytes, doc_cnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
}
|
||||
|
||||
pq_chunk_num_ =
|
||||
pq_chunk_num_ < max_pq_chunk_num_ ? pq_chunk_num_ : max_pq_chunk_num_;
|
||||
|
||||
// A chunk num of 0 (public API default) or the internal sentinel means
|
||||
// "auto": quantize into half the dimensions. Resolve it before the
|
||||
// upper-bound check so the default never reaches the divide-by-chunk path.
|
||||
if (pq_chunk_num_ == 0 || pq_chunk_num_ == kDefaultPqChunkNum) {
|
||||
pq_chunk_num_ = build_meta_.dimension() / 2;
|
||||
LOG_INFO(
|
||||
"No Chunk Num input. Quantizing %u dimension data into %u dimension.",
|
||||
build_meta_.dimension(), pq_chunk_num_);
|
||||
}
|
||||
|
||||
if (pq_chunk_num_ > build_meta_.dimension()) {
|
||||
LOG_ERROR("PQ Chunk Num is more than dimension, chunk num: %u, dim: %u",
|
||||
pq_chunk_num_, build_meta_.dimension());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
LOG_INFO("Quantizing %u dimension data into %u bytes.",
|
||||
build_meta_.dimension(), pq_chunk_num_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::build_internal(IndexThreads::Pointer threads) {
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
std::atomic<uint64_t> finished{0};
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_build, i,
|
||||
threads->count(), &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Built cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
|
||||
finished.load() * 100.0f / entity_.doc_cnt());
|
||||
}
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::prune_internal(IndexThreads::Pointer threads) {
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
std::atomic<uint64_t> finished{0};
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_prune, i,
|
||||
threads->count(), &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to purne index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Prune cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
|
||||
finished.load() * 100.0f / entity_.doc_cnt());
|
||||
}
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to prune index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train_quantized_data(IndexThreads::Pointer threads) {
|
||||
LOG_INFO("Starting Train: Chunk Num: %u", pq_chunk_num_);
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = trainer_->train_quantized_data(
|
||||
threads, holder_, build_meta_, entity_.pq_full_pivot_data(),
|
||||
entity_.pq_centroid(), entity_.pq_chunk_offsets(), pq_chunk_num_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Train Quantized Data Error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t pq_time = timer.milli_seconds();
|
||||
LOG_INFO("Train Quantized Data Done, time: %zu ms", pq_time);
|
||||
|
||||
(*entity_.mutable_pq_meta()).full_pivot_data_size =
|
||||
entity_.pq_full_pivot_data().size();
|
||||
(*entity_.mutable_pq_meta()).centroid_data_size =
|
||||
entity_.pq_centroid().size();
|
||||
(*entity_.mutable_pq_meta()).chunk_num = pq_chunk_num_;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::generate_quantized_data(IndexThreads::Pointer threads) {
|
||||
LOG_INFO("Starting PQ Generate: Query Memory Limit: %lf, Chunk Num: %u",
|
||||
memory_limit_, pq_chunk_num_);
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = trainer_->generate_quantized_data(
|
||||
threads, holder_, build_meta_, entity_.pq_centroid(),
|
||||
entity_.block_compressed_data(), pq_chunk_num_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Generate Quantized Data Error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t pq_time = timer.milli_seconds();
|
||||
LOG_INFO("Generate Quantized Data Done, time: %zu ms", pq_time);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnBuilder::do_build(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished) {
|
||||
AILEGO_DEFER([&]() {
|
||||
std::lock_guard<std::mutex> latch(mutex_);
|
||||
cond_.notify_one();
|
||||
});
|
||||
|
||||
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
|
||||
build_meta_, metric_,
|
||||
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
|
||||
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to create context");
|
||||
errcode_ = IndexError_NoMemory;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
DiskAnnContext::Pointer auto_ptr(ctx);
|
||||
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
|
||||
build_meta_.element_size());
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
|
||||
ctx->reset_query(entity_.get_vector(id));
|
||||
int ret = algo_->add_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("DiskAnn graph add node failed");
|
||||
errcode_ = ret;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx->clear();
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
void DiskAnnBuilder::do_prune(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished) {
|
||||
AILEGO_DEFER([&]() {
|
||||
std::lock_guard<std::mutex> latch(mutex_);
|
||||
cond_.notify_one();
|
||||
});
|
||||
|
||||
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
|
||||
build_meta_, metric_,
|
||||
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
|
||||
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to create context");
|
||||
errcode_ = IndexError_NoMemory;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
DiskAnnContext::Pointer auto_ptr(ctx);
|
||||
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
|
||||
build_meta_.element_size());
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
|
||||
ctx->reset_query(entity_.get_vector(id));
|
||||
int ret = algo_->prune_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("DiskAnn graph add node failed");
|
||||
errcode_ = ret;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx->clear();
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::train by trainer");
|
||||
|
||||
stats_.set_trained_count(0UL);
|
||||
stats_.set_trained_costtime(0UL);
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::train by trainer");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::train");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
holder_ = std::move(holder);
|
||||
|
||||
LOG_INFO("Start to calculate chunk num");
|
||||
int ret = calculate_pq_chunk_num();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
ret = train_quantized_data(threads);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
stats_.set_trained_count(holder_->count());
|
||||
|
||||
stats_.set_trained_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
holder_.reset();
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::train");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::do_norm(const void *data_ptr, std::string *norm_data) {
|
||||
size_t dimension = build_meta_.dimension();
|
||||
const float *float_data_ptr = reinterpret_cast<const float *>(data_ptr);
|
||||
|
||||
norm_data->resize(dimension * sizeof(float));
|
||||
float *output_buf = reinterpret_cast<float *>(&((*norm_data)[0]));
|
||||
std::memcpy(output_buf, float_data_ptr, dimension * sizeof(float));
|
||||
|
||||
float norm = 0.0f;
|
||||
ailego::Normalizer<float>::L2(output_buf, dimension, &norm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
LOG_INFO("Start DiskAnnBuilder::build");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
holder_ = holder;
|
||||
|
||||
if (!threads) {
|
||||
threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(holder->count() == 0)) {
|
||||
LOG_ERROR("Holder is empty");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
int ret = entity_.reserve_space(holder->count());
|
||||
|
||||
error_ = false;
|
||||
while (iter->is_valid()) {
|
||||
ret = entity_.add_vector(iter->key(), iter->data());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
iter->next();
|
||||
}
|
||||
|
||||
LOG_INFO("Finished saving vector");
|
||||
|
||||
LOG_INFO("Start to calculate entrypoint");
|
||||
ret = calculate_entry_point();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start to build vamana graph");
|
||||
ret = build_internal(threads);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start final cleanup..");
|
||||
ret = prune_internal(threads);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start to generate quantized data");
|
||||
ret = generate_quantized_data(threads);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = BUILD_STATE_BUILT;
|
||||
|
||||
stats_.set_built_count(entity_.doc_cnt());
|
||||
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::build");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::dump(const IndexDumper::Pointer &dumper) {
|
||||
if (state_ != BUILD_STATE_BUILT) {
|
||||
LOG_INFO("Build the index before DiskAnnBuilder::dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::dump");
|
||||
|
||||
raw_meta_.set_searcher("DiskAnnSearcher", 0, ailego::Params());
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::SerializeToDumper(raw_meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.dump(holder_, raw_meta_, dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Index dump failed, ret: %u", ret);
|
||||
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
stats_.set_dumped_count(holder_->count());
|
||||
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
LOG_INFO("DiskAnnBuilder::dump");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_BUILDER(DiskAnnBuilder);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_builder.h>
|
||||
#include "diskann_algorithm.h"
|
||||
#include "diskann_builder_entity.h"
|
||||
#include "diskann_pq_trainer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Constructor
|
||||
DiskAnnBuilder() = default;
|
||||
|
||||
//! Initialize the builder
|
||||
virtual int init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(const IndexTrainer::Pointer &trainer) override;
|
||||
|
||||
//! Build the index
|
||||
virtual int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
int do_norm(const void *data_ptr, std::string *norm_data);
|
||||
|
||||
private:
|
||||
int train_quantized_data(IndexThreads::Pointer threads);
|
||||
int generate_quantized_data(IndexThreads::Pointer threads);
|
||||
int build_internal(IndexThreads::Pointer threads);
|
||||
int prune_internal(IndexThreads::Pointer threads);
|
||||
|
||||
void do_build(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished);
|
||||
|
||||
void do_prune(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished);
|
||||
|
||||
int calculate_entry_point();
|
||||
|
||||
int calculate_pq_chunk_num();
|
||||
|
||||
double get_memory_in_bytes(double search_ram_budget) {
|
||||
return search_ram_budget * 1024 * 1024 * 1024;
|
||||
}
|
||||
|
||||
private:
|
||||
enum BUILD_STATE {
|
||||
BUILD_STATE_INIT = 0,
|
||||
BUILD_STATE_INITED = 1,
|
||||
BUILD_STATE_TRAINED = 2,
|
||||
BUILD_STATE_BUILT = 3
|
||||
};
|
||||
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
constexpr static uint32_t kDefaultListSize = 50U;
|
||||
constexpr static uint32_t kDefaultMaxDegree = 100U;
|
||||
constexpr static uint32_t kDefaultPqChunkNum = -1U;
|
||||
|
||||
std::string data_file_;
|
||||
|
||||
uint32_t max_degree_{kDefaultMaxDegree};
|
||||
uint32_t list_size_{kDefaultListSize};
|
||||
double memory_limit_{0.0};
|
||||
bool memory_limit_set_{false};
|
||||
uint32_t max_pq_chunk_num_{kDefaultPqChunkNum};
|
||||
uint32_t pq_chunk_num_{kDefaultPqChunkNum};
|
||||
uint32_t build_thread_count_{0};
|
||||
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
|
||||
double train_sample_ratio_{PQTable::kTrainSampleRatio};
|
||||
std::string universal_label_{""};
|
||||
std::string codebook_prefix_{""};
|
||||
std::string index_path_prefix_{"./diskann"};
|
||||
|
||||
BUILD_STATE state_;
|
||||
Stats stats_;
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
|
||||
IndexMetric::Pointer metric_{};
|
||||
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
|
||||
IndexMeta raw_meta_;
|
||||
IndexMeta build_meta_;
|
||||
|
||||
DiskAnnBuilderEntity entity_{};
|
||||
|
||||
IndexHolder::Pointer holder_;
|
||||
|
||||
DiskAnnAlgorithm::UPointer algo_;
|
||||
DiskAnnPqTrainer::UPointer trainer_;
|
||||
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,643 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_builder_entity.h"
|
||||
#include <iostream>
|
||||
#include "diskann_algorithm.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnBuilderEntity::init(const IndexMeta &meta, uint32_t max_degree,
|
||||
uint32_t list_size, double memory_limit,
|
||||
uint32_t build_threads) {
|
||||
meta_ = meta;
|
||||
|
||||
max_degree_ = max_degree;
|
||||
list_size_ = list_size;
|
||||
|
||||
memory_limit_ = memory_limit;
|
||||
|
||||
num_threads_ = build_threads;
|
||||
|
||||
max_build_degree_ = max_degree_ * kDefaultGraphSlackFactor;
|
||||
|
||||
neighbor_size_ = sizeof(uint32_t) + max_build_degree_ * sizeof(diskann_id_t);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::reserve_space(uint32_t docs) {
|
||||
vectors_buffer_.reserve(meta_.element_size() * docs);
|
||||
keys_buffer_.reserve(sizeof(diskann_key_t) * docs);
|
||||
neighbors_buffer_.reserve(neighbor_size_ * docs);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::add_vector(diskann_key_t key, const void *vec) {
|
||||
vectors_buffer_.append(reinterpret_cast<const char *>(vec),
|
||||
meta_.element_size());
|
||||
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
|
||||
|
||||
uint32_t neighbor_cnt = 0;
|
||||
std::vector<diskann_id_t> neighbor{max_build_degree_, 0};
|
||||
|
||||
neighbors_buffer_.append(reinterpret_cast<const char *>(&neighbor_cnt),
|
||||
sizeof(uint32_t));
|
||||
neighbors_buffer_.append(reinterpret_cast<const char *>(neighbor.data()),
|
||||
sizeof(diskann_id_t) * max_build_degree_);
|
||||
|
||||
(*mutable_doc_cnt())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const void *DiskAnnBuilderEntity::get_vector(diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * meta_.element_size();
|
||||
return vectors_buffer_.data() + offset;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnBuilderEntity::get_key(diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * sizeof(diskann_key_t);
|
||||
|
||||
return *(
|
||||
reinterpret_cast<const diskann_key_t *>(keys_buffer_.data() + offset));
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnBuilderEntity::get_id(diskann_key_t /*key*/) const {
|
||||
LOG_ERROR("DiskAnnBuilderEntity::get_id not implemented.");
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> DiskAnnBuilderEntity::get_neighbors(
|
||||
diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
const uint8_t *start_ptr =
|
||||
reinterpret_cast<const uint8_t *>(neighbors_buffer_.data()) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = *(reinterpret_cast<const uint32_t *>(start_ptr));
|
||||
|
||||
const diskann_id_t *neighbors =
|
||||
reinterpret_cast<const diskann_id_t *>(start_ptr + sizeof(uint32_t));
|
||||
|
||||
return std::make_pair(neighbor_cnt, neighbors);
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::set_neighbors(
|
||||
diskann_id_t id, const std::vector<diskann_id_t> &neighbor_ids) {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
uint8_t *start_ptr =
|
||||
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = neighbor_ids.size();
|
||||
|
||||
memcpy(start_ptr + sizeof(uint32_t), neighbor_ids.data(),
|
||||
sizeof(diskann_id_t) * neighbor_cnt);
|
||||
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
|
||||
|
||||
if (max_observed_degree_ < neighbor_cnt) {
|
||||
max_observed_degree_ = neighbor_cnt;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::add_neighbor(diskann_id_t id,
|
||||
diskann_id_t neighbor_id) {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
uint8_t *start_ptr =
|
||||
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = *reinterpret_cast<uint32_t *>(start_ptr);
|
||||
|
||||
memcpy(start_ptr + sizeof(uint32_t) + sizeof(diskann_id_t) * neighbor_cnt,
|
||||
&neighbor_id, sizeof(diskann_id_t));
|
||||
|
||||
neighbor_cnt += 1;
|
||||
|
||||
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
|
||||
|
||||
if (max_observed_degree_ < neighbor_cnt) {
|
||||
max_observed_degree_ = neighbor_cnt;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int64_t DiskAnnBuilderEntity::dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id,
|
||||
const void *data,
|
||||
size_t size) const {
|
||||
size_t len = dumper->write(data, size);
|
||||
if (len != size) {
|
||||
LOG_ERROR("Dump segment %s data failed, expect: %lu, actual: %lu",
|
||||
segment_id.c_str(), size, len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(size) - size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t crc = ailego::Crc32c::Hash(data, size);
|
||||
int ret = dumper->append(segment_id, size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_pq_meta_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
uint32_t crc = 0U;
|
||||
|
||||
// write meta
|
||||
size_t size_pq_meta = dumper->write(&pq_meta_, sizeof(DiskAnnPqMeta));
|
||||
if (size_pq_meta != sizeof(DiskAnnPqMeta)) {
|
||||
LOG_ERROR("Failed to dump PQ meta data, expect: %lu, actual: %lu",
|
||||
sizeof(DiskAnnPqMeta), size_pq_meta);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(&pq_meta_, sizeof(DiskAnnPqMeta), crc);
|
||||
|
||||
// write full pivot data
|
||||
size_t size_full_pivot_data =
|
||||
dumper->write(pq_full_pivot_data_.data(), pq_meta_.full_pivot_data_size);
|
||||
if (size_full_pivot_data != pq_meta_.full_pivot_data_size) {
|
||||
LOG_ERROR("Failed to dump full pivot data, expect: %zu, actual: %zu",
|
||||
(size_t)pq_meta_.full_pivot_data_size, size_full_pivot_data);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_full_pivot_data_.data(),
|
||||
pq_meta_.full_pivot_data_size, crc);
|
||||
|
||||
// write centroid num
|
||||
size_t size_centroid =
|
||||
dumper->write(pq_centroid_.data(), pq_meta_.centroid_data_size);
|
||||
if (size_centroid != pq_meta_.centroid_data_size) {
|
||||
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
|
||||
(size_t)pq_meta_.centroid_data_size, size_centroid);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_centroid_.data(), pq_meta_.centroid_data_size,
|
||||
crc);
|
||||
|
||||
// write chunk offset
|
||||
size_t size_chunk_offset = dumper->write(
|
||||
pq_chunk_offsets_.data(), (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
|
||||
if (size_chunk_offset != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
|
||||
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
|
||||
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
|
||||
size_chunk_offset);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_chunk_offsets_.data(),
|
||||
(pq_meta_.chunk_num + 1) * sizeof(uint32_t), crc);
|
||||
|
||||
// write size
|
||||
size_t size_total =
|
||||
size_pq_meta + size_full_pivot_data + size_centroid + size_chunk_offset;
|
||||
|
||||
// write pad
|
||||
size_t padding_size = AlignSize(size_total) - size_total;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
int ret =
|
||||
dumper->append(kDiskAnnPqMetaSegmentId, size_total, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump PQ segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_pq_data_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
uint64_t doc_cnt = meta_header_.doc_cnt;
|
||||
uint64_t chunk_num = pq_meta_.chunk_num;
|
||||
|
||||
uint32_t crc = 0U;
|
||||
|
||||
// write pq data
|
||||
size_t size_total =
|
||||
dumper->write(block_compressed_data_.data(), doc_cnt * chunk_num);
|
||||
|
||||
if (size_total != doc_cnt * chunk_num) {
|
||||
LOG_ERROR("Failed to dump block compressed data, expect: %zu, actual: %zu",
|
||||
(size_t)(doc_cnt * chunk_num), size_total);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(block_compressed_data_.data(), doc_cnt * chunk_num,
|
||||
crc);
|
||||
|
||||
// write pad
|
||||
size_t padding_size = AlignSize(size_total) - size_total;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
int ret =
|
||||
dumper->append(kDiskAnnPqDataSegmentId, size_total, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump PQ data segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_dummy_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
// to make offset aligned with 4K
|
||||
size_t dumper_header_size = dumper->size();
|
||||
|
||||
size_t dummy_size =
|
||||
DiskAnnUtil::round_up(dumper_header_size, DiskAnnUtil::kSectorSize) -
|
||||
dumper_header_size;
|
||||
|
||||
if (dummy_size != 0) {
|
||||
std::string dummpy_data(dummy_size, '\0');
|
||||
if (dumper->write(dummpy_data.data(), dummy_size) != dummy_size) {
|
||||
LOG_ERROR("write dummy failed, size %lu", dummy_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
int ret = dumper->append(kDiskAnnDummpySegmentId, dummy_size, 0, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump dummy data segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_key_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
//! Dump keys
|
||||
size_t key_segment_size = doc_cnt() * sizeof(diskann_key_t);
|
||||
int64_t keys_size = dump_segment(dumper, kDiskAnnKeySegmentId,
|
||||
keys_buffer_.data(), key_segment_size);
|
||||
if (keys_size < 0) {
|
||||
return keys_size;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_key_mapping_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
std::vector<diskann_id_t> mapping(doc_cnt());
|
||||
|
||||
const diskann_key_t *keys = reinterpret_cast<diskann_key_t *>(
|
||||
const_cast<char *>(keys_buffer_.data()));
|
||||
|
||||
std::iota(mapping.begin(), mapping.end(), 0U);
|
||||
std::sort(mapping.begin(), mapping.end(),
|
||||
[&](diskann_id_t i, diskann_id_t j) { return keys[i] < keys[j]; });
|
||||
|
||||
size_t size = mapping.size() * sizeof(diskann_id_t);
|
||||
int64_t ret =
|
||||
dump_segment(dumper, kDiskAnnKeyMappingSegmentId, mapping.data(), size);
|
||||
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_entrypoint_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
std::string entrypoint_buffer;
|
||||
|
||||
size_t size = sizeof(uint32_t) + entrypoints_.size() * sizeof(diskann_id_t);
|
||||
entrypoint_buffer.resize(size);
|
||||
|
||||
uint8_t *buffer_ptr = reinterpret_cast<uint8_t *>(&entrypoint_buffer[0]);
|
||||
|
||||
uint32_t point_cnt = entrypoints_.size();
|
||||
memcpy(buffer_ptr, &point_cnt, sizeof(uint32_t));
|
||||
memcpy(buffer_ptr + sizeof(uint32_t), entrypoints_.data(),
|
||||
entrypoints_.size() * sizeof(diskann_id_t));
|
||||
|
||||
int64_t ret = dump_segment(dumper, kDiskAnnEntryPointSegmentId,
|
||||
entrypoint_buffer.data(), size);
|
||||
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump entrypoint segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump(IndexHolder::Pointer holder, IndexMeta &meta,
|
||||
const IndexDumper::Pointer &dumper) {
|
||||
uint64_t doc_cnt = holder->count();
|
||||
uint64_t max_node_size =
|
||||
(uint64_t)max_observed_degree_ * sizeof(diskann_id_t) + sizeof(uint32_t) +
|
||||
meta_.element_size();
|
||||
uint64_t node_per_sector =
|
||||
DiskAnnUtil::kSectorSize /
|
||||
max_node_size; // 0 if max_node_size > DiskAnnUtil::kSectorSize
|
||||
|
||||
std::string node_buf;
|
||||
node_buf.resize(max_node_size);
|
||||
|
||||
diskann_id_t *neighbor_buf =
|
||||
(diskann_id_t *)(node_buf.data() + (meta_.element_size()) +
|
||||
sizeof(uint32_t));
|
||||
|
||||
LOG_INFO(
|
||||
"Dump Data, medoid: %zu, max node size: %zu, node per sector: %zu, "
|
||||
"max observed degree: %zu",
|
||||
(size_t)medoid(), (size_t)max_node_size, (size_t)node_per_sector,
|
||||
(size_t)max_observed_degree_);
|
||||
|
||||
// write a dummy segment to make data align
|
||||
int ret = dump_dummy_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump dummy segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump data by sector
|
||||
size_t write_size = 0;
|
||||
uint32_t crc = 0U;
|
||||
size_t len = 0;
|
||||
|
||||
// no need to write first sector
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
uint64_t index_size = 0;
|
||||
uint32_t neighbor_num;
|
||||
if (node_per_sector > 0) {
|
||||
uint64_t sector_num =
|
||||
DiskAnnUtil::round_up(doc_cnt, node_per_sector) / node_per_sector;
|
||||
|
||||
diskann_id_t cur_node_id = 0;
|
||||
|
||||
std::string sector_buf;
|
||||
sector_buf.resize(DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t sector = 0; sector < sector_num; sector++) {
|
||||
if (sector != 0 && sector % 100000 == 0) {
|
||||
LOG_INFO("Sector #%zu written", (size_t)sector);
|
||||
}
|
||||
|
||||
memset(&(sector_buf[0]), 0, DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t sector_node_id = 0;
|
||||
sector_node_id < node_per_sector && cur_node_id < doc_cnt;
|
||||
sector_node_id++) {
|
||||
memset(&(node_buf[0]), 0, max_node_size);
|
||||
|
||||
auto neighbors = get_neighbors(cur_node_id);
|
||||
neighbor_num = neighbors.first;
|
||||
|
||||
ailego_assert(neighbor_num > 0);
|
||||
ailego_assert(neighbor_num <= max_observed_degree_);
|
||||
|
||||
memcpy(&(neighbor_buf[0]), neighbors.second,
|
||||
neighbors.first * sizeof(diskann_id_t));
|
||||
|
||||
if (iter->is_valid()) {
|
||||
const void *vec = iter->data();
|
||||
memcpy(&(node_buf[0]), vec, meta.element_size());
|
||||
|
||||
iter->next();
|
||||
} else {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// write neighbor num
|
||||
*(uint32_t *)(node_buf.data() + meta_.element_size()) = neighbor_num;
|
||||
|
||||
// write neighbor buffer
|
||||
memcpy(&(node_buf[0]) + meta_.element_size() + sizeof(uint32_t),
|
||||
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
// get offset into sector_buf
|
||||
char *sector_node_buf = §or_buf[sector_node_id * max_node_size];
|
||||
|
||||
// copy node buf into sector_node_buf
|
||||
memcpy(sector_node_buf, node_buf.data(), max_node_size);
|
||||
|
||||
cur_node_id++;
|
||||
}
|
||||
|
||||
// flush sector to disk
|
||||
len = dumper->write(sector_buf.data(), DiskAnnUtil::kSectorSize);
|
||||
if (len != DiskAnnUtil::kSectorSize) {
|
||||
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
|
||||
(size_t)DiskAnnUtil::kSectorSize);
|
||||
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
write_size += DiskAnnUtil::kSectorSize;
|
||||
crc = ailego::Crc32c::Hash(sector_buf.data(), DiskAnnUtil::kSectorSize,
|
||||
crc);
|
||||
}
|
||||
|
||||
LOG_INFO("Total Sector #%zu written", (size_t)sector_num);
|
||||
|
||||
index_size = sector_num * DiskAnnUtil::kSectorSize;
|
||||
} else {
|
||||
// Write multi-sector nodes
|
||||
std::string multisector_buf;
|
||||
multisector_buf.resize(
|
||||
DiskAnnUtil::round_up(max_node_size, DiskAnnUtil::kSectorSize));
|
||||
|
||||
uint64_t sector_num_per_node =
|
||||
DiskAnnUtil::div_round_up(max_node_size, DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t i = 0; i < doc_cnt; i++) {
|
||||
if (i != 0 && (i * sector_num_per_node) % 100000 == 0) {
|
||||
LOG_INFO("Sector # %zu written", (size_t)(i * sector_num_per_node));
|
||||
}
|
||||
|
||||
memset(&(multisector_buf[0]), 0,
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize);
|
||||
memset(&(node_buf[0]), 0, max_node_size);
|
||||
|
||||
auto neighbors = get_neighbors(i);
|
||||
neighbor_num = neighbors.first;
|
||||
|
||||
ailego_assert(neighbor_num > 0);
|
||||
ailego_assert(neighbor_num <= max_observed_degree_);
|
||||
|
||||
// read node's nhood
|
||||
memcpy((char *)neighbor_buf, neighbors.second,
|
||||
neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
if (iter->is_valid()) {
|
||||
const void *vec = iter->data();
|
||||
memcpy(&(multisector_buf[0]), vec, meta.element_size());
|
||||
|
||||
iter->next();
|
||||
} else {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// write neighbor
|
||||
*(uint32_t *)(&(multisector_buf[0]) + meta_.element_size()) =
|
||||
neighbor_num;
|
||||
|
||||
// write nhood next
|
||||
memcpy(&(multisector_buf[0]) + meta_.element_size() + sizeof(uint32_t),
|
||||
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
// flush sector to disk
|
||||
len = dumper->write(multisector_buf.data(),
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize);
|
||||
if (len != sector_num_per_node * DiskAnnUtil::kSectorSize) {
|
||||
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
|
||||
(size_t)(sector_num_per_node * DiskAnnUtil::kSectorSize));
|
||||
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
write_size += sector_num_per_node * DiskAnnUtil::kSectorSize;
|
||||
|
||||
crc = ailego::Crc32c::Hash(multisector_buf.data(),
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize,
|
||||
crc);
|
||||
}
|
||||
|
||||
LOG_INFO("Total Sector #%zu written",
|
||||
(size_t)(doc_cnt * sector_num_per_node));
|
||||
|
||||
index_size = doc_cnt * sector_num_per_node * DiskAnnUtil::kSectorSize;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(write_size) - write_size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
ret = dumper->append(kDiskAnnVectorSegmentId, write_size, 0UL, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump diskann meta
|
||||
meta_header_.doc_cnt = doc_cnt;
|
||||
meta_header_.ndims = meta_.dimension();
|
||||
meta_header_.medoid = medoid();
|
||||
meta_header_.max_node_size = max_node_size;
|
||||
meta_header_.max_degree = max_observed_degree_;
|
||||
meta_header_.node_per_sector = node_per_sector;
|
||||
meta_header_.vamana_frozen_num = 0;
|
||||
meta_header_.vamana_frozen_loc = medoid();
|
||||
meta_header_.append_reorder_data = 0;
|
||||
meta_header_.index_size = index_size;
|
||||
|
||||
ret = dump_segment(dumper, kDiskAnnMetaSegmentId, &meta_header_,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump pq meta
|
||||
ret = dump_pq_meta_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump pq meta segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump pq data
|
||||
ret = dump_pq_data_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump pq data segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump key
|
||||
ret = dump_key_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump key segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump key mapping
|
||||
ret = dump_key_mapping_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump key mapping segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump entrypint
|
||||
ret = dump_entrypoint_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump entrypoint segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("DiskAnn Index File Dumped");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,107 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// wrapper class aligned with diskann
|
||||
class DiskAnnBuilderEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnBuilderEntity>;
|
||||
|
||||
DiskAnnBuilderEntity() = default;
|
||||
virtual ~DiskAnnBuilderEntity() = default;
|
||||
|
||||
public:
|
||||
int add_vector(diskann_key_t key, const void *vec) override;
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t id) const override;
|
||||
|
||||
int set_neighbors(diskann_id_t id,
|
||||
const std::vector<diskann_id_t> &neighbor_ids) override;
|
||||
|
||||
int add_neighbor(diskann_id_t id, diskann_id_t neighbor_id) override;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
public:
|
||||
int init(const IndexMeta &meta, uint32_t max_degree, uint32_t list_size,
|
||||
double memory_limit, uint32_t build_threads);
|
||||
|
||||
int dump(IndexHolder::Pointer holder, IndexMeta &meta,
|
||||
const IndexDumper::Pointer &dumper);
|
||||
|
||||
int64_t dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id, const void *data,
|
||||
size_t size) const;
|
||||
int dump_dummy_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_pq_meta_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_pq_data_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_key_mapping_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_entrypoint_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_key_segment(const IndexDumper::Pointer &dumper) const;
|
||||
|
||||
int reserve_space(uint32_t docs);
|
||||
|
||||
std::vector<uint8_t> &pq_full_pivot_data() {
|
||||
return pq_full_pivot_data_;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> &pq_centroid() {
|
||||
return pq_centroid_;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> &pq_chunk_offsets() {
|
||||
return pq_chunk_offsets_;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> &block_compressed_data() {
|
||||
return block_compressed_data_;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t max_degree_{0};
|
||||
uint32_t list_size_{0};
|
||||
double memory_limit_{0};
|
||||
uint32_t num_threads_{0};
|
||||
uint32_t max_build_degree_{0};
|
||||
uint32_t max_observed_degree_{0};
|
||||
uint32_t neighbor_size_{0};
|
||||
|
||||
std::string mem_index_file_{""};
|
||||
std::string index_path_prefix_{""};
|
||||
|
||||
std::string vectors_buffer_{};
|
||||
std::string keys_buffer_{};
|
||||
std::string neighbors_buffer_{};
|
||||
std::vector<diskann_id_t> entrypoints_{};
|
||||
|
||||
IndexMeta meta_;
|
||||
|
||||
std::vector<uint8_t> pq_full_pivot_data_;
|
||||
std::vector<uint8_t> pq_centroid_;
|
||||
std::vector<uint32_t> pq_chunk_offsets_;
|
||||
std::vector<uint8_t> block_compressed_data_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_context.h"
|
||||
#include <chrono>
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnContext::DiskAnnContext(const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity)
|
||||
: dc_(entity.get(), measure, meta.dimension()), entity_{entity} {}
|
||||
|
||||
int DiskAnnContext::init(ContextType type, uint32_t graph_degree,
|
||||
uint32_t pq_chunk_num, uint32_t element_size) {
|
||||
type_ = type;
|
||||
element_size_ = element_size;
|
||||
pq_chunk_num_ = pq_chunk_num;
|
||||
|
||||
DiskAnnUtil::alloc_aligned((void **)&query_, element_size_, 32);
|
||||
DiskAnnUtil::alloc_aligned((void **)&query_rotated_, element_size_, 32);
|
||||
|
||||
int ret;
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
|
||||
entity_->doc_cnt(), negative_probility_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case kSearcherContext:
|
||||
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(),
|
||||
entity_->doc_cnt(), negative_probility_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
|
||||
DiskAnnUtil::alloc_aligned(
|
||||
(void **)&pq_table_dist_buffer_,
|
||||
PQTable::kPQCentroidNum * pq_chunk_num_ * sizeof(float), 256);
|
||||
DiskAnnUtil::alloc_aligned((void **)&pq_coord_buffer_,
|
||||
graph_degree * pq_chunk_num_ * sizeof(uint8_t),
|
||||
256);
|
||||
DiskAnnUtil::alloc_aligned((void **)&coord_buffer_, element_size_, 256);
|
||||
DiskAnnUtil::alloc_aligned(
|
||||
(void **)§or_buffer_,
|
||||
DiskAnnUtil::kMaxSectorReadNum * DiskAnnUtil::kSectorSize,
|
||||
DiskAnnUtil::kSectorSize);
|
||||
|
||||
ret = setup_io_ctx(io_ctx_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("setup io ctx error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("Init context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
DiskAnnContext::~DiskAnnContext() {
|
||||
free(query_);
|
||||
free(query_rotated_);
|
||||
free(pq_table_dist_buffer_);
|
||||
free(pq_coord_buffer_);
|
||||
free(coord_buffer_);
|
||||
free(sector_buffer_);
|
||||
|
||||
if (type_ == kSearcherContext) {
|
||||
destroy_io_ctx(io_ctx_);
|
||||
}
|
||||
}
|
||||
|
||||
int DiskAnnContext::update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity,
|
||||
uint32_t magic_num) {
|
||||
if (ailego_unlikely(type != type_)) {
|
||||
LOG_ERROR(
|
||||
"DiskAnnContext does not support shared by different type, "
|
||||
"src=%u dst=%u",
|
||||
type_, type);
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
magic_ = kInvalidMgic;
|
||||
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
LOG_ERROR("BuildContext does not support update");
|
||||
return IndexError_NotImplemented;
|
||||
|
||||
case kSearcherContext:
|
||||
break;
|
||||
|
||||
case kReducerContext:
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("update context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
entity_ = entity;
|
||||
dc_.update(measure, meta.dimension());
|
||||
magic_ = magic_num;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,372 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include "utility/topk_result_builder.h"
|
||||
#include "diskann_dist_calculator.h"
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_visit_filter.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct SearchStats {
|
||||
public:
|
||||
float total_us = 0;
|
||||
float io_us = 0;
|
||||
float cpu_us = 0;
|
||||
uint64_t disk_page_reads = 0;
|
||||
uint64_t io_num = 0;
|
||||
uint64_t dist_num = 0;
|
||||
uint64_t cache_hits = 0;
|
||||
uint64_t hop_num = 0;
|
||||
};
|
||||
|
||||
class DiskAnnContext : public IndexContext,
|
||||
public TopkResultBuilder<DiskAnnContext, TopkHeap> {
|
||||
friend class TopkResultBuilder<DiskAnnContext, TopkHeap>;
|
||||
|
||||
public:
|
||||
//! Index Context Pointer
|
||||
using Pointer = std::unique_ptr<DiskAnnContext>;
|
||||
|
||||
enum ContextType {
|
||||
kUnknownContext = 0,
|
||||
kSearcherContext = 1,
|
||||
kBuilderContext = 2,
|
||||
kReducerContext = 3
|
||||
};
|
||||
|
||||
//! Construct
|
||||
DiskAnnContext(const IndexMeta &meta, const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~DiskAnnContext();
|
||||
|
||||
public:
|
||||
//! Init
|
||||
int init(ContextType type, uint32_t graph_degree, uint32_t pq_chunk_num,
|
||||
uint32_t element_size);
|
||||
|
||||
//! Update context, the context may be shared by different searcher/streamer
|
||||
int update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity, uint32_t magic_num);
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
void set_magic(uint32_t magic) {
|
||||
magic_ = magic;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
return debug_mode_;
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
return std::string("");
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params & /*params*/) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline DistCalculator &dist_calculator() {
|
||||
return dc_;
|
||||
}
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(val);
|
||||
}
|
||||
|
||||
void set_list_size(uint32_t list_size) {
|
||||
list_size_ = list_size;
|
||||
}
|
||||
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
|
||||
//! Get topk
|
||||
inline uint32_t topk() const override {
|
||||
return topk_;
|
||||
}
|
||||
|
||||
inline uint32_t list_size() const {
|
||||
return list_size_;
|
||||
}
|
||||
|
||||
inline void reset_query(const void *query) {
|
||||
memcpy(query_, query, element_size_);
|
||||
memcpy(query_rotated_, query, element_size_);
|
||||
|
||||
dc_.reset_query(query);
|
||||
}
|
||||
|
||||
inline TopkHeap &topk_heap() {
|
||||
return topk_heap_;
|
||||
}
|
||||
|
||||
inline void *query() {
|
||||
return query_;
|
||||
}
|
||||
|
||||
inline void *query_rotated() {
|
||||
return query_rotated_;
|
||||
}
|
||||
|
||||
inline float *pq_table_dist_buffer() {
|
||||
return pq_table_dist_buffer_;
|
||||
}
|
||||
|
||||
inline void *pq_coord_buffer() {
|
||||
return pq_coord_buffer_;
|
||||
}
|
||||
|
||||
inline void *coord_buffer() {
|
||||
return coord_buffer_;
|
||||
}
|
||||
|
||||
inline void *sector_buffer() {
|
||||
return sector_buffer_;
|
||||
}
|
||||
|
||||
inline IOContext &io_ctx() {
|
||||
return io_ctx_;
|
||||
}
|
||||
|
||||
inline void resize_results(size_t size) {
|
||||
if (group_by_search()) {
|
||||
group_results_.resize(size);
|
||||
} else {
|
||||
results_.resize(size);
|
||||
}
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return has_error_;
|
||||
}
|
||||
|
||||
inline void set_error(bool err) {
|
||||
has_error_ = err;
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
for (auto &it : results_) {
|
||||
it.clear();
|
||||
}
|
||||
|
||||
best_list_nodes_.clear();
|
||||
expanded_nodes_.clear();
|
||||
visit_filter_.clear();
|
||||
has_error_ = false;
|
||||
}
|
||||
|
||||
SearchStats &query_stats() {
|
||||
return query_stats_;
|
||||
}
|
||||
|
||||
const DiskAnnEntity &get_entity() const {
|
||||
return *entity_;
|
||||
}
|
||||
|
||||
NeighborPriorityQueue &best_list_nodes() {
|
||||
return best_list_nodes_;
|
||||
}
|
||||
|
||||
std::vector<Neighbor> &expanded_nodes() {
|
||||
return expanded_nodes_;
|
||||
}
|
||||
|
||||
std::vector<float> &occlude_factor() {
|
||||
return occlude_factor_;
|
||||
}
|
||||
|
||||
VisitFilter &visit_filter() {
|
||||
return visit_filter_;
|
||||
}
|
||||
|
||||
//! Reset context
|
||||
void reset(void) override {
|
||||
set_filter(nullptr);
|
||||
reset_threshold();
|
||||
set_fetch_vector(false);
|
||||
set_group_params(0, 0);
|
||||
reset_group_by();
|
||||
}
|
||||
|
||||
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
|
||||
return group_topk_heaps_;
|
||||
}
|
||||
|
||||
//! Get group topk
|
||||
inline uint32_t group_topk() const {
|
||||
return group_topk_;
|
||||
}
|
||||
|
||||
//! Get group num
|
||||
inline uint32_t group_num() const {
|
||||
return group_num_;
|
||||
}
|
||||
|
||||
//! Get if group by search
|
||||
inline bool group_by_search() {
|
||||
return group_num_ > 0;
|
||||
}
|
||||
|
||||
//! Set group params
|
||||
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
|
||||
group_num_ = group_num;
|
||||
group_topk_ = group_topk;
|
||||
|
||||
topk_ = group_topk_ * group_num_;
|
||||
|
||||
topk_heap_.limit(topk_);
|
||||
|
||||
group_topk_heaps_.clear();
|
||||
}
|
||||
|
||||
// topk_to_result / topk_to_single_result / topk_to_group_result are provided
|
||||
// by TopkResultBuilder via the hooks below (result_score,
|
||||
// emplace_result_doc).
|
||||
using TopkResultBuilder<DiskAnnContext, TopkHeap>::topk_to_result;
|
||||
|
||||
void set_to_result(uint32_t idx, const std::vector<diskann_id_t> &result_ids,
|
||||
const std::vector<float> &result_dists) {
|
||||
if (result_ids.size() != result_dists.size()) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t size = result_ids.size();
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
results_[idx].emplace_back(result_ids[i], result_dists[i], 0);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
//! Hooks required by TopkResultBuilder.
|
||||
std::vector<IndexDocumentList> &mutable_results() {
|
||||
return results_;
|
||||
}
|
||||
|
||||
std::vector<IndexGroupDocumentList> &mutable_group_results() {
|
||||
return group_results_;
|
||||
}
|
||||
|
||||
float result_score(const VectorInfo &info) const {
|
||||
return info.dist_;
|
||||
}
|
||||
|
||||
void emplace_result_doc(IndexDocumentList &docs, diskann_id_t id, float score,
|
||||
const VectorInfo &info) {
|
||||
if (fetch_vector_) {
|
||||
docs.emplace_back(entity_->get_key(id), score, id, info.vec_);
|
||||
} else {
|
||||
docs.emplace_back(entity_->get_key(id), score, id);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
constexpr static uint32_t kInvalidMgic = -1U;
|
||||
|
||||
uint32_t type_{kUnknownContext};
|
||||
|
||||
DistCalculator dc_;
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
|
||||
uint32_t topk_{0};
|
||||
uint32_t magic_{0U};
|
||||
bool debug_mode_{false};
|
||||
uint32_t pq_chunk_num_{0};
|
||||
uint32_t element_size_{0};
|
||||
uint32_t element_rotated_size_{0};
|
||||
uint32_t list_size_{0};
|
||||
|
||||
TopkHeap topk_heap_{};
|
||||
|
||||
uint32_t group_topk_{0};
|
||||
uint32_t group_num_{0};
|
||||
std::map<std::string, TopkHeap> group_topk_heaps_{};
|
||||
|
||||
IOContext io_ctx_{0};
|
||||
SearchStats query_stats_;
|
||||
|
||||
float *pq_table_dist_buffer_{nullptr};
|
||||
void *pq_coord_buffer_{nullptr};
|
||||
void *query_{nullptr};
|
||||
void *query_rotated_{nullptr};
|
||||
void *coord_buffer_{nullptr};
|
||||
void *sector_buffer_{nullptr};
|
||||
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
std::vector<IndexGroupDocumentList> group_results_{};
|
||||
|
||||
bool fetch_vector_{false};
|
||||
bool has_error_{false};
|
||||
|
||||
NeighborPriorityQueue best_list_nodes_;
|
||||
std::vector<Neighbor> expanded_nodes_;
|
||||
std::vector<float> occlude_factor_;
|
||||
|
||||
VisitFilter visit_filter_{};
|
||||
uint32_t filter_mode_{VisitFilter::ByteMap};
|
||||
float negative_probility_{DiskAnnEntity::kDefaultBFNegativeProbility};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include <zvec/core/framework/index_meta.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DistCalculator {
|
||||
public:
|
||||
typedef std::shared_ptr<DistCalculator> Pointer;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
DistCalculator(const DiskAnnEntity *entity,
|
||||
const IndexMetric::Pointer &measure, uint32_t dim)
|
||||
: entity_(entity),
|
||||
distance_(measure->distance()),
|
||||
query_(nullptr),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
void update(const IndexMetric::Pointer &measure, uint32_t dim) {
|
||||
distance_ = measure->distance();
|
||||
dim_ = dim;
|
||||
}
|
||||
|
||||
inline void update_distance(const IndexMetric::MatrixDistance &distance) {
|
||||
distance_ = distance;
|
||||
}
|
||||
|
||||
//! Reset query vector data
|
||||
inline void reset_query(const void *query) {
|
||||
error_ = false;
|
||||
query_ = query;
|
||||
}
|
||||
|
||||
//! Returns distance
|
||||
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
|
||||
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Nullptr of dense vector");
|
||||
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
float score{0.0f};
|
||||
distance_(vec_lhs, vec_rhs, dim_, &score);
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
//! Returns distance between query and vec.
|
||||
inline dist_t dist(const void *vec) {
|
||||
compare_cnt_++;
|
||||
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(diskann_id_t id) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *vec = entity_->get_vector(id);
|
||||
if (ailego_unlikely(vec == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(diskann_id_t lhs, diskann_id_t rhs) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *vec_lhs = entity_->get_vector(lhs);
|
||||
if (ailego_unlikely(vec_lhs == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, lhs id=%u", lhs);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
const void *vec_rhs = entity_->get_vector(rhs);
|
||||
if (ailego_unlikely(vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, rhs id=%u", rhs);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(vec_lhs, vec_rhs);
|
||||
}
|
||||
|
||||
dist_t operator()(const void *vec) {
|
||||
return dist(vec);
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
compare_cnt_ = 0;
|
||||
error_ = false;
|
||||
}
|
||||
|
||||
inline void clear_compare_cnt() {
|
||||
compare_cnt_ = 0;
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return error_;
|
||||
}
|
||||
|
||||
//! Get distances compute times
|
||||
inline uint32_t compare_cnt() const {
|
||||
return compare_cnt_;
|
||||
}
|
||||
|
||||
inline uint32_t dimension() const {
|
||||
return dim_;
|
||||
}
|
||||
|
||||
private:
|
||||
DistCalculator(const DistCalculator &) = delete;
|
||||
DistCalculator &operator=(const DistCalculator &) = delete;
|
||||
|
||||
private:
|
||||
const DiskAnnEntity *entity_;
|
||||
|
||||
IndexMetric::MatrixDistance distance_;
|
||||
const void *query_;
|
||||
uint32_t dim_;
|
||||
|
||||
uint32_t compare_cnt_; // record distance compute times
|
||||
bool error_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const std::string DiskAnnEntity::kDiskAnnVectorSegmentId = "diskann.vector";
|
||||
const std::string DiskAnnEntity::kDiskAnnMetaSegmentId = "diskann.meta";
|
||||
const std::string DiskAnnEntity::kDiskAnnPqMetaSegmentId = "diskann.pq_meta";
|
||||
const std::string DiskAnnEntity::kDiskAnnPqDataSegmentId = "diskann.pq_data";
|
||||
const std::string DiskAnnEntity::kDiskAnnDummpySegmentId = "diskann.dummy";
|
||||
const std::string DiskAnnEntity::kDiskAnnKeyMappingSegmentId =
|
||||
"diskann.key_mapping";
|
||||
const std::string DiskAnnEntity::kDiskAnnEntryPointSegmentId =
|
||||
"diskann.entrypoint";
|
||||
const std::string DiskAnnEntity::kDiskAnnKeySegmentId = "diskann.key";
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,244 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
using dist_t = float;
|
||||
using diskann_key_t = uint64_t;
|
||||
using diskann_id_t = uint32_t;
|
||||
|
||||
constexpr diskann_id_t kInvalidId = static_cast<diskann_id_t>(-1);
|
||||
constexpr diskann_key_t kInvalidKey = static_cast<key_t>(-1);
|
||||
|
||||
struct VectorInfo {
|
||||
float dist_;
|
||||
std::string vec_;
|
||||
|
||||
VectorInfo() = default;
|
||||
VectorInfo(float dist, const std::string &vec) : dist_{dist}, vec_{vec} {}
|
||||
VectorInfo(float dist, std::string &&vec)
|
||||
: dist_{dist}, vec_{std::move(vec)} {}
|
||||
};
|
||||
|
||||
/*! Key Value Vecotr Heap Comparer
|
||||
*/
|
||||
struct KeyValueVectorHeapComparer {
|
||||
//! Function call
|
||||
bool operator()(const std::pair<diskann_id_t, VectorInfo> &lhs,
|
||||
const std::pair<diskann_id_t, VectorInfo> &rhs) const {
|
||||
return compare_(lhs.second.dist_, rhs.second.dist_);
|
||||
}
|
||||
|
||||
private:
|
||||
std::less<float> compare_;
|
||||
};
|
||||
|
||||
/*! Key Value Vector Heap
|
||||
*/
|
||||
using TopkHeap = ailego::Heap<std::pair<diskann_id_t, VectorInfo>,
|
||||
KeyValueVectorHeapComparer>;
|
||||
|
||||
struct DiskAnnMetaHeader {
|
||||
public:
|
||||
uint64_t doc_cnt;
|
||||
uint64_t ndims;
|
||||
uint64_t medoid;
|
||||
uint64_t max_node_size;
|
||||
uint64_t max_degree;
|
||||
uint64_t node_per_sector;
|
||||
uint64_t vamana_frozen_num;
|
||||
uint64_t vamana_frozen_loc;
|
||||
uint64_t append_reorder_data;
|
||||
uint64_t index_size;
|
||||
uint8_t reserved[4016]; /// pad DiskAnnMetaHeader to 4096 bytes
|
||||
|
||||
DiskAnnMetaHeader() {
|
||||
clear();
|
||||
}
|
||||
|
||||
DiskAnnMetaHeader(const DiskAnnMetaHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
|
||||
DiskAnnMetaHeader &operator=(const DiskAnnMetaHeader &header) {
|
||||
if (this != &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
void reset() {
|
||||
doc_cnt = 0U;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
memset(this, 0, sizeof(DiskAnnMetaHeader));
|
||||
}
|
||||
};
|
||||
|
||||
struct DiskAnnPqMeta {
|
||||
public:
|
||||
uint64_t full_pivot_data_size{0};
|
||||
uint64_t centroid_data_size{0};
|
||||
uint64_t chunk_offsets_size{0};
|
||||
uint64_t chunk_num{0};
|
||||
uint8_t reserved[128];
|
||||
|
||||
DiskAnnPqMeta() {
|
||||
clear();
|
||||
}
|
||||
|
||||
DiskAnnPqMeta(const DiskAnnPqMeta &meta) {
|
||||
memcpy(this, &meta, sizeof(meta));
|
||||
}
|
||||
|
||||
DiskAnnPqMeta &operator=(const DiskAnnPqMeta &meta) {
|
||||
memcpy(this, &meta, sizeof(meta));
|
||||
return *this;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
memset(this, 0, sizeof(DiskAnnPqMeta));
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(DiskAnnMetaHeader) == 4096,
|
||||
"DiskAnnMetaHeader size must stay 4096 bytes (on-disk format)");
|
||||
|
||||
static_assert(sizeof(DiskAnnPqMeta) % 32 == 0,
|
||||
"DiskAnnPqMeta size must be a multiple of 32 bytes");
|
||||
|
||||
class DiskAnnEntity {
|
||||
public:
|
||||
DiskAnnEntity() = default;
|
||||
virtual ~DiskAnnEntity() = default;
|
||||
|
||||
//! Constructor
|
||||
DiskAnnEntity(const DiskAnnMetaHeader &meta_header,
|
||||
const DiskAnnPqMeta &pq_meta) {
|
||||
meta_header_ = meta_header;
|
||||
pq_meta_ = pq_meta;
|
||||
}
|
||||
|
||||
//! DiskAnnEntity Pointerd;
|
||||
typedef std::shared_ptr<DiskAnnEntity> Pointer;
|
||||
|
||||
public:
|
||||
static size_t AlignSize(size_t size) {
|
||||
return (size + 0xFFF) & (~0xFFF);
|
||||
}
|
||||
|
||||
public:
|
||||
virtual int add_vector(diskann_key_t /*key*/, const void * /*vec*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual const void *get_vector(diskann_id_t /*id*/) const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t /*id*/) const {
|
||||
return std::make_pair(0, nullptr);
|
||||
}
|
||||
|
||||
virtual int set_neighbors(
|
||||
diskann_id_t /*id*/, const std::vector<diskann_id_t> & /*neighbor_ids*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual int add_neighbor(diskann_id_t /*id*/, diskann_id_t /*neighbor_id*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get node id of primary key
|
||||
virtual diskann_id_t get_id(diskann_key_t key) const = 0;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual diskann_key_t get_key(diskann_id_t id) const = 0;
|
||||
|
||||
public:
|
||||
uint64_t max_node_size() const {
|
||||
return meta_header_.max_node_size;
|
||||
}
|
||||
|
||||
uint64_t medoid() const {
|
||||
return meta_header_.medoid;
|
||||
}
|
||||
|
||||
uint64_t *mutable_medoid() {
|
||||
return &meta_header_.medoid;
|
||||
}
|
||||
|
||||
uint64_t node_per_sector() const {
|
||||
return meta_header_.node_per_sector;
|
||||
}
|
||||
|
||||
uint64_t pq_chunk_num() {
|
||||
return pq_meta_.chunk_num;
|
||||
}
|
||||
|
||||
uint64_t doc_cnt() const {
|
||||
return meta_header_.doc_cnt;
|
||||
}
|
||||
|
||||
uint64_t *mutable_doc_cnt() {
|
||||
return &meta_header_.doc_cnt;
|
||||
}
|
||||
|
||||
uint64_t max_degree() {
|
||||
return meta_header_.max_degree;
|
||||
}
|
||||
|
||||
DiskAnnPqMeta *mutable_pq_meta() {
|
||||
return &pq_meta_;
|
||||
}
|
||||
|
||||
public:
|
||||
virtual const DiskAnnEntity::Pointer clone() const {
|
||||
LOG_ERROR("Update neighbors not implemented");
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
public:
|
||||
const static std::string kDiskAnnVectorSegmentId;
|
||||
const static std::string kDiskAnnMetaSegmentId;
|
||||
const static std::string kDiskAnnPqMetaSegmentId;
|
||||
const static std::string kDiskAnnPqDataSegmentId;
|
||||
const static std::string kDiskAnnDummpySegmentId;
|
||||
const static std::string kDiskAnnMappingSegmentId;
|
||||
const static std::string kDiskAnnKeyMappingSegmentId;
|
||||
const static std::string kDiskAnnEntryPointSegmentId;
|
||||
const static std::string kDiskAnnKeySegmentId;
|
||||
|
||||
constexpr static float kDefaultBFNegativeProbility = 0.001f;
|
||||
constexpr static float kDefaultGraphSlackFactor = 1.3f;
|
||||
constexpr static float kDefaultAlpha = 1.2f;
|
||||
constexpr static uint32_t kDefaultMaxOcclusionSize = 750;
|
||||
constexpr static uint32_t kDefaultMaxDegree = 100;
|
||||
constexpr static uint32_t kDefaultCompressBatchSize = 5000000;
|
||||
constexpr static uint32_t kRevision = 0U;
|
||||
|
||||
protected:
|
||||
DiskAnnMetaHeader meta_header_;
|
||||
DiskAnnPqMeta pq_meta_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,299 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_file_reader.h"
|
||||
#include <cassert>
|
||||
#include <cerrno>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
|
||||
#define MAX_EVENTS 1024
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
typedef struct io_event io_event_t;
|
||||
typedef struct iocb iocb_t;
|
||||
#endif
|
||||
|
||||
int setup_io_ctx(IOContext &ctx) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
int ret = io_setup(MAX_EVENTS, &ctx);
|
||||
|
||||
return ret;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int destroy_io_ctx(IOContext &ctx) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
int ret = io_destroy(ctx);
|
||||
|
||||
return ret;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int execute_io_pread(int fd, std::vector<AlignedRead> &read_reqs) {
|
||||
for (auto &req : read_reqs) {
|
||||
ssize_t bytes_read = ::pread(fd, req.buf, req.len, req.offset);
|
||||
if (bytes_read < 0) {
|
||||
LOG_ERROR("pread failed; errno=%d, %s, offset=%lu, len=%lu", errno,
|
||||
::strerror(errno), (unsigned long)req.offset,
|
||||
(unsigned long)req.len);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if ((size_t)bytes_read != req.len) {
|
||||
LOG_ERROR("pread short read; got=%zd, expected=%lu", bytes_read,
|
||||
(unsigned long)req.len);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int execute_io(IOContext ctx, int fd, std::vector<AlignedRead> &read_reqs,
|
||||
uint64_t n_retries = 0) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
uint64_t iters = DiskAnnUtil::div_round_up(read_reqs.size(), MAX_EVENTS);
|
||||
|
||||
for (uint64_t iter = 0; iter < iters; iter++) {
|
||||
uint64_t n_ops = std::min((uint64_t)read_reqs.size() - (iter * MAX_EVENTS),
|
||||
(uint64_t)MAX_EVENTS);
|
||||
|
||||
std::vector<iocb_t *> cbs(n_ops, nullptr);
|
||||
std::vector<io_event_t> evts(n_ops);
|
||||
std::vector<struct iocb> cb(n_ops);
|
||||
for (uint64_t j = 0; j < n_ops; j++) {
|
||||
io_prep_pread(cb.data() + j, fd, read_reqs[j + iter * MAX_EVENTS].buf,
|
||||
read_reqs[j + iter * MAX_EVENTS].len,
|
||||
read_reqs[j + iter * MAX_EVENTS].offset);
|
||||
}
|
||||
|
||||
for (uint64_t i = 0; i < n_ops; i++) {
|
||||
cbs[i] = cb.data() + i;
|
||||
}
|
||||
|
||||
size_t n_tries = 0;
|
||||
// Phase 1: io_submit with retry.
|
||||
while (true) {
|
||||
int ret = io_submit(ctx, (int64_t)n_ops, cbs.data());
|
||||
if (ret == (int)n_ops) {
|
||||
break;
|
||||
}
|
||||
if ((ret == -EAGAIN || ret == -EINTR) && n_tries < n_retries) {
|
||||
n_tries++;
|
||||
continue;
|
||||
}
|
||||
LOG_WARN(
|
||||
"io_submit failed; returned: %d, expected=%lu. falling back to "
|
||||
"pread",
|
||||
ret, n_ops);
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
|
||||
// Phase 2: io_getevents with retry (never re-submits).
|
||||
n_tries = 0;
|
||||
while (true) {
|
||||
int ret = io_getevents(ctx, (int64_t)n_ops, (int64_t)n_ops, evts.data(),
|
||||
nullptr);
|
||||
if (ret == (int)n_ops) {
|
||||
break;
|
||||
}
|
||||
if (ret == -EINTR && n_tries < n_retries) {
|
||||
n_tries++;
|
||||
continue;
|
||||
}
|
||||
LOG_WARN(
|
||||
"io_getevents failed; returned: %d, expected=%lu, errno=%d, %s, "
|
||||
"falling back to pread",
|
||||
ret, n_ops, errno, ::strerror(-ret));
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
|
||||
// Phase 3: verify each completed read (res must equal requested length).
|
||||
bool all_ok = true;
|
||||
for (uint64_t i = 0; i < n_ops; i++) {
|
||||
int64_t expected_len = read_reqs[i + iter * MAX_EVENTS].len;
|
||||
if ((int64_t)evts[i].res != expected_len) {
|
||||
LOG_WARN("aio request %zu failed: res=%ld, expected=%ld, offset=%zu",
|
||||
(size_t)i, (long)evts[i].res, (long)expected_len,
|
||||
(size_t)read_reqs[i + iter * MAX_EVENTS].offset);
|
||||
all_ok = false;
|
||||
}
|
||||
}
|
||||
if (!all_ok) {
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
#else
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
#endif
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::LinuxAlignedFileReader(int file_desc) {
|
||||
this->file_desc = file_desc;
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::LinuxAlignedFileReader() {
|
||||
this->file_desc = -1;
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::~LinuxAlignedFileReader() {
|
||||
deregister_all_threads();
|
||||
if (file_desc >= 0) {
|
||||
::close(file_desc);
|
||||
file_desc = -1;
|
||||
}
|
||||
}
|
||||
|
||||
IOContext &LinuxAlignedFileReader::get_ctx() {
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
auto it = ctx_map.find(std::this_thread::get_id());
|
||||
if (it == ctx_map.end()) {
|
||||
LOG_ERROR("bad thread access; returning -1 as io_context_t");
|
||||
return this->bad_ctx;
|
||||
} else {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::register_thread() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
auto thread_id = std::this_thread::get_id();
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
if (ctx_map.find(thread_id) != ctx_map.end()) {
|
||||
LOG_ERROR("multiple calls to register_thread from the same thread");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
IOContext ctx = nullptr;
|
||||
|
||||
int ret = io_setup(MAX_EVENTS, &ctx);
|
||||
if (ret != 0) {
|
||||
lk.unlock();
|
||||
if (ret == -EAGAIN) {
|
||||
LOG_ERROR(
|
||||
"io_setup failed with EAGAIN: Consider increasing "
|
||||
"/proc/sys/fs/aio-max-nr");
|
||||
} else {
|
||||
LOG_ERROR("io_setup failed; returned: %d, %s", ret, ::strerror(-ret));
|
||||
;
|
||||
}
|
||||
} else {
|
||||
LOG_INFO("allocating ctx: %lu", (uint64_t)ctx);
|
||||
|
||||
ctx_map[thread_id] = ctx;
|
||||
}
|
||||
|
||||
lk.unlock();
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::deregister_thread() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
auto thread_id = std::this_thread::get_id();
|
||||
IOContext ctx;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(ctx_mut);
|
||||
auto it = ctx_map.find(thread_id);
|
||||
if (it == ctx_map.end()) {
|
||||
LOG_ERROR("deregister_thread: thread not registered");
|
||||
return;
|
||||
}
|
||||
ctx = it->second;
|
||||
ctx_map.erase(it);
|
||||
}
|
||||
|
||||
// io_destroy is a syscall; keep it outside the lock to avoid blocking others
|
||||
io_destroy(ctx);
|
||||
LOG_INFO("returned ctx from thread");
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::deregister_all_threads() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
for (auto x = ctx_map.begin(); x != ctx_map.end(); x++) {
|
||||
IOContext ctx = x->second;
|
||||
io_destroy(ctx);
|
||||
}
|
||||
ctx_map.clear();
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::open(const std::string &fname) {
|
||||
int flags = O_RDONLY;
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
flags |= O_DIRECT | O_LARGEFILE;
|
||||
#endif
|
||||
|
||||
this->file_desc = ::open(fname.c_str(), flags);
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
// O_DIRECT may not be supported on all filesystems (e.g. tmpfs, overlay).
|
||||
// Fall back to regular buffered I/O when it fails.
|
||||
if (this->file_desc == -1) {
|
||||
LOG_WARN(
|
||||
"open with O_DIRECT failed for %s (errno=%d: %s), "
|
||||
"falling back to buffered I/O",
|
||||
fname.c_str(), errno, ::strerror(errno));
|
||||
this->file_desc = ::open(fname.c_str(), O_RDONLY | O_LARGEFILE);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (this->file_desc == -1) {
|
||||
LOG_ERROR("Failed to open file: %s (errno=%d: %s)", fname.c_str(), errno,
|
||||
::strerror(errno));
|
||||
}
|
||||
|
||||
LOG_INFO("Opened file : %s", fname.c_str());
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::close() {
|
||||
if (file_desc >= 0) {
|
||||
::close(file_desc);
|
||||
file_desc = -1;
|
||||
}
|
||||
}
|
||||
|
||||
int LinuxAlignedFileReader::read(std::vector<AlignedRead> &read_reqs,
|
||||
IOContext &ctx, bool async) {
|
||||
if (async == true) {
|
||||
LOG_WARN("Async currently not supported");
|
||||
}
|
||||
|
||||
if (this->file_desc == -1) {
|
||||
LOG_ERROR("Attempt to read from invalid file descriptor");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
int ret = execute_io(ctx, this->file_desc, read_reqs);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#define MAX_IO_DEPTH 128
|
||||
|
||||
#include <fcntl.h>
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
#include <libaio.h>
|
||||
#endif
|
||||
|
||||
#include <unistd.h>
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
typedef io_context_t IOContext;
|
||||
#else
|
||||
typedef uint32_t IOContext;
|
||||
#endif
|
||||
|
||||
int setup_io_ctx(IOContext &ctx);
|
||||
int destroy_io_ctx(IOContext &ctx);
|
||||
|
||||
struct AlignedRead {
|
||||
uint64_t offset;
|
||||
uint64_t len;
|
||||
void *buf;
|
||||
|
||||
AlignedRead() : offset(0), len(0), buf(nullptr) {}
|
||||
|
||||
AlignedRead(uint64_t offset, uint64_t len, void *buf)
|
||||
: offset(offset), len(len), buf(buf) {
|
||||
ailego_assert(static_cast<size_t>(offset) % 512 == 0);
|
||||
ailego_assert(static_cast<size_t>(len) % 512 == 0);
|
||||
ailego_assert(reinterpret_cast<size_t>(buf) % 512 == 0);
|
||||
}
|
||||
};
|
||||
|
||||
class AlignedFileReader {
|
||||
protected:
|
||||
std::map<std::thread::id, IOContext> ctx_map;
|
||||
std::mutex ctx_mut;
|
||||
|
||||
public:
|
||||
virtual IOContext &get_ctx() = 0;
|
||||
|
||||
virtual ~AlignedFileReader() {};
|
||||
|
||||
virtual void register_thread() = 0;
|
||||
virtual void deregister_thread() = 0;
|
||||
virtual void deregister_all_threads() = 0;
|
||||
|
||||
virtual void open(const std::string &fname) = 0;
|
||||
virtual void close() = 0;
|
||||
|
||||
virtual int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
|
||||
bool async = false) = 0;
|
||||
};
|
||||
|
||||
class LinuxAlignedFileReader : public AlignedFileReader {
|
||||
private:
|
||||
uint64_t file_sz;
|
||||
int file_desc;
|
||||
|
||||
IOContext bad_ctx = (IOContext)-1;
|
||||
|
||||
public:
|
||||
LinuxAlignedFileReader();
|
||||
LinuxAlignedFileReader(int file_desc);
|
||||
~LinuxAlignedFileReader();
|
||||
|
||||
public:
|
||||
IOContext &get_ctx();
|
||||
|
||||
void register_thread();
|
||||
void deregister_thread();
|
||||
void deregister_all_threads();
|
||||
void open(const std::string &fname);
|
||||
void close();
|
||||
|
||||
int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
|
||||
bool async = false);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,325 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct DiskAnnIndexHolderMeta {
|
||||
uint32_t element_size_;
|
||||
uint32_t key_size_;
|
||||
uint32_t sector_size_;
|
||||
uint32_t doc_cnt_;
|
||||
uint8_t reserve_[];
|
||||
};
|
||||
|
||||
class DiskAnnIndexHolder : public IndexHolder {
|
||||
public:
|
||||
typedef std::shared_ptr<DiskAnnIndexHolder> Pointer;
|
||||
|
||||
public:
|
||||
enum Status { STATUS_UNINITED = 0, STATUS_WRITE = 1, STATUS_READ = 2 };
|
||||
|
||||
public:
|
||||
static constexpr uint32_t kDataSectorSize = 128 * 1024;
|
||||
static constexpr uint32_t kMetaSectorSize = 4096;
|
||||
|
||||
public:
|
||||
inline static uint32_t get_sector_id(uint32_t id, uint32_t sector_vec_num) {
|
||||
return id / sector_vec_num;
|
||||
}
|
||||
|
||||
inline static uint32_t get_sector_offset(uint32_t id, uint32_t sector_vec_num,
|
||||
uint32_t data_size) {
|
||||
return (id % sector_vec_num) * data_size;
|
||||
}
|
||||
|
||||
public:
|
||||
/*! Random Access Index Holder Iterator
|
||||
*/
|
||||
class Iterator : public IndexHolder::Iterator {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::unique_ptr<Iterator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
Iterator(DiskAnnIndexHolder *owner)
|
||||
: holder_(owner), sector_id_{0}, sector_offset_{0} {
|
||||
path_ = holder_->path();
|
||||
data_size_ = holder_->data_size();
|
||||
data_sector_size_ = holder_->data_sector_size();
|
||||
meta_sector_size_ = holder_->meta_sector_size();
|
||||
|
||||
sector_buffer_.resize(data_sector_size_);
|
||||
|
||||
sector_vec_num_ = data_sector_size_ / data_size_;
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {
|
||||
if (file_.is_open()) {
|
||||
file_.close();
|
||||
}
|
||||
}
|
||||
|
||||
int init() {
|
||||
file_.open(path_, std::ios::in);
|
||||
if (!file_.is_open()) {
|
||||
LOG_ERROR("file can not create, %s", path_.c_str());
|
||||
return IndexError_OpenFile;
|
||||
}
|
||||
|
||||
file_.seekg(meta_sector_size_);
|
||||
|
||||
read_sector();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const void *data(void) const override {
|
||||
const uint8_t *data_ptr =
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
|
||||
return data_ptr + sector_offset_ + sizeof(diskann_key_t);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
bool is_valid(void) const override {
|
||||
return id_ < holder_->count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
uint64_t key(void) const override {
|
||||
const uint8_t *data_ptr =
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
|
||||
uint64_t key =
|
||||
*reinterpret_cast<const uint64_t *>(data_ptr + sector_offset_);
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
void next(void) override {
|
||||
++id_;
|
||||
|
||||
uint32_t sector_id = get_sector_id(id_, sector_vec_num_);
|
||||
if (sector_id > sector_id_) {
|
||||
file_.seekg(sector_id * data_sector_size_ + meta_sector_size_);
|
||||
read_sector();
|
||||
sector_id_ = sector_id;
|
||||
}
|
||||
|
||||
sector_offset_ = get_sector_offset(id_, sector_vec_num_, data_size_);
|
||||
}
|
||||
|
||||
int read_sector() {
|
||||
file_.read(&((sector_buffer_)[0]), data_sector_size_);
|
||||
if (!file_) {
|
||||
LOG_ERROR("Failed to read sector from file: %s", path_.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
DiskAnnIndexHolder *holder_{nullptr};
|
||||
std::string path_;
|
||||
std::ifstream file_;
|
||||
uint32_t sector_id_{0};
|
||||
std::string sector_buffer_;
|
||||
uint32_t id_{0};
|
||||
uint32_t data_size_{0};
|
||||
uint32_t sector_offset_{0};
|
||||
uint32_t data_sector_size_{0};
|
||||
uint32_t meta_sector_size_{0};
|
||||
uint32_t sector_vec_num_{0};
|
||||
};
|
||||
|
||||
public:
|
||||
DiskAnnIndexHolder(IndexMeta &meta, std::string &path) {
|
||||
path_ = path;
|
||||
|
||||
data_size_ = meta.element_size() + sizeof(diskann_key_t);
|
||||
dimension_ = meta.dimension();
|
||||
type_ = meta.data_type();
|
||||
|
||||
element_size_ = meta.element_size();
|
||||
sector_vec_num_ = data_sector_size_ / data_size_;
|
||||
padding_size_ = data_sector_size_ - sector_vec_num_ * data_size_;
|
||||
sector_buffer_.resize(data_sector_size_);
|
||||
sector_internal_id_ = 0;
|
||||
}
|
||||
|
||||
~DiskAnnIndexHolder() override {
|
||||
if (file_.is_open()) {
|
||||
file_.close();
|
||||
}
|
||||
}
|
||||
|
||||
//! Init
|
||||
int init() {
|
||||
file_.open(path_, std::ios::out | std::ios::trunc);
|
||||
|
||||
if (!file_.is_open()) {
|
||||
LOG_ERROR("file can not create, %s", path_.c_str());
|
||||
return IndexError_OpenFile;
|
||||
}
|
||||
|
||||
DiskAnnIndexHolderMeta holder_meta;
|
||||
holder_meta.element_size_ = element_size_;
|
||||
holder_meta.key_size_ = sizeof(diskann_key_t);
|
||||
holder_meta.sector_size_ = data_sector_size_;
|
||||
|
||||
std::vector<uint8_t> empty_sector;
|
||||
empty_sector.resize(meta_sector_size_);
|
||||
|
||||
std::memset(&(empty_sector[0]), 0, meta_sector_size_);
|
||||
std::memcpy(&(empty_sector[0]), &holder_meta,
|
||||
sizeof(DiskAnnIndexHolderMeta));
|
||||
|
||||
file_.write(reinterpret_cast<const char *>(&(empty_sector[0])),
|
||||
meta_sector_size_);
|
||||
if (!file_) {
|
||||
LOG_ERROR("Failed to write meta sector to file: %s", path_.c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
status_ = STATUS_WRITE;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int close() {
|
||||
if (sector_internal_id_ != 0) {
|
||||
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
|
||||
data_sector_size_);
|
||||
}
|
||||
|
||||
file_.close();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
size_t count(void) const override {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
size_t dimension(void) const override {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return type_;
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
size_t element_size(void) const override {
|
||||
return element_size_;
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
auto pointer = std::make_unique<DiskAnnIndexHolder::Iterator>(this);
|
||||
|
||||
if (pointer->init() != 0) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return pointer;
|
||||
}
|
||||
|
||||
int emplace(uint64_t pkey, const void *vec) {
|
||||
if (status_ != STATUS_WRITE) {
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
uint8_t *data_ptr = reinterpret_cast<uint8_t *>(&(sector_buffer_[0])) +
|
||||
sector_internal_id_ * data_size_;
|
||||
std::memcpy(data_ptr, &pkey, sizeof(diskann_key_t));
|
||||
std::memcpy(data_ptr + sizeof(diskann_key_t), vec, element_size_);
|
||||
|
||||
sector_internal_id_++;
|
||||
if (sector_internal_id_ >= sector_vec_num_) {
|
||||
std::memset(data_ptr + data_size_, 0, padding_size_);
|
||||
|
||||
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
|
||||
data_sector_size_);
|
||||
|
||||
sector_internal_id_ = 0;
|
||||
sector_id_++;
|
||||
}
|
||||
|
||||
count_++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t data_sector_size() {
|
||||
return data_sector_size_;
|
||||
}
|
||||
|
||||
uint32_t meta_sector_size() {
|
||||
return meta_sector_size_;
|
||||
}
|
||||
|
||||
uint32_t data_size() {
|
||||
return data_size_;
|
||||
}
|
||||
|
||||
uint32_t *mutable_sector_id() {
|
||||
return §or_id_;
|
||||
}
|
||||
|
||||
uint32_t sector_id() {
|
||||
return sector_id_;
|
||||
}
|
||||
|
||||
std::string &path() {
|
||||
return path_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string path_;
|
||||
std::ofstream file_;
|
||||
uint32_t element_size_{0};
|
||||
uint32_t dimension_{0};
|
||||
IndexMeta::DataType type_{IndexMeta::DataType::DT_UNDEFINED};
|
||||
uint32_t sector_vec_num_{0};
|
||||
uint32_t data_size_{0};
|
||||
uint32_t padding_size_{0};
|
||||
uint32_t meta_sector_size_{DiskAnnIndexHolder::kMetaSectorSize};
|
||||
uint32_t data_sector_size_{DiskAnnIndexHolder::kDataSectorSize};
|
||||
std::string sector_buffer_;
|
||||
uint32_t sector_internal_id_{0};
|
||||
uint32_t sector_id_{0};
|
||||
uint32_t count_{0};
|
||||
uint32_t status_{STATUS_UNINITED};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_provider.h>
|
||||
#include <zvec/core/framework/index_searcher.h>
|
||||
#include <zvec/core/framework/index_streamer.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! IndexProvider implementation backed by a DiskAnn entity.
|
||||
//!
|
||||
//! Used by ``MixedStreamerReducer`` during segment merge: the reducer needs
|
||||
//! to walk every vector held by a source DiskAnn streamer and feed it into
|
||||
//! the merge target. Vectors are read on demand from the entity's on-disk
|
||||
//! vector segment via ``DiskAnnEntity::get_vector(id)``.
|
||||
class DiskAnnIndexProvider : public IndexProvider {
|
||||
public:
|
||||
DiskAnnIndexProvider(const IndexMeta &meta,
|
||||
const DiskAnnEntity::Pointer &entity,
|
||||
const std::string &owner)
|
||||
: meta_(meta), entity_(entity), owner_class_(owner) {}
|
||||
|
||||
DiskAnnIndexProvider(const DiskAnnIndexProvider &) = delete;
|
||||
DiskAnnIndexProvider &operator=(const DiskAnnIndexProvider &) = delete;
|
||||
|
||||
public:
|
||||
IndexProvider::Iterator::Pointer create_iterator() override {
|
||||
return IndexProvider::Iterator::Pointer(new (std::nothrow)
|
||||
Iterator(entity_));
|
||||
}
|
||||
|
||||
size_t count(void) const override {
|
||||
return entity_->doc_cnt();
|
||||
}
|
||||
|
||||
size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
diskann_id_t id = entity_->get_id(static_cast<diskann_key_t>(key));
|
||||
if (id == kInvalidId) {
|
||||
return nullptr;
|
||||
}
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
explicit Iterator(const DiskAnnEntity::Pointer &entity)
|
||||
: entity_(entity), cur_id_(0U) {
|
||||
cur_id_ = next_valid_id(0U);
|
||||
}
|
||||
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < static_cast<diskann_id_t>(entity_->doc_cnt());
|
||||
}
|
||||
|
||||
uint64_t key(void) const override {
|
||||
return static_cast<uint64_t>(entity_->get_key(cur_id_));
|
||||
}
|
||||
|
||||
void next(void) override {
|
||||
cur_id_ = next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Skip ids that map to ``kInvalidKey`` (deleted / never populated slots).
|
||||
diskann_id_t next_valid_id(diskann_id_t start_id) const {
|
||||
const auto total = static_cast<diskann_id_t>(entity_->doc_cnt());
|
||||
for (diskann_id_t i = start_id; i < total; ++i) {
|
||||
if (entity_->get_key(i) != kInvalidKey) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
diskann_id_t cur_id_;
|
||||
};
|
||||
|
||||
IndexMeta meta_;
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
std::string owner_class_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_searcher_entity.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnIndexer {
|
||||
public:
|
||||
typedef std::shared_ptr<DiskAnnIndexer> Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnIndexer(const IndexMeta &meta);
|
||||
~DiskAnnIndexer();
|
||||
|
||||
public:
|
||||
int init(DiskAnnSearcherEntity &entity);
|
||||
int load_cache_list(const std::vector<diskann_id_t> &node_list);
|
||||
|
||||
void cache_bfs_levels(uint64_t num_nodes_to_cache,
|
||||
std::vector<diskann_id_t> &node_list);
|
||||
|
||||
int cached_beam_search(DiskAnnContext *ctx);
|
||||
int cached_beam_search_by_group(DiskAnnContext *ctx);
|
||||
|
||||
int cached_beam_search_in_mem(DiskAnnContext *ctx);
|
||||
|
||||
int knn_search(DiskAnnContext *ctx);
|
||||
int linear_search(DiskAnnContext *ctx);
|
||||
int keys_search(const std::vector<diskann_key_t> &keys, DiskAnnContext *ctx);
|
||||
|
||||
int get_vector(diskann_id_t id, IndexContext::Pointer &context,
|
||||
std::string &vector);
|
||||
|
||||
diskann_key_t get_key(diskann_id_t id) const;
|
||||
diskann_id_t get_id(diskann_key_t key) const;
|
||||
|
||||
//! Copy element_size() bytes from src into a new vector value string
|
||||
std::string make_vector_copy(const void *src) const {
|
||||
return std::string(static_cast<const char *>(src), meta_.element_size());
|
||||
}
|
||||
|
||||
std::vector<bool> read_nodes(
|
||||
const std::vector<diskann_id_t> &node_ids,
|
||||
std::vector<void *> &coord_buffers,
|
||||
std::vector<std::pair<uint32_t, diskann_id_t *>> &nbr_buffers);
|
||||
|
||||
protected:
|
||||
int use_medroids_data_as_centroids();
|
||||
|
||||
private:
|
||||
DiskAnnSearcherEntity *entity_;
|
||||
|
||||
IndexStorage::Pointer storage_{};
|
||||
IndexMeta meta_;
|
||||
|
||||
uint32_t max_degree_{0};
|
||||
uint32_t node_per_sector_{0};
|
||||
uint32_t max_node_size_{0};
|
||||
uint64_t pq_chunk_num_{0};
|
||||
uint64_t disk_bytes_per_point_{0};
|
||||
uint64_t aligned_dim_{0};
|
||||
uint64_t index_segment_offset_{0};
|
||||
uint64_t sector_num_per_node_{0};
|
||||
|
||||
float *centroid_data_{nullptr};
|
||||
|
||||
diskann_id_t medoid_;
|
||||
std::vector<diskann_id_t> entrypoints_;
|
||||
|
||||
std::shared_ptr<LinuxAlignedFileReader> reader_{nullptr};
|
||||
|
||||
PQTable::Pointer pq_table_;
|
||||
|
||||
IOContext init_ctx_{0};
|
||||
|
||||
std::vector<diskann_id_t> neighbor_cache_buffer_;
|
||||
void *coord_cache_buf_{nullptr};
|
||||
|
||||
std::map<diskann_id_t, void *> coord_cache_;
|
||||
std::map<diskann_id_t, std::pair<uint32_t, diskann_id_t *>> neighbor_cache_;
|
||||
|
||||
uint32_t beam_width_{2};
|
||||
uint32_t io_limit_{std::numeric_limits<uint32_t>::max()};
|
||||
|
||||
uint64_t doc_cnt_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_DEGREE(
|
||||
"zvec.diskann.builder.max_degree");
|
||||
static const std::string PARAM_DISKANN_BUILDER_LIST_SIZE(
|
||||
"zvec.diskann.builder.list_size");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MEMORY_LIMIT(
|
||||
"zvec.diskann.builder.memory_limit");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MEMORY_BUDGET(
|
||||
"zvec.diskann.builder.memory_budget");
|
||||
static const std::string PARAM_DISKANN_BUILDER_DISK_PQ_DIM(
|
||||
"zvec.diskann.builder.disk_pq_dim");
|
||||
static const std::string PARAM_DISKANN_BUILDER_THREAD_COUNT(
|
||||
"zvec.diskann.builder.thread_count");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT(
|
||||
"zvec.diskann.builder.max_train_sample_count");
|
||||
static const std::string PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO(
|
||||
"zvec.diskann.builder.train_sample_ratio");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM(
|
||||
"zvec.diskann.builder.max_pq_chunk_num");
|
||||
|
||||
static const std::string PARAM_DISKANN_SEARCHER_LIST_SIZE(
|
||||
"zvec.diskann.searcher.list_size");
|
||||
static const std::string PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM(
|
||||
"zvec.diskann.searcher.cache_node_num");
|
||||
|
||||
static const std::string PARAM_DISKANN_REDUCER_INDEX_NAME(
|
||||
"zvec.diskann.reducer.index_name");
|
||||
static const std::string PARAM_DISKANN_REDUCER_WORKING_PATH(
|
||||
"zvec.diskann.reducer.working_path");
|
||||
static const std::string PARAM_DISKANN_REDUCER_NUM_OF_ADD_THREADS(
|
||||
"zvec.diskann.reducer.num_of_add_threads");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
PQTable::PQTable(const IndexMeta &meta, uint32_t chunk_num)
|
||||
: chunk_num_(chunk_num) {
|
||||
meta_ = meta;
|
||||
|
||||
if (meta.metric_name() == "Cosine") {
|
||||
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
|
||||
meta_.set_dimension(meta.dimension() - 1);
|
||||
} else {
|
||||
meta_.set_dimension(meta.dimension() - 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PQTable::~PQTable() {}
|
||||
|
||||
int PQTable::init(std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets,
|
||||
std::vector<uint8_t> &pq_data) {
|
||||
full_pivot_data_ = std::move(full_pivot_data);
|
||||
centroid_ = std::move(centroid);
|
||||
chunk_offsets_ = std::move(chunk_offsets);
|
||||
pq_data_ = std::move(pq_data);
|
||||
|
||||
// alloc and compute transpose
|
||||
transposed_tables_.resize(kPQCentroidNum * meta_.element_size());
|
||||
|
||||
uint32_t dim = meta_.dimension();
|
||||
uint32_t type = meta_.data_type();
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
float *transposed_tables_ptr =
|
||||
reinterpret_cast<float *>(&transposed_tables_[0]);
|
||||
float *full_pivot_data_ptr =
|
||||
reinterpret_cast<float *>(&full_pivot_data_[0]);
|
||||
for (size_t i = 0; i < kPQCentroidNum; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
transposed_tables_ptr[j * kPQCentroidNum + i] =
|
||||
full_pivot_data_ptr[i * dim + j];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
ailego::Float16 *transposed_tables_ptr =
|
||||
reinterpret_cast<ailego::Float16 *>(&transposed_tables_[0]);
|
||||
ailego::Float16 *full_pivot_data_ptr =
|
||||
reinterpret_cast<ailego::Float16 *>(&full_pivot_data_[0]);
|
||||
for (size_t i = 0; i < kPQCentroidNum; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
transposed_tables_ptr[j * kPQCentroidNum + i] =
|
||||
full_pivot_data_ptr[i * dim + j];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
LOG_ERROR("unsupported type, type: %u", type);
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void PQTable::aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
|
||||
const uint8_t *all_coords, size_t dim,
|
||||
uint8_t *out) {
|
||||
for (size_t i = 0; i < id_num; i++) {
|
||||
memcpy(out + i * dim, all_coords + ids[i] * dim, dim * sizeof(uint8_t));
|
||||
}
|
||||
}
|
||||
|
||||
void PQTable::pq_dist_lookup(const uint8_t *pq_ids, size_t id_num,
|
||||
size_t pq_nchunks, const float *pq_dist_buffer,
|
||||
float *dists_out) {
|
||||
ailego_prefetch(dists_out);
|
||||
ailego_prefetch(pq_ids);
|
||||
ailego_prefetch(pq_ids + 64);
|
||||
ailego_prefetch(pq_ids + 128);
|
||||
|
||||
memset(dists_out, 0, id_num * sizeof(float));
|
||||
|
||||
for (size_t chunk = 0; chunk < pq_nchunks; chunk++) {
|
||||
const float *chunk_dists = pq_dist_buffer + kPQCentroidNum * chunk;
|
||||
if (chunk < pq_nchunks - 1) {
|
||||
ailego_prefetch(chunk_dists + kPQCentroidNum);
|
||||
}
|
||||
for (size_t idx = 0; idx < id_num; idx++) {
|
||||
uint8_t pq_centerid = pq_ids[pq_nchunks * idx + chunk];
|
||||
dists_out[idx] += chunk_dists[pq_centerid];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class PQTable {
|
||||
public:
|
||||
typedef std::shared_ptr<PQTable> Pointer;
|
||||
|
||||
public:
|
||||
static constexpr uint32_t kPQBitNum = 8;
|
||||
static constexpr uint32_t kPQCentroidNum = 1 << kPQBitNum;
|
||||
static constexpr uint32_t kMaxTrainSampleCount = 200000;
|
||||
static constexpr double kTrainSampleRatio = 1.0;
|
||||
static constexpr uint32_t kMeanIterNum = 12;
|
||||
|
||||
public:
|
||||
PQTable(const IndexMeta &meta, uint32_t chunk_num);
|
||||
virtual ~PQTable();
|
||||
|
||||
int init(std::vector<uint8_t> &table, std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets, std::vector<uint8_t> &pq_data);
|
||||
|
||||
|
||||
template <typename T>
|
||||
void compute_distance_table_ip(const T *query_vec, float *dist_vec) {
|
||||
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
|
||||
|
||||
const T *transposed_tables_ptr =
|
||||
reinterpret_cast<const T *>(transposed_tables_.data());
|
||||
// chunk wise distance computation
|
||||
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
|
||||
// sum (q-c)^2 for the dimensions associated with this chunk
|
||||
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
|
||||
|
||||
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
|
||||
j++) {
|
||||
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
|
||||
|
||||
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
|
||||
float centor_data = centers_dim_vec[idx];
|
||||
float query_data = query_vec[j];
|
||||
float dim_score = centor_data * query_data;
|
||||
chunk_dists[idx] += -dim_score;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void compute_distance_table(const T *query_vec, float *dist_vec) {
|
||||
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
|
||||
|
||||
const T *transposed_tables_ptr =
|
||||
reinterpret_cast<const T *>(transposed_tables_.data());
|
||||
// chunk wise distance computation
|
||||
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
|
||||
// sum (q-c)^2 for the dimensions associated with this chunk
|
||||
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
|
||||
|
||||
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
|
||||
j++) {
|
||||
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
|
||||
|
||||
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
|
||||
float diff = centers_dim_vec[idx] - query_vec[j];
|
||||
chunk_dists[idx] += (diff * diff);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void preprocess_query(T *query) {
|
||||
const T *centroid_ptr = reinterpret_cast<const T *>(centroid_.data());
|
||||
for (size_t i = 0; i < meta_.dimension(); i++) {
|
||||
query[i] -= centroid_ptr[i];
|
||||
}
|
||||
}
|
||||
|
||||
void aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
|
||||
const uint8_t *all_coords, size_t dim, uint8_t *out);
|
||||
|
||||
void pq_dist_lookup(const uint8_t *pq_ids, size_t id_num, size_t pq_nchunks,
|
||||
const float *pq_dist_buffer, float *dists);
|
||||
|
||||
void compute_dists(uint32_t id_num, const diskann_id_t *ids,
|
||||
uint32_t chunk_num, float *pq_dist_buffer,
|
||||
void *coord_buffer, float *dists) {
|
||||
uint8_t *coord_buffer_ptr = reinterpret_cast<uint8_t *>(coord_buffer);
|
||||
|
||||
aggregate_coords(id_num, ids, this->pq_data(), chunk_num, coord_buffer_ptr);
|
||||
|
||||
pq_dist_lookup(coord_buffer_ptr, id_num, chunk_num, pq_dist_buffer, dists);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
int preprocess_pq_dist_table(void *query_rotated, float *dist_buffer) {
|
||||
switch (meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
preprocess_query(reinterpret_cast<float *>(query_rotated));
|
||||
compute_distance_table(reinterpret_cast<float *>(query_rotated),
|
||||
dist_buffer);
|
||||
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
preprocess_query(reinterpret_cast<ailego::Float16 *>(query_rotated));
|
||||
compute_distance_table(
|
||||
reinterpret_cast<ailego::Float16 *>(query_rotated), dist_buffer);
|
||||
break;
|
||||
default:
|
||||
LOG_ERROR("Unsupported Type: %u", meta_.data_type());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
public:
|
||||
const uint8_t *pq_data() const {
|
||||
return pq_data_.data();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> full_pivot_data_;
|
||||
std::vector<uint8_t> transposed_tables_;
|
||||
|
||||
std::vector<uint8_t> centroid_;
|
||||
std::vector<uint32_t> chunk_offsets_;
|
||||
std::vector<uint8_t> pq_data_;
|
||||
|
||||
IndexMeta meta_;
|
||||
uint64_t chunk_num_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,417 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_pq_trainer.h"
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnPqTrainer::DiskAnnPqTrainer(uint32_t max_train_sample_count)
|
||||
: max_train_sample_count_{max_train_sample_count} {}
|
||||
|
||||
DiskAnnPqTrainer::~DiskAnnPqTrainer() {}
|
||||
|
||||
int DiskAnnPqTrainer::gen_random_sample(IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta,
|
||||
std::string &sample_data,
|
||||
size_t &sample_size) {
|
||||
double train_sample_ratio =
|
||||
max_train_sample_count_ < 1 ? max_train_sample_count_ : 1;
|
||||
|
||||
uint32_t max_train_sample_count = train_sample_ratio * holder->count();
|
||||
max_train_sample_count = max_train_sample_count > max_train_sample_count_
|
||||
? max_train_sample_count_
|
||||
: max_train_sample_count;
|
||||
|
||||
std::vector<std::vector<uint8_t>> sample_vecs;
|
||||
|
||||
// Use a fixed seed for deterministic sampling across runs.
|
||||
uint32_t x = 456321;
|
||||
std::mt19937 gen(x);
|
||||
std::uniform_real_distribution<float> dist(0, 1);
|
||||
|
||||
uint32_t vec_size = meta.element_size();
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t sample_count = 0;
|
||||
while (iter->is_valid() && sample_count < max_train_sample_count) {
|
||||
float random = dist(gen);
|
||||
|
||||
if (random < train_sample_ratio) {
|
||||
const void *vec = iter->data();
|
||||
|
||||
std::vector<uint8_t> temp_vec;
|
||||
temp_vec.resize(vec_size);
|
||||
|
||||
std::memcpy(reinterpret_cast<uint8_t *>(&temp_vec[0]), vec, vec_size);
|
||||
|
||||
sample_vecs.push_back(std::move(temp_vec));
|
||||
|
||||
sample_count++;
|
||||
}
|
||||
|
||||
iter->next();
|
||||
}
|
||||
|
||||
sample_size = sample_vecs.size();
|
||||
sample_data.reserve(sample_size * vec_size);
|
||||
|
||||
for (size_t i = 0; i < sample_size; i++) {
|
||||
sample_data.append(reinterpret_cast<const char *>(sample_vecs[i].data()),
|
||||
vec_size);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int DiskAnnPqTrainer::prepare_pq_train_data(
|
||||
const IndexMeta &meta, size_t num_train, std::string &train_data,
|
||||
bool use_zero_mean, std::vector<uint8_t> ¢roid,
|
||||
std::shared_ptr<CompactIndexFeatures> &train_features) {
|
||||
uint32_t dim = meta.dimension();
|
||||
uint32_t vec_size = meta.element_size();
|
||||
|
||||
std::string train_data_processed;
|
||||
train_data_processed.resize(num_train * vec_size);
|
||||
|
||||
std::memcpy(&(train_data_processed[0]), train_data.data(),
|
||||
num_train * vec_size);
|
||||
|
||||
// use fp32 to accumulate to avoid overflow
|
||||
std::vector<float> centroid_temp(dim);
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
centroid_temp[d] = 0;
|
||||
}
|
||||
|
||||
T *train_data_processed_ptr = reinterpret_cast<T *>(&train_data_processed[0]);
|
||||
|
||||
if (use_zero_mean) {
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
for (uint64_t p = 0; p < num_train; p++) {
|
||||
centroid_temp[d] += train_data_processed_ptr[p * dim + d];
|
||||
}
|
||||
centroid_temp[d] /= num_train;
|
||||
}
|
||||
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
for (uint64_t p = 0; p < num_train; p++) {
|
||||
train_data_processed_ptr[p * dim + d] -= centroid_temp[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num_train; ++i) {
|
||||
train_features->emplace(train_data_processed_ptr + i * dim);
|
||||
}
|
||||
|
||||
// copy the centroid out
|
||||
centroid.resize(vec_size);
|
||||
T *centroid_ptr = reinterpret_cast<T *>(centroid.data());
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
centroid_ptr[d] = centroid_temp[d];
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int DiskAnnPqTrainer::convert_pivot_data(
|
||||
const IndexMeta &meta, uint32_t num_centers, uint32_t pq_chunk_num,
|
||||
const std::vector<uint32_t> &chunk_dims,
|
||||
const std::vector<uint32_t> &chunk_offsets,
|
||||
IndexCluster::CentroidList ¢roids,
|
||||
std::vector<uint8_t> &full_pivot_data) {
|
||||
uint32_t dim = meta.dimension();
|
||||
uint32_t element_size = meta.element_size();
|
||||
|
||||
full_pivot_data.resize(num_centers * element_size);
|
||||
|
||||
for (size_t chunk = 0; chunk < pq_chunk_num; ++chunk) {
|
||||
for (size_t cluster = 0; cluster < num_centers; ++cluster) {
|
||||
size_t idx = chunk * num_centers + cluster;
|
||||
|
||||
T *pivot_data_ptr = reinterpret_cast<T *>(&(full_pivot_data[0])) +
|
||||
cluster * dim + chunk_offsets[chunk];
|
||||
const T *feature_ptr =
|
||||
reinterpret_cast<const T *>(centroids[idx].feature());
|
||||
for (size_t d = 0; d <= chunk_dims[chunk]; ++d) {
|
||||
pivot_data_ptr[d] = feature_ptr[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::train_pq(IndexThreads::Pointer threads,
|
||||
const IndexMeta &meta, std::string &train_data,
|
||||
size_t num_train, uint32_t num_centers,
|
||||
uint32_t pq_chunk_num, uint32_t max_iterations,
|
||||
bool use_zero_mean,
|
||||
std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets) {
|
||||
uint32_t dim = meta.dimension();
|
||||
if (pq_chunk_num > dim) {
|
||||
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
|
||||
pq_chunk_num, dim);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> train_features(
|
||||
new CompactIndexFeatures(meta));
|
||||
|
||||
uint32_t type = meta.data_type();
|
||||
|
||||
int ret;
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
ret = prepare_pq_train_data<float>(
|
||||
meta, num_train, train_data, use_zero_mean, centroid, train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to prepare pq train data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
ret = prepare_pq_train_data<ailego::Float16>(
|
||||
meta, num_train, train_data, use_zero_mean, centroid, train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to prepare pq train data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Do Train
|
||||
ailego::Params params;
|
||||
params.set(MULTI_CHUNK_CLUSTER_COUNT, num_centers);
|
||||
params.set(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, pq_chunk_num);
|
||||
params.set(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, max_iterations);
|
||||
|
||||
ret = chunk_cluster_.init(meta, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to get chunk cluster");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
ret = chunk_cluster_.mount(train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Cannot mount train features");
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
std::vector<uint32_t> labels;
|
||||
|
||||
ret = chunk_cluster_.cluster(threads, cluster_centroids_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to cluster");
|
||||
return ret;
|
||||
}
|
||||
|
||||
chunk_offsets = chunk_cluster_.chunk_dim_offsets();
|
||||
auto chunk_dims = chunk_cluster_.chunk_dims();
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
ret = convert_pivot_data<float>(meta, num_centers, pq_chunk_num,
|
||||
chunk_dims, chunk_offsets,
|
||||
cluster_centroids_, full_pivot_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to convert pivot data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
ret = convert_pivot_data<ailego::Float16>(
|
||||
meta, num_centers, pq_chunk_num, chunk_dims, chunk_offsets,
|
||||
cluster_centroids_, full_pivot_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to convert pivot data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::train_quantized_data(
|
||||
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta, std::vector<uint8_t> &pq_full_pivot_data,
|
||||
std::vector<uint8_t> &pq_centroid, std::vector<uint32_t> &pq_chunk_offsets,
|
||||
size_t pq_chunk_num) {
|
||||
size_t train_size;
|
||||
std::string train_data;
|
||||
|
||||
int ret = gen_random_sample(holder, meta, train_data, train_size);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Get Random Sample Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Training data with %zu samples loaded.", train_size);
|
||||
|
||||
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
|
||||
// false);
|
||||
bool use_zero_mean = false;
|
||||
|
||||
ret = train_pq(threads, meta, train_data, train_size, PQTable::kPQCentroidNum,
|
||||
pq_chunk_num, PQTable::kMeanIterNum, use_zero_mean,
|
||||
pq_full_pivot_data, pq_centroid, pq_chunk_offsets);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Train PQ Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::generate_pq(IndexThreads::Pointer threads,
|
||||
const IndexMeta &meta,
|
||||
IndexHolder::Pointer holder,
|
||||
uint32_t pq_chunk_num,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint8_t> &block_compressed_data) {
|
||||
uint32_t type = meta.data_type();
|
||||
uint32_t dim = meta.dimension();
|
||||
|
||||
if (pq_chunk_num > dim) {
|
||||
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
|
||||
pq_chunk_num, dim);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// Do Label
|
||||
std::vector<uint32_t> labels;
|
||||
size_t num_vecs = holder->count();
|
||||
size_t batch_size =
|
||||
num_vecs <= compress_batch_size_ ? num_vecs : compress_batch_size_;
|
||||
|
||||
std::vector<uint32_t> block_compressed_base(batch_size * pq_chunk_num);
|
||||
|
||||
std::memset(&block_compressed_base[0], 0,
|
||||
batch_size * pq_chunk_num * sizeof(uint32_t));
|
||||
|
||||
std::vector<uint8_t> block_data(batch_size * meta.element_size());
|
||||
std::vector<uint8_t> block_data_converted(batch_size * meta.element_size());
|
||||
|
||||
size_t block_num = DiskAnnUtil::div_round_up(num_vecs, batch_size);
|
||||
|
||||
block_compressed_data.resize(num_vecs * pq_chunk_num);
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
for (size_t block = 0; block < block_num; block++) {
|
||||
size_t start_id = block * batch_size;
|
||||
size_t end_id = std::min((block + 1) * batch_size, num_vecs);
|
||||
|
||||
size_t cur_block_size = end_id - start_id;
|
||||
|
||||
for (size_t i = 0; i < cur_block_size && iter->is_valid(); i++) {
|
||||
const void *vec = iter->data();
|
||||
std::memcpy(
|
||||
reinterpret_cast<uint8_t *>(&block_data[0]) + i * meta.element_size(),
|
||||
vec, meta.element_size());
|
||||
iter->next();
|
||||
}
|
||||
|
||||
std::memcpy(block_data_converted.data(), block_data.data(),
|
||||
cur_block_size * meta.element_size());
|
||||
|
||||
LOG_INFO("Processing Docs, Range: [%zu, %zu)..", start_id, end_id);
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> block_features(
|
||||
new CompactIndexFeatures(meta));
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
DiskAnnUtil::convert_vector_to_residual<float>(
|
||||
reinterpret_cast<float *>(block_data_converted.data()),
|
||||
cur_block_size, dim, centroid.data());
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
DiskAnnUtil::convert_vector_to_residual<ailego::Float16>(
|
||||
reinterpret_cast<ailego::Float16 *>(block_data_converted.data()),
|
||||
cur_block_size, dim, centroid.data());
|
||||
break;
|
||||
default:
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < cur_block_size; i++) {
|
||||
block_features->emplace(block_data_converted.data() +
|
||||
i * meta.element_size());
|
||||
}
|
||||
|
||||
int ret = chunk_cluster_.mount(block_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Cannot mount block features");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = chunk_cluster_.label(threads, cluster_centroids_, &labels);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to label");
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> compressed_data(cur_block_size * pq_chunk_num);
|
||||
|
||||
DiskAnnUtil::convert_types_uint32_to_uint8(
|
||||
labels.data(), compressed_data.data(), cur_block_size, pq_chunk_num);
|
||||
|
||||
memcpy(&(block_compressed_data[0]) + start_id * pq_chunk_num,
|
||||
compressed_data.data(), cur_block_size * pq_chunk_num);
|
||||
|
||||
LOG_INFO("Generate PQ Data Done.");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::generate_quantized_data(
|
||||
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta, std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint8_t> &block_compressed_data, size_t pq_chunk_num) {
|
||||
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
|
||||
// false);
|
||||
|
||||
int ret = generate_pq(threads, meta, holder, pq_chunk_num, pq_centroid,
|
||||
block_compressed_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Generate PQ Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "../cluster/multi_chunk_cluster.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnPqTrainer {
|
||||
public:
|
||||
typedef std::unique_ptr<DiskAnnPqTrainer> UPointer;
|
||||
|
||||
public:
|
||||
DiskAnnPqTrainer(uint32_t max_train_sample_count);
|
||||
virtual ~DiskAnnPqTrainer();
|
||||
|
||||
public:
|
||||
template <typename T>
|
||||
static int prepare_pq_train_data(
|
||||
const IndexMeta &meta, size_t num_train, std::string &train_data,
|
||||
bool use_zero_mean, std::vector<uint8_t> ¢roid,
|
||||
std::shared_ptr<CompactIndexFeatures> &train_features);
|
||||
|
||||
template <typename T>
|
||||
static int convert_pivot_data(const IndexMeta &meta, uint32_t num_centers,
|
||||
uint32_t pq_chunk_num,
|
||||
const std::vector<uint32_t> &chunk_dims,
|
||||
const std::vector<uint32_t> &chunk_offsets,
|
||||
IndexCluster::CentroidList ¢roids,
|
||||
std::vector<uint8_t> &full_pivot_data);
|
||||
|
||||
int gen_random_sample(IndexHolder::Pointer holder, const IndexMeta &meta,
|
||||
std::string &sample_data, size_t &sample_size);
|
||||
|
||||
int generate_quantized_data(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta,
|
||||
std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint8_t> &block_compressed_data,
|
||||
size_t num_pq_chunks);
|
||||
|
||||
int generate_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
|
||||
IndexHolder::Pointer holder, uint32_t num_pq_chunks,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint8_t> &block_compressed_data);
|
||||
|
||||
int train_quantized_data(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder, const IndexMeta &meta,
|
||||
std::vector<uint8_t> &pq_full_pivot_data,
|
||||
std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint32_t> &pq_chunk_offsets,
|
||||
size_t num_pq_chunks);
|
||||
|
||||
int train_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
|
||||
std::string &train_data, size_t num_train, uint32_t num_centers,
|
||||
uint32_t num_pq_chunks, uint32_t max_iterations,
|
||||
bool use_zero_mean, std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets);
|
||||
|
||||
private:
|
||||
static constexpr uint32_t compress_batch_size_{
|
||||
DiskAnnEntity::kDefaultCompressBatchSize};
|
||||
|
||||
// pq cluster
|
||||
MultiChunkCluster chunk_cluster_;
|
||||
IndexCluster::CentroidList cluster_centroids_;
|
||||
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_reducer.h"
|
||||
#include <vector>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include <zvec/core/framework/index_reducer.h>
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnReducer::init(const ailego::Params ¶ms) {
|
||||
params.get(PARAM_DISKANN_REDUCER_WORKING_PATH, &working_path_);
|
||||
if (working_path_.empty()) {
|
||||
LOG_ERROR("Missing parameter. %s",
|
||||
PARAM_DISKANN_REDUCER_WORKING_PATH.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::string index_name =
|
||||
params.get_as_string(PARAM_DISKANN_REDUCER_INDEX_NAME);
|
||||
if (index_name.empty()) {
|
||||
index_name = std::to_string(std::clock());
|
||||
}
|
||||
|
||||
reducer_file_path_ = ailego::StringHelper::Concat(
|
||||
working_path_, "/", kReducerFileName, index_name);
|
||||
|
||||
holder_file_path_ = ailego::StringHelper::Concat(working_path_, "/",
|
||||
kHolderFileName, index_name);
|
||||
|
||||
state_ = STATE_INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducer::cleanup(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Reduce operator with filter
|
||||
int DiskAnnReducer::reduce(const IndexFilter &filter) {
|
||||
if (entities_.empty() || state_ != STATE_FEED) {
|
||||
LOG_ERROR("No container to reduce, feed first");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (use_mem_holder_) {
|
||||
mem_holder_ = std::make_shared<RandomAccessIndexHolder>(meta_);
|
||||
for (auto entity : entities_) {
|
||||
size_t doc_cnt = entity->doc_cnt();
|
||||
for (size_t id = 0; id < doc_cnt; ++id) {
|
||||
diskann_key_t pkey = entity->get_key(id);
|
||||
|
||||
if (filter.is_valid() && filter(pkey)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const void *vec = entity->get_vector(id);
|
||||
mem_holder_->emplace(pkey, vec);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
disk_holder_ =
|
||||
std::make_shared<DiskAnnIndexHolder>(meta_, holder_file_path_);
|
||||
|
||||
int ret = disk_holder_->init();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("DiskAnn Index Holder init failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
for (auto entity : entities_) {
|
||||
size_t doc_cnt = entity->doc_cnt();
|
||||
for (size_t id = 0; id < doc_cnt; ++id) {
|
||||
diskann_key_t pkey = entity->get_key(id);
|
||||
|
||||
if (filter.is_valid() && filter(pkey)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const void *vec = entity->get_vector(id);
|
||||
disk_holder_->emplace(pkey, vec);
|
||||
}
|
||||
}
|
||||
|
||||
disk_holder_->close();
|
||||
}
|
||||
|
||||
builder_ = IndexFactory::CreateBuilder(kDiskAnnBuilderName);
|
||||
if (!builder_) {
|
||||
LOG_ERROR("Create builder failed. name[%s]", kDiskAnnBuilderName.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (thread_pool_ == nullptr) {
|
||||
LOG_ERROR(
|
||||
"Only support multi-thread mode. Thread pool is not set for reducer.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Start diskann reduce");
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
|
||||
auto params = meta_.builder_params();
|
||||
|
||||
int ret = builder_->init(meta_, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Init proxima streamer failed. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (use_mem_holder_) {
|
||||
ret = builder_->train(mem_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = builder_->build(mem_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
ret = builder_->train(disk_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = builder_->build(disk_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
auto &stats = builder_->stats();
|
||||
|
||||
stats_.set_reduced_costtime(timer.seconds());
|
||||
stats_.set_filtered_count(stats.discarded_count());
|
||||
|
||||
state_ = STATE_REDUCE;
|
||||
|
||||
LOG_INFO("End DiskAnn reduce. cost time: [%zu]s", (size_t)timer.seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Dump index by dumper
|
||||
int DiskAnnReducer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("Begin diskann reducer dump");
|
||||
|
||||
if (state_ != STATE_REDUCE) {
|
||||
LOG_WARN("Reduce first before dump.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = builder_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("diskann reducer dump failed. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
stats_.set_dumped_costtime(timer.seconds());
|
||||
|
||||
LOG_INFO("End diskann reducer dump, dump costtime=[%zu]s",
|
||||
(size_t)(stats_.dumped_costtime()));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_REDUCER(DiskAnnReducer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <utility/sparse_utility.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include <zvec/core/framework/index_reducer.h>
|
||||
#include "diskann_holder.h"
|
||||
#include "diskann_reducer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnReducer : public IndexReducer {
|
||||
public:
|
||||
//! Constructor
|
||||
DiskAnnReducer(void) = default;
|
||||
|
||||
protected:
|
||||
//! Initialize Reducer
|
||||
virtual int init(const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Reducer
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Feed indexes from containers
|
||||
// virtual int feed(IndexStorage::Pointer container) override;
|
||||
|
||||
//! Reduce operator (with filter)
|
||||
virtual int reduce(const IndexFilter &filter) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
private:
|
||||
enum State {
|
||||
STATE_UNINITED = 0,
|
||||
STATE_INITED = 1,
|
||||
STATE_FEED = 2,
|
||||
STATE_REDUCE = 3
|
||||
};
|
||||
|
||||
std::string working_path_{""};
|
||||
|
||||
IndexMeta meta_{};
|
||||
std::vector<DiskAnnReducerEntity::Pointer> entities_{};
|
||||
|
||||
// bool use_mem_holder_{true};
|
||||
bool use_mem_holder_{false};
|
||||
RandomAccessIndexHolder::Pointer mem_holder_;
|
||||
DiskAnnIndexHolder::Pointer disk_holder_;
|
||||
|
||||
IndexBuilder::Pointer builder_{nullptr};
|
||||
std::string reducer_file_path_{""};
|
||||
std::string holder_file_path_{""};
|
||||
|
||||
Stats stats_{};
|
||||
State state_{STATE_UNINITED};
|
||||
|
||||
const std::string kDiskAnnBuilderName{"DiskAnnBuilder"};
|
||||
const std::string kReducerFileName{"diskann.reducer.builder."};
|
||||
const std::string kHolderFileName{"diskann.reducer.holder."};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_reducer_entity.h"
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnReducerEntity::load(const IndexStorage::Pointer &container,
|
||||
bool check_crc) {
|
||||
container_ = container;
|
||||
|
||||
int ret = load_segments(check_crc);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
sector_num_per_node_ = node_per_sector() > 0
|
||||
? 1
|
||||
: DiskAnnUtil::div_round_up(
|
||||
max_node_size(), DiskAnnUtil::kSectorSize);
|
||||
|
||||
loaded_ = true;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_segments(bool /*check_crc*/) {
|
||||
int ret;
|
||||
ret = load_header_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_vector_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_header_segment() {
|
||||
const void *data = nullptr;
|
||||
meta_segment_ = container_->get(kDiskAnnMetaSegmentId);
|
||||
if (!meta_segment_ ||
|
||||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(DiskAnnMetaHeader)) !=
|
||||
sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
::memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_vector_segment() {
|
||||
vector_segment_ = container_->get(kDiskAnnVectorSegmentId);
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_key_segment() {
|
||||
// load key
|
||||
key_segment_ = container_->get(kDiskAnnKeySegmentId);
|
||||
if (!key_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_data_len = doc_cnt() * sizeof(key_t);
|
||||
|
||||
// load key mapping
|
||||
key_mapping_segment_ = container_->get(kDiskAnnKeyMappingSegmentId);
|
||||
const void *data = nullptr;
|
||||
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_data_len) != key_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_buffer_.resize(key_data_len);
|
||||
memcpy(&(key_buffer_[0]), data, key_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool DiskAnnReducerEntity::do_crc_check(
|
||||
std::vector<SegmentPointer> &segments) const {
|
||||
constexpr size_t blk_size = 4096;
|
||||
const void *data;
|
||||
|
||||
for (auto &segment : segments) {
|
||||
size_t offset = 0;
|
||||
size_t rd_size;
|
||||
uint32_t crc = 0;
|
||||
while (offset < segment->data_size()) {
|
||||
size_t size = std::min(blk_size, segment->data_size() - offset);
|
||||
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
|
||||
break;
|
||||
}
|
||||
offset += rd_size;
|
||||
crc = ailego::Crc32c::Hash(data, rd_size, crc);
|
||||
}
|
||||
if (crc != segment->data_crc()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnReducerEntity::get_id(diskann_key_t key) const {
|
||||
const diskann_id_t *key_mapping_data_ptr =
|
||||
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
//! Do binary search
|
||||
diskann_id_t start = 0UL;
|
||||
diskann_id_t end = doc_cnt();
|
||||
diskann_id_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
diskann_id_t local_id = key_mapping_data_ptr[idx];
|
||||
|
||||
const diskann_key_t local_key = key_data_ptr[local_id];
|
||||
|
||||
if (local_key < key) {
|
||||
start = idx + 1;
|
||||
} else if (local_key > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnReducerEntity::get_key(diskann_id_t id) const {
|
||||
const void *key;
|
||||
if (ailego_unlikely(key_segment_->read(id * sizeof(diskann_key_t), &key,
|
||||
sizeof(diskann_key_t)) !=
|
||||
sizeof(diskann_key_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return kInvalidKey;
|
||||
}
|
||||
|
||||
return *(reinterpret_cast<const diskann_key_t *>(key));
|
||||
}
|
||||
|
||||
const void *DiskAnnReducerEntity::get_vector(diskann_id_t id) const {
|
||||
size_t read_size = sector_num_per_node_ * DiskAnnUtil::kSectorSize;
|
||||
size_t sector_id = DiskAnnUtil::get_node_sector(
|
||||
node_per_sector(), max_node_size(), DiskAnnUtil::kSectorSize, id);
|
||||
size_t offset = sector_id * DiskAnnUtil::kSectorSize;
|
||||
|
||||
if (sector_id != sector_id_) {
|
||||
const void *sector_data;
|
||||
if (ailego_unlikely(vector_segment_->read(offset, §or_data,
|
||||
read_size) != read_size)) {
|
||||
LOG_ERROR("Read vector from segment failed");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
sector_id_ = sector_id;
|
||||
sector_buffer_.assign(reinterpret_cast<const char *>(sector_data),
|
||||
read_size);
|
||||
}
|
||||
|
||||
return DiskAnnUtil::offset_to_node_const(
|
||||
node_per_sector(), max_node_size(),
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data()), id);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnReducerEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnReducerEntity>;
|
||||
using SegmentPointer = IndexStorage::Segment::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnReducerEntity() = default;
|
||||
virtual ~DiskAnnReducerEntity() = default;
|
||||
|
||||
int load(const IndexStorage::Pointer &container, bool check_crc);
|
||||
int load_segments(bool check_crc);
|
||||
int load_header_segment();
|
||||
int load_vector_segment();
|
||||
int load_key_segment();
|
||||
int load_key_mapping_segment();
|
||||
|
||||
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
private:
|
||||
IndexStorage::Pointer container_{};
|
||||
IndexStorage::Segment::Pointer meta_segment_{};
|
||||
IndexStorage::Segment::Pointer vector_segment_{};
|
||||
IndexStorage::Segment::Pointer key_segment_{};
|
||||
IndexStorage::Segment::Pointer key_mapping_segment_{};
|
||||
|
||||
std::string key_buffer_;
|
||||
std::string key_mapping_buffer_;
|
||||
|
||||
size_t sector_num_per_node_{0};
|
||||
|
||||
mutable size_t sector_id_{-1U};
|
||||
mutable std::string sector_buffer_;
|
||||
|
||||
bool loaded_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_searcher.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnSearcher::DiskAnnSearcher() {}
|
||||
|
||||
DiskAnnSearcher::~DiskAnnSearcher() {}
|
||||
|
||||
int DiskAnnSearcher::init(const ailego::Params &search_params) {
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnSearcher::print_debug_info() {}
|
||||
|
||||
int DiskAnnSearcher::cleanup() {
|
||||
LOG_INFO("Begin DiskAnnSearcher:cleanup");
|
||||
|
||||
LOG_INFO("End DiskAnnSearcher:cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::load(IndexStorage::Pointer storage,
|
||||
IndexMetric::Pointer measure) {
|
||||
LOG_INFO("DiskAnnSearcher::load Begin");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from storage");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.load(meta_, storage);
|
||||
if (ret != 0) {
|
||||
LOG_INFO("Searcher Entity Load Failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
|
||||
|
||||
int res = diskann_indexer_->init(entity_);
|
||||
if (res != 0) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (cache_nodes_num_ != 0) {
|
||||
std::vector<diskann_id_t> node_list;
|
||||
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
|
||||
|
||||
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
|
||||
|
||||
diskann_indexer_->load_cache_list(node_list);
|
||||
|
||||
node_list.clear();
|
||||
node_list.shrink_to_fit();
|
||||
}
|
||||
|
||||
if (measure) {
|
||||
measure_ = measure;
|
||||
} else {
|
||||
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!measure_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = measure_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
if (measure_->query_metric()) {
|
||||
measure_ = measure_->query_metric();
|
||||
}
|
||||
}
|
||||
|
||||
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
state_ = STATE_LOADED;
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
LOG_INFO("DiskAnnSearcher::load Done");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::unload() {
|
||||
LOG_INFO("DiskAnnSearcher unload index");
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::update_context(DiskAnnContext *ctx) const {
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
|
||||
entity, magic_);
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
// do search
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
// Context is pooled per index type. When switching between DiskAnn indexes
|
||||
// with different element sizes (e.g., fp16 vs fp32), the cached context has
|
||||
// undersized buffers. Recreate it to ensure correct buffer allocations.
|
||||
if (ctx->magic() != magic_) {
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->knn_search(ctx);
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->linear_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->keys_search(p_keys[i], ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const {
|
||||
return diskann_indexer_->get_vector(key, context, vector);
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer DiskAnnSearcher::create_context() const {
|
||||
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
|
||||
if (!search_ctx_entity) {
|
||||
LOG_ERROR("Failed to create search context entity");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx =
|
||||
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
|
||||
if (ctx == nullptr) {
|
||||
LOG_ERROR("Failed to allocate DiskAnn Context");
|
||||
return Context::Pointer();
|
||||
}
|
||||
if (ailego_unlikely(ctx->init(
|
||||
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
|
||||
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
|
||||
LOG_ERROR("Init DiskAnn Context failed");
|
||||
delete ctx;
|
||||
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->set_list_size(list_size_);
|
||||
ctx->set_magic(magic_);
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_SEARCHER(DiskAnnSearcher);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,166 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
|
||||
class LinuxAlignedFileReader;
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnSearcher : public IndexSearcher {
|
||||
public:
|
||||
using ContextPointer = IndexSearcher::Context::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnSearcher(void);
|
||||
~DiskAnnSearcher(void);
|
||||
|
||||
DiskAnnSearcher(const DiskAnnSearcher &) = delete;
|
||||
DiskAnnSearcher &operator=(const DiskAnnSearcher &) = delete;
|
||||
|
||||
protected:
|
||||
//! Initialize Searcher
|
||||
int init(const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load Index from storage
|
||||
int load(IndexStorage::Pointer storage, IndexMetric::Pointer metric) override;
|
||||
|
||||
//! Unload index from storage
|
||||
int unload(void) override;
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices,
|
||||
const void *sparse_query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
|
||||
sparse_query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(
|
||||
const void * /*query*/, const uint32_t * /*sparse_count*/,
|
||||
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
|
||||
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
|
||||
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
|
||||
ContextPointer & /*context*/) const override {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get vector by key
|
||||
int get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
ContextPointer create_context() const override;
|
||||
|
||||
//! Create a new iterator
|
||||
IndexSearcher::Provider::Pointer create_provider(void) const override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
template <typename T, typename LabelT = uint32_t>
|
||||
int search_disk_index(const std::string &query_file,
|
||||
const uint32_t num_nodes_to_cache,
|
||||
const uint32_t recall_at, const uint32_t beamwidth);
|
||||
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(DiskAnnContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
IndexMetric::Pointer measure_{};
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
|
||||
DiskAnnSearcherEntity entity_{};
|
||||
|
||||
uint32_t magic_{0U};
|
||||
|
||||
Stats stats_;
|
||||
State state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,441 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_searcher_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const DiskAnnEntity::Pointer DiskAnnSearcherEntity::clone() const {
|
||||
auto meta_segment = meta_segment_->clone();
|
||||
if (ailego_unlikely(!meta_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto pq_meta_segment = pq_meta_segment_->clone();
|
||||
if (ailego_unlikely(!pq_meta_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnPqMetaSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto pq_data_segment = pq_data_segment_->clone();
|
||||
if (ailego_unlikely(!pq_data_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnPqDataSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto vector_segment = vector_segment_->clone();
|
||||
if (ailego_unlikely(!vector_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnVectorSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto key_segment = key_segment_->clone();
|
||||
if (ailego_unlikely(!key_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnKeySegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto key_mapping_segment = key_mapping_segment_->clone();
|
||||
if (ailego_unlikely(!key_mapping_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto entrypoint_segment = entrypoint_segment_->clone();
|
||||
if (ailego_unlikely(!entrypoint_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnSearcherEntity *entity = new (std::nothrow) DiskAnnSearcherEntity(
|
||||
meta_header_, pq_meta_, meta_segment, pq_meta_segment, pq_data_segment,
|
||||
vector_segment, key_segment, key_mapping_segment, entrypoint_segment,
|
||||
num_threads_, list_size_, cache_nodes_num_, warm_up_, beam_size_, meta_,
|
||||
pq_table_, key_buffer_, key_mapping_buffer_, entrypoints_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("DiskAnnSearcherEntity new failed");
|
||||
}
|
||||
|
||||
return DiskAnnEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load(const IndexMeta &meta,
|
||||
IndexStorage::Pointer storage) {
|
||||
meta_ = meta;
|
||||
|
||||
storage_ = storage;
|
||||
|
||||
int ret;
|
||||
ret = load_header_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_pq_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load PQ Meta Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_mapping_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_entrypoint_segment();
|
||||
if (ret != 0) {
|
||||
LOG_WARN("Load EntryPoint Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_vector_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_pq_segment() {
|
||||
const void *data = nullptr;
|
||||
|
||||
// load pq meta
|
||||
pq_meta_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqMetaSegmentId);
|
||||
if (!pq_meta_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t read_size;
|
||||
size_t offset = 0;
|
||||
|
||||
// 1. read pq meta
|
||||
read_size = pq_meta_segment_->read(offset, &data, sizeof(DiskAnnPqMeta));
|
||||
if (read_size != sizeof(DiskAnnPqMeta)) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
sizeof(DiskAnnPqMeta), read_size);
|
||||
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
memcpy(reinterpret_cast<uint8_t *>(&pq_meta_), data, sizeof(DiskAnnPqMeta));
|
||||
offset += read_size;
|
||||
|
||||
// 2. read full pivot data
|
||||
std::vector<uint8_t> full_pivot_data;
|
||||
full_pivot_data.resize(pq_meta_.full_pivot_data_size);
|
||||
|
||||
read_size =
|
||||
pq_meta_segment_->read(offset, &data, pq_meta_.full_pivot_data_size);
|
||||
if (read_size != pq_meta_.full_pivot_data_size) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(pq_meta_.full_pivot_data_size), (size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(full_pivot_data[0]), data, read_size);
|
||||
offset += read_size;
|
||||
|
||||
// 3. read centroid
|
||||
std::vector<uint8_t> centroid;
|
||||
centroid.resize(pq_meta_.centroid_data_size);
|
||||
|
||||
read_size =
|
||||
pq_meta_segment_->read(offset, &data, pq_meta_.centroid_data_size);
|
||||
if (read_size != pq_meta_.centroid_data_size) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(pq_meta_.centroid_data_size), (size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(centroid[0]), data, read_size);
|
||||
offset += read_size;
|
||||
|
||||
// 4. chunk offset
|
||||
std::vector<uint32_t> chunk_offsets;
|
||||
chunk_offsets.resize(pq_meta_.chunk_num + 1);
|
||||
|
||||
read_size = pq_meta_segment_->read(
|
||||
offset, &data, (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
|
||||
if (read_size != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
|
||||
(size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(chunk_offsets[0]), data, read_size);
|
||||
|
||||
// load pq data
|
||||
std::vector<uint8_t> pq_data;
|
||||
pq_data_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqDataSegmentId);
|
||||
if (!pq_data_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnPqDataSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
pq_data.resize(meta_header_.doc_cnt * pq_meta_.chunk_num);
|
||||
|
||||
void *pq_data_ptr = &pq_data[0];
|
||||
read_size = pq_data_segment_->fetch(
|
||||
0, pq_data_ptr, meta_header_.doc_cnt * pq_meta_.chunk_num);
|
||||
|
||||
if (read_size != meta_header_.doc_cnt * pq_meta_.chunk_num) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(meta_header_.doc_cnt * pq_meta_.chunk_num),
|
||||
(size_t)read_size);
|
||||
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
pq_table_ = std::make_shared<PQTable>(meta_, pq_meta_.chunk_num);
|
||||
|
||||
pq_table_->init(full_pivot_data, centroid, chunk_offsets, pq_data);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_header_segment() {
|
||||
const void *data = nullptr;
|
||||
meta_segment_ = storage_->get(kDiskAnnMetaSegmentId);
|
||||
if (!meta_segment_ ||
|
||||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(DiskAnnMetaHeader)) !=
|
||||
sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_vector_segment() {
|
||||
vector_segment_ = storage_->get(kDiskAnnVectorSegmentId);
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_key_segment() {
|
||||
// load key
|
||||
key_segment_ = storage_->get(kDiskAnnKeySegmentId);
|
||||
if (!key_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_data_len = doc_cnt() * sizeof(diskann_key_t);
|
||||
|
||||
const void *data = nullptr;
|
||||
if (key_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_data_len) != key_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_buffer_.resize(key_data_len);
|
||||
memcpy(&(key_buffer_[0]), data, key_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_entrypoint_segment() {
|
||||
entrypoint_segment_ = storage_->get(kDiskAnnEntryPointSegmentId);
|
||||
if (!entrypoint_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
const void *data = nullptr;
|
||||
|
||||
if (entrypoint_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(uint32_t)) != sizeof(uint32_t)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
uint32_t entrypoint_cnt = 0;
|
||||
memcpy(&entrypoint_cnt, data, sizeof(uint32_t));
|
||||
|
||||
if (entrypoint_cnt != 0) {
|
||||
size_t entrypoint_data_len = entrypoint_cnt * sizeof(diskann_id_t);
|
||||
|
||||
if (entrypoint_segment_->read(sizeof(uint32_t),
|
||||
reinterpret_cast<const void **>(&data),
|
||||
entrypoint_data_len) != entrypoint_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
entrypoints_.resize(entrypoint_cnt);
|
||||
memcpy(&(entrypoints_[0]), data, entrypoint_data_len);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int DiskAnnSearcherEntity::load_key_mapping_segment() {
|
||||
key_mapping_segment_ = storage_->get(kDiskAnnKeyMappingSegmentId);
|
||||
if (!key_mapping_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_mapping_data_len = doc_cnt() * sizeof(diskann_id_t);
|
||||
|
||||
const void *data = nullptr;
|
||||
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_mapping_data_len) !=
|
||||
key_mapping_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_mapping_buffer_.resize(key_mapping_data_len);
|
||||
memcpy(&(key_mapping_buffer_[0]), data, key_mapping_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnSearcherEntity::get_id(diskann_key_t key) const {
|
||||
const diskann_id_t *key_mapping_data_ptr =
|
||||
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
|
||||
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
//! Do binary search
|
||||
diskann_id_t start = 0UL;
|
||||
diskann_id_t end = doc_cnt();
|
||||
diskann_id_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
diskann_id_t local_id = key_mapping_data_ptr[idx];
|
||||
|
||||
const diskann_key_t local_key = key_data_ptr[local_id];
|
||||
|
||||
if (local_key < key) {
|
||||
start = idx + 1;
|
||||
} else if (local_key > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnSearcherEntity::get_key(diskann_id_t id) const {
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
return key_data_ptr[id];
|
||||
}
|
||||
|
||||
const void *DiskAnnSearcherEntity::get_vector(diskann_id_t id) const {
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Vector segment is null");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint64_t sector_offset =
|
||||
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
|
||||
DiskAnnUtil::kSectorSize, id) *
|
||||
DiskAnnUtil::kSectorSize;
|
||||
uint64_t within_sector_offset =
|
||||
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
|
||||
uint64_t total_offset = sector_offset + within_sector_offset;
|
||||
|
||||
size_t read_size = meta_.element_size();
|
||||
const void *vec;
|
||||
if (ailego_unlikely(vector_segment_->read(total_offset, &vec, read_size) !=
|
||||
read_size)) {
|
||||
LOG_ERROR("Read vector from segment failed, id: %u, offset: %lu", id,
|
||||
total_offset);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return vec;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> DiskAnnSearcherEntity::get_neighbors(
|
||||
diskann_id_t id) const {
|
||||
if (!vector_segment_) {
|
||||
return std::make_pair(0, nullptr);
|
||||
}
|
||||
|
||||
uint64_t read_sector_offset =
|
||||
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
|
||||
DiskAnnUtil::kSectorSize, id) *
|
||||
DiskAnnUtil::kSectorSize;
|
||||
uint64_t node_vec_offset =
|
||||
read_sector_offset +
|
||||
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
|
||||
|
||||
const void *data;
|
||||
if (ailego_unlikely(
|
||||
vector_segment_->read(node_vec_offset, &data, max_node_size()) !=
|
||||
max_node_size())) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
|
||||
const uint8_t *data_ptr = reinterpret_cast<const uint8_t *>(data);
|
||||
const diskann_id_t *node_neighbor =
|
||||
reinterpret_cast<const diskann_id_t *>(data_ptr + meta_.element_size());
|
||||
|
||||
auto neighbor_num = *node_neighbor;
|
||||
|
||||
return std::make_pair(neighbor_num, node_neighbor + 1);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnSearcherEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnSearcherEntity>;
|
||||
using SegmentPointer = IndexStorage::Segment::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnSearcherEntity() = default;
|
||||
virtual ~DiskAnnSearcherEntity() = default;
|
||||
|
||||
public:
|
||||
virtual const DiskAnnEntity::Pointer clone() const override;
|
||||
|
||||
int load(const IndexMeta &meta, IndexStorage::Pointer storage);
|
||||
int load_pq_segment();
|
||||
int load_header_segment();
|
||||
int load_vector_segment();
|
||||
int load_key_segment();
|
||||
int load_key_mapping_segment();
|
||||
int load_entrypoint_segment();
|
||||
|
||||
PQTable::Pointer get_pq_table() {
|
||||
return pq_table_;
|
||||
}
|
||||
|
||||
IndexStorage::Pointer get_storage() {
|
||||
return storage_;
|
||||
}
|
||||
|
||||
SegmentPointer get_vector_segment() {
|
||||
return vector_segment_;
|
||||
}
|
||||
|
||||
std::vector<diskann_id_t> &entrypoints() {
|
||||
return entrypoints_;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t id) const override;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
private:
|
||||
DiskAnnSearcherEntity(
|
||||
const DiskAnnMetaHeader &meta_header, const DiskAnnPqMeta &pq_meta,
|
||||
const SegmentPointer &meta_segment, const SegmentPointer &pq_meta_segment,
|
||||
const SegmentPointer &pq_data_segment,
|
||||
const SegmentPointer &vector_segment, const SegmentPointer &key_segment,
|
||||
const SegmentPointer &key_mapping_segment,
|
||||
const SegmentPointer &entrypoint_segment, uint32_t num_threads,
|
||||
uint32_t list_size, uint32_t cache_nodes_num, bool warm_up,
|
||||
uint32_t beam_size, const IndexMeta meta, PQTable::Pointer pq_table,
|
||||
const std::string &key_buffer, const std::string &key_mapping_buffer,
|
||||
const std::vector<diskann_id_t> &entrypoints)
|
||||
: DiskAnnEntity(meta_header, pq_meta),
|
||||
meta_segment_(meta_segment),
|
||||
pq_meta_segment_(pq_meta_segment),
|
||||
pq_data_segment_(pq_data_segment),
|
||||
vector_segment_(vector_segment),
|
||||
key_segment_(key_segment),
|
||||
key_mapping_segment_(key_mapping_segment),
|
||||
entrypoint_segment_{entrypoint_segment},
|
||||
num_threads_{num_threads},
|
||||
list_size_{list_size},
|
||||
cache_nodes_num_{cache_nodes_num},
|
||||
warm_up_{warm_up},
|
||||
beam_size_{beam_size},
|
||||
meta_{meta},
|
||||
pq_table_{pq_table},
|
||||
key_buffer_{key_buffer},
|
||||
key_mapping_buffer_{key_mapping_buffer},
|
||||
entrypoints_{entrypoints} {}
|
||||
|
||||
IndexStorage::Pointer storage_{};
|
||||
|
||||
SegmentPointer meta_segment_{nullptr};
|
||||
SegmentPointer pq_meta_segment_{nullptr};
|
||||
SegmentPointer pq_data_segment_{nullptr};
|
||||
SegmentPointer vector_segment_{nullptr};
|
||||
SegmentPointer key_segment_{nullptr};
|
||||
SegmentPointer key_mapping_segment_{nullptr};
|
||||
SegmentPointer entrypoint_segment_{nullptr};
|
||||
|
||||
uint32_t num_threads_{1};
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
IndexMeta meta_;
|
||||
|
||||
PQTable::Pointer pq_table_;
|
||||
std::string key_buffer_;
|
||||
std::string key_mapping_buffer_;
|
||||
std::vector<diskann_id_t> entrypoints_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,345 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_streamer.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_index_provider.h"
|
||||
#include "diskann_indexer.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnStreamer::DiskAnnStreamer() {}
|
||||
|
||||
DiskAnnStreamer::~DiskAnnStreamer() {}
|
||||
|
||||
int DiskAnnStreamer::init(const IndexMeta &meta,
|
||||
const ailego::Params &search_params) {
|
||||
meta_ = meta;
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnStreamer::print_debug_info() {}
|
||||
|
||||
int DiskAnnStreamer::cleanup() {
|
||||
LOG_INFO("Begin DiskAnnStreamer:cleanup");
|
||||
|
||||
LOG_INFO("End DiskAnnStreamer:cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::open(IndexStorage::Pointer storage) {
|
||||
LOG_INFO("DiskAnnStreamer::load Begin");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from storage");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.load(meta_, storage);
|
||||
if (ret != 0) {
|
||||
LOG_INFO("Searcher Entity Load Failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
|
||||
|
||||
int res = diskann_indexer_->init(entity_);
|
||||
if (res != 0) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (cache_nodes_num_ != 0) {
|
||||
std::vector<diskann_id_t> node_list;
|
||||
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
|
||||
|
||||
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
|
||||
|
||||
diskann_indexer_->load_cache_list(node_list);
|
||||
|
||||
node_list.clear();
|
||||
node_list.shrink_to_fit();
|
||||
}
|
||||
|
||||
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!measure_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = measure_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
if (measure_->query_metric()) {
|
||||
measure_ = measure_->query_metric();
|
||||
}
|
||||
|
||||
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
state_ = STATE_LOADED;
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
LOG_INFO("DiskAnnStreamer::load Done");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::unload() {
|
||||
LOG_INFO("DiskAnnStreamer unload index");
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::update_context(DiskAnnContext *ctx) const {
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
|
||||
entity, magic_);
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
// do search
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
// Context is pooled per index type. When switching between DiskAnn indexes
|
||||
// with different element sizes (e.g., fp16 vs fp32), the cached context has
|
||||
// undersized buffers. Recreate it to ensure correct buffer allocations.
|
||||
if (ctx->magic() != magic_) {
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->knn_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->linear_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->keys_search(p_keys[i], ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const {
|
||||
return diskann_indexer_->get_vector(key, context, vector);
|
||||
}
|
||||
|
||||
const void *DiskAnnStreamer::get_vector_by_id(uint32_t id) const {
|
||||
// DiskAnn vectors are stored on disk in sector format;
|
||||
// a const void* access requires sector I/O via create_context
|
||||
// Return nullptr to indicate this path is not supported.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const {
|
||||
// Lazily create a reusable context for fetch operations
|
||||
if (!fetch_ctx_) {
|
||||
fetch_ctx_ = create_context();
|
||||
if (!fetch_ctx_) {
|
||||
LOG_ERROR("Failed to create context for get_vector_by_id");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
int ret = diskann_indexer_->get_vector(id, fetch_ctx_, fetch_vector_buffer_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to get vector by id: %u", id);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
block.reset((void *)fetch_vector_buffer_.data());
|
||||
return 0;
|
||||
}
|
||||
|
||||
IndexSearcher::Provider::Pointer DiskAnnStreamer::create_provider(void) const {
|
||||
if (state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before creating a provider");
|
||||
return nullptr;
|
||||
}
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone DiskAnn entity for provider");
|
||||
return nullptr;
|
||||
}
|
||||
return IndexProvider::Pointer(new (std::nothrow) DiskAnnIndexProvider(
|
||||
meta_, entity, "DiskAnnStreamer"));
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer DiskAnnStreamer::create_context() const {
|
||||
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
|
||||
if (!search_ctx_entity) {
|
||||
LOG_ERROR("Failed to create search context entity");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx =
|
||||
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
|
||||
if (ctx == nullptr) {
|
||||
LOG_ERROR("Failed to allocate DiskAnn Context");
|
||||
return Context::Pointer();
|
||||
}
|
||||
if (ailego_unlikely(ctx->init(
|
||||
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
|
||||
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
|
||||
LOG_ERROR("Init DiskAnn Context failed");
|
||||
delete ctx;
|
||||
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER(DiskAnnStreamer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,180 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
|
||||
class LinuxAlignedFileReader;
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnStreamer : public IndexStreamer {
|
||||
public:
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnStreamer(void);
|
||||
~DiskAnnStreamer(void);
|
||||
|
||||
DiskAnnStreamer(const DiskAnnStreamer &) = delete;
|
||||
DiskAnnStreamer &operator=(const DiskAnnStreamer &) = delete;
|
||||
|
||||
protected:
|
||||
//! Initialize Searcher
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load Index from storage
|
||||
int open(IndexStorage::Pointer storage) override;
|
||||
|
||||
//! Unload index from storage
|
||||
int unload(void) override;
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices,
|
||||
const void *sparse_query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
|
||||
sparse_query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(
|
||||
const void * /*query*/, const uint32_t * /*sparse_count*/,
|
||||
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
|
||||
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
|
||||
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
|
||||
ContextPointer & /*context*/) const override {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get vector by key
|
||||
int get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const override;
|
||||
|
||||
//! Fetch vector by id
|
||||
const void *get_vector_by_id(uint32_t id) const override;
|
||||
|
||||
//! Fetch vector by id into memory block
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
ContextPointer create_context() const override;
|
||||
|
||||
//! Create a vector iterator backed by the on-disk vector segment.
|
||||
//! Used by the merge code path (``MixedStreamerReducer``) to walk every
|
||||
//! vector held by this streamer.
|
||||
IndexSearcher::Provider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual int flush(uint64_t /*check_point*/) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual int close(void) override {
|
||||
return this->unload();
|
||||
}
|
||||
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
template <typename T, typename LabelT = uint32_t>
|
||||
int search_disk_index(const std::string &query_file,
|
||||
const uint32_t num_nodes_to_cache,
|
||||
const uint32_t recall_at, const uint32_t beamwidth);
|
||||
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(DiskAnnContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
IndexMetric::Pointer measure_{};
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
|
||||
DiskAnnSearcherEntity entity_{};
|
||||
|
||||
// Mutable members for get_vector_by_id (caches context and buffer)
|
||||
mutable ContextPointer fetch_ctx_{};
|
||||
mutable std::string fetch_vector_buffer_;
|
||||
|
||||
uint32_t magic_{0U};
|
||||
|
||||
Stats stats_;
|
||||
State state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,221 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnUtil {
|
||||
public:
|
||||
static constexpr uint64_t kSectorSize = 4096;
|
||||
static constexpr uint64_t kMaxSectorReadNum = 128;
|
||||
|
||||
public:
|
||||
static inline size_t div_round_up(size_t x, size_t y) {
|
||||
return (x / y + (x % y != 0));
|
||||
}
|
||||
|
||||
static inline size_t round_up(size_t x, size_t y) {
|
||||
return div_round_up(x, y) * y;
|
||||
}
|
||||
|
||||
static inline void alloc_aligned(void **ptr, size_t size, size_t align) {
|
||||
*ptr = ::aligned_alloc(align, size);
|
||||
}
|
||||
|
||||
static inline void free_aligned(void *ptr) {
|
||||
if (ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
free(ptr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline void convert_vector_to_residual(T *data, uint32_t blocksize_,
|
||||
size_t dim, void *centroid) {
|
||||
const T *centroid_ptr = reinterpret_cast<const T *>(centroid);
|
||||
for (size_t i = 0; i < blocksize_; i++) {
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
float data_float = data[i * dim + d];
|
||||
data_float -= centroid_ptr[d];
|
||||
data[i * dim + d] = data_float;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void convert_types_uint32_to_uint8(const uint32_t *src,
|
||||
uint8_t *dest, size_t npts,
|
||||
size_t dim) {
|
||||
for (size_t i = 0; i < npts; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
dest[i * dim + j] = src[i * dim + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline uint64_t get_node_sector(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
uint32_t sectorsize_,
|
||||
diskann_id_t node_id) {
|
||||
return (node_per_sector > 0
|
||||
? node_id / node_per_sector
|
||||
: node_id * div_round_up(max_nodesize_, sectorsize_));
|
||||
}
|
||||
|
||||
static inline uint32_t *offset_to_node_neighbor(uint8_t *node_buf,
|
||||
uint32_t elementsize_) {
|
||||
return (uint32_t *)(node_buf + elementsize_);
|
||||
}
|
||||
|
||||
static inline uint8_t *offset_to_node(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
uint8_t *sector_buf,
|
||||
diskann_id_t node_id) {
|
||||
return sector_buf + (node_per_sector == 0
|
||||
? 0
|
||||
: (node_id % node_per_sector) * max_nodesize_);
|
||||
}
|
||||
|
||||
static inline const uint8_t *offset_to_node_const(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
const uint8_t *sector_buf,
|
||||
diskann_id_t node_id) {
|
||||
return sector_buf + (node_per_sector == 0
|
||||
? 0
|
||||
: (node_id % node_per_sector) * max_nodesize_);
|
||||
}
|
||||
};
|
||||
|
||||
//! Neighbor
|
||||
struct Neighbor {
|
||||
public:
|
||||
Neighbor() = default;
|
||||
|
||||
Neighbor(diskann_id_t id, float distance)
|
||||
: id{id}, distance{distance}, expanded(false) {}
|
||||
|
||||
inline bool operator<(const Neighbor &other) const {
|
||||
return distance < other.distance ||
|
||||
(distance == other.distance && id < other.id);
|
||||
}
|
||||
|
||||
inline bool operator==(const Neighbor &other) const {
|
||||
return (id == other.id);
|
||||
}
|
||||
|
||||
public:
|
||||
diskann_id_t id;
|
||||
float distance;
|
||||
bool expanded;
|
||||
};
|
||||
|
||||
//! NeighborPriorityQueue
|
||||
class NeighborPriorityQueue {
|
||||
public:
|
||||
NeighborPriorityQueue() : size_(0), capacity_(0), cur_(0) {}
|
||||
|
||||
explicit NeighborPriorityQueue(size_t capacity)
|
||||
: size_(0), capacity_(capacity), cur_(0), data_(capacity + 1) {}
|
||||
|
||||
void insert(const Neighbor &nbr) {
|
||||
if (size_ == capacity_ && data_[size_ - 1] < nbr) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t low = 0, high = size_;
|
||||
while (low < high) {
|
||||
size_t mid = (low + high) >> 1;
|
||||
if (nbr < data_[mid]) {
|
||||
high = mid;
|
||||
} else if (data_[mid].id == nbr.id) {
|
||||
return;
|
||||
} else {
|
||||
low = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (low < capacity_) {
|
||||
std::memmove(&data_[low + 1], &data_[low],
|
||||
(size_ - low) * sizeof(Neighbor));
|
||||
}
|
||||
|
||||
data_[low] = {nbr.id, nbr.distance};
|
||||
if (size_ < capacity_) {
|
||||
size_++;
|
||||
}
|
||||
|
||||
if (low < cur_) {
|
||||
cur_ = low;
|
||||
}
|
||||
}
|
||||
|
||||
Neighbor closest_unexpanded() {
|
||||
data_[cur_].expanded = true;
|
||||
size_t pre = cur_;
|
||||
while (cur_ < size_ && data_[cur_].expanded) {
|
||||
cur_++;
|
||||
}
|
||||
return data_[pre];
|
||||
}
|
||||
|
||||
bool has_unexpanded_node() const {
|
||||
return cur_ < size_;
|
||||
}
|
||||
|
||||
size_t size() const {
|
||||
return size_;
|
||||
}
|
||||
|
||||
size_t capacity() const {
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
void reserve(size_t capacity) {
|
||||
if (capacity + 1 > data_.size()) {
|
||||
data_.resize(capacity + 1);
|
||||
}
|
||||
capacity_ = capacity;
|
||||
}
|
||||
|
||||
Neighbor &operator[](size_t i) {
|
||||
return data_[i];
|
||||
}
|
||||
|
||||
Neighbor operator[](size_t i) const {
|
||||
return data_[i];
|
||||
}
|
||||
|
||||
void sort() {
|
||||
std::sort(data_.begin(), data_.begin() + size_);
|
||||
}
|
||||
|
||||
void clear() {
|
||||
size_ = 0;
|
||||
cur_ = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t size_;
|
||||
size_t capacity_;
|
||||
size_t cur_;
|
||||
std::vector<Neighbor> data_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <ailego/io/mmap_file.h>
|
||||
#include <framework/index_framework.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
enum VecsBitMapIndex {
|
||||
BITMAP_INDEX_KEY = 0,
|
||||
BITMAP_INDEX_DENSE = 1,
|
||||
BITMAP_INDEX_SPARSE = 2,
|
||||
BITMAP_INDEX_PARTITION = 3,
|
||||
BITMAP_INDEX_TAGLIST = 4
|
||||
};
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeaderDetect {
|
||||
uint64_t num_vecs;
|
||||
uint16_t reserved; // in v0, it's meta size
|
||||
uint16_t version; // 0 for v0 format, 1 for new format
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeader {
|
||||
uint64_t num_vecs;
|
||||
uint32_t meta_size;
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeaderV1 {
|
||||
uint64_t num_vecs;
|
||||
uint16_t meta_size_v1;
|
||||
uint16_t version;
|
||||
uint32_t meta_size;
|
||||
uint64_t bitmap; // set for data section
|
||||
uint64_t key_offset; // offset for key
|
||||
uint64_t key_size; // size for key
|
||||
uint64_t dense_offset; // offset for dense
|
||||
uint64_t dense_size; // size for dense
|
||||
uint64_t sparse_offset; // offset for sparse
|
||||
uint64_t sparse_size; // size for sparse
|
||||
uint64_t partition_offset; // offset for partition
|
||||
uint64_t partition_size; // size for partition
|
||||
uint64_t taglist_offset; // offset for taglist
|
||||
uint64_t taglist_size; // size for taglist
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
class DiskAnnVecsReader {
|
||||
public:
|
||||
DiskAnnVecsReader()
|
||||
: mmap_file_(),
|
||||
index_meta_(),
|
||||
num_vecs_(0),
|
||||
vector_base_(nullptr),
|
||||
key_base_(nullptr),
|
||||
sparse_base_meta_{nullptr},
|
||||
sparse_base_data_{nullptr},
|
||||
partition_base_{nullptr},
|
||||
taglist_base_meta_{nullptr},
|
||||
taglist_base_data_{nullptr},
|
||||
taglist_size_{0} {}
|
||||
|
||||
void set_measure(const std::string &name, const IndexParams ¶ms) {
|
||||
index_meta_.set_measure(name, 0, params);
|
||||
}
|
||||
|
||||
bool load(const std::string &fname) {
|
||||
return load(fname.c_str());
|
||||
}
|
||||
|
||||
bool load(const char *fname) {
|
||||
if (!fname) {
|
||||
std::cerr << "Load fname is nullptr" << std::endl;
|
||||
return false;
|
||||
}
|
||||
if (!mmap_file_.open(fname, true)) {
|
||||
std::cerr << "Open file error: " << fname << std::endl;
|
||||
return false;
|
||||
}
|
||||
if (mmap_file_.size() < sizeof(VecsHeaderDetect)) {
|
||||
std::cerr << "File size is too small: " << mmap_file_.size() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const VecsHeaderDetect *header =
|
||||
reinterpret_cast<const VecsHeaderDetect *>(mmap_file_.region());
|
||||
|
||||
if (header->version == 0) {
|
||||
return load_v0();
|
||||
} else if (header->version == 1) {
|
||||
return load_v1();
|
||||
}
|
||||
|
||||
std::cerr << "Can not recognize version: " << header->version << std::endl;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool load_v0() {
|
||||
const VecsHeader *header =
|
||||
reinterpret_cast<const VecsHeader *>(mmap_file_.region());
|
||||
// check
|
||||
num_vecs_ = header->num_vecs;
|
||||
|
||||
// deserialize
|
||||
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
|
||||
if (!bret) {
|
||||
std::cerr << "deserialize index meta error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!index_meta_.hybrid_vector()) {
|
||||
if ((mmap_file_.size() - sizeof(*header) - header->meta_size) %
|
||||
num_vecs_ !=
|
||||
0) {
|
||||
std::cerr << "input file foramt check error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!index_meta_.hybrid_vector()) {
|
||||
vector_base_ =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
} else {
|
||||
vector_base_ =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
sparse_base_meta_ = reinterpret_cast<const char *>(key_base_ + num_vecs_);
|
||||
sparse_base_data_ = reinterpret_cast<const char *>(
|
||||
sparse_base_meta_ + num_vecs_ * sizeof(uint64_t));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool load_v1() {
|
||||
const VecsHeaderV1 *header =
|
||||
reinterpret_cast<const VecsHeaderV1 *>(mmap_file_.region());
|
||||
// check
|
||||
num_vecs_ = header->num_vecs;
|
||||
|
||||
// deserialize
|
||||
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
|
||||
if (!bret) {
|
||||
std::cerr << "deserialize index meta error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const char *data_base_ptr =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
|
||||
vector_base_ = reinterpret_cast<const char *>(data_base_ptr);
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
|
||||
if (header->sparse_offset != -1LLU) {
|
||||
sparse_base_meta_ = data_base_ptr + header->sparse_offset;
|
||||
sparse_base_data_ = sparse_base_meta_ + num_vecs_ * sizeof(uint64_t);
|
||||
}
|
||||
|
||||
if (header->partition_offset != -1LLU) {
|
||||
partition_base_ = reinterpret_cast<const uint32_t *>(
|
||||
data_base_ptr + header->partition_offset);
|
||||
}
|
||||
|
||||
if (header->taglist_offset != -1LLU) {
|
||||
taglist_base_meta_ = data_base_ptr + header->taglist_offset;
|
||||
taglist_base_data_ = taglist_base_meta_ + num_vecs_;
|
||||
taglist_size_ = header->taglist_size;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t num_vecs() const {
|
||||
return num_vecs_;
|
||||
}
|
||||
|
||||
const void *vector_base() const {
|
||||
return vector_base_;
|
||||
}
|
||||
|
||||
const uint64_t *key_base() const {
|
||||
return key_base_;
|
||||
}
|
||||
|
||||
const IndexMeta &index_meta() const {
|
||||
return index_meta_;
|
||||
}
|
||||
|
||||
uint64_t get_key(size_t index) const {
|
||||
return key_base_[index];
|
||||
}
|
||||
|
||||
const void *get_vector(size_t index) const {
|
||||
return vector_base_ + index * index_meta_.element_size();
|
||||
}
|
||||
|
||||
uint32_t get_sparse_count(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t sparse_count =
|
||||
*((uint32_t *)(sparse_base_data_ + sparse_offset));
|
||||
|
||||
return sparse_count;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t *get_sparse_indices(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t *sparse_indices =
|
||||
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t));
|
||||
|
||||
return sparse_indices;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const void *get_sparse_data(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t sparse_count =
|
||||
*((uint32_t *)(sparse_base_data_ + sparse_offset));
|
||||
void *sparse_data =
|
||||
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t) +
|
||||
sparse_count * sizeof(uint32_t));
|
||||
|
||||
return sparse_data;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t get_total_sparse_count(void) const {
|
||||
size_t total_sparse_count = 0;
|
||||
for (size_t i = 0; i < num_vecs_; ++i) {
|
||||
total_sparse_count += get_sparse_count(i);
|
||||
}
|
||||
|
||||
return total_sparse_count;
|
||||
}
|
||||
|
||||
bool has_taglist(void) const {
|
||||
return taglist_base_meta_ != nullptr;
|
||||
}
|
||||
|
||||
uint64_t get_taglist_count(size_t index) const {
|
||||
if (!taglist_base_data_ || !taglist_base_meta_) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t taglist_count = *reinterpret_cast<const uint64_t *>(
|
||||
taglist_base_data_ + taglist_base_meta_[index]);
|
||||
return taglist_count;
|
||||
}
|
||||
|
||||
const uint64_t *get_taglist(size_t index) const {
|
||||
if (!taglist_base_data_ || !taglist_base_meta_) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return reinterpret_cast<const uint64_t *>(taglist_base_data_ +
|
||||
taglist_base_meta_[index]) +
|
||||
1;
|
||||
}
|
||||
|
||||
const void *get_taglist_data(size_t &size) const {
|
||||
size = taglist_size_;
|
||||
|
||||
return taglist_base_meta_;
|
||||
}
|
||||
|
||||
private:
|
||||
ailego::MMapFile mmap_file_;
|
||||
IndexMeta index_meta_;
|
||||
size_t num_vecs_;
|
||||
const char *vector_base_;
|
||||
const uint64_t *key_base_;
|
||||
const char *sparse_base_meta_;
|
||||
const char *sparse_base_data_;
|
||||
const uint32_t *partition_base_;
|
||||
const char *taglist_base_meta_;
|
||||
const char *taglist_base_data_;
|
||||
uint64_t taglist_size_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,419 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <random>
|
||||
#include <tuple>
|
||||
#include <ailego/container/bloom_filter.h>
|
||||
#include <ailego/utility/bitset_helper.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct VisitFilterHeader {
|
||||
VisitFilterHeader() : maxDocCnt(0), maxScanNum(0) {}
|
||||
uint64_t maxDocCnt;
|
||||
uint64_t maxScanNum;
|
||||
};
|
||||
|
||||
constexpr int PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB = 0;
|
||||
|
||||
class VisitBloomFilter {
|
||||
public:
|
||||
static constexpr int mode = 1;
|
||||
|
||||
static constexpr int N = 5;
|
||||
struct Context {
|
||||
Context()
|
||||
: mt(std::chrono::system_clock::now().time_since_epoch().count()) {};
|
||||
VisitFilterHeader h;
|
||||
std::mt19937 mt;
|
||||
ailego::BloomFilter<N> *filter{nullptr};
|
||||
int offset[N] = {0};
|
||||
};
|
||||
#define BLOOM_FILTER_HASH_BITS_OFFSETS(i) \
|
||||
i + c->offset[0], i + c->offset[1], i + c->offset[2], i + c->offset[3], \
|
||||
i + c->offset[4]
|
||||
|
||||
VisitBloomFilter() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->filter->force_insert(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *, id_t) {
|
||||
// TODO
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->filter->has(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
if (maxScanNum == c->h.maxScanNum) {
|
||||
return 0;
|
||||
}
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
if (c->filter->reset(maxScanNum, c->filter->probability()) != 0) {
|
||||
LOG_ERROR("reset BloomFilter failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
genRandomHashBits(c);
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->filter->clear();
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t max_scan_num) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
max_scan_num > c->h.maxScanNum)) {
|
||||
// Create a new one, if failed, we can reuse the old one
|
||||
auto filter = new (std::nothrow) ailego::BloomFilter<VisitBloomFilter::N>(
|
||||
max_scan_num, c->filter->probability());
|
||||
if (ailego_unlikely(filter == nullptr)) {
|
||||
LOG_ERROR("reset bloomfilter failed, maxScanNum %zu prob %f",
|
||||
(size_t)max_scan_num, c->filter->probability());
|
||||
c->filter->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
delete c->filter;
|
||||
c->filter = filter;
|
||||
c->h.maxScanNum = max_scan_num;
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
genRandomHashBits(c);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline static void genRandomHashBits(Context *c) {
|
||||
std::uniform_int_distribution<int> dt(0, c->h.maxDocCnt);
|
||||
for (size_t i = 0; i < sizeof(c->offset) / sizeof(c->offset[0]); ++i) {
|
||||
int r = dt(c->mt);
|
||||
size_t j = 0;
|
||||
do { // gen distinct number
|
||||
for (j = 0; j < i; ++j) {
|
||||
if (c->offset[j] == r) {
|
||||
r = dt(c->mt);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (j < i);
|
||||
c->offset[i] = r;
|
||||
}
|
||||
std::sort(c->offset, c->offset + N);
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
float p =
|
||||
std::get<PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB>(tpl);
|
||||
c->filter = new (std::nothrow)
|
||||
ailego::BloomFilter<VisitBloomFilter::N>(maxScanNum, p);
|
||||
if (c->filter == nullptr) {
|
||||
LOG_ERROR("New BloomFilter failed, reuse old one");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
genRandomHashBits(c);
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete c->filter;
|
||||
delete c;
|
||||
}
|
||||
#undef BLOOM_FILTER_HASH_BITS_OFFSETS
|
||||
}; // end of VisitBloomFilter
|
||||
|
||||
class VisitBitMap {
|
||||
public:
|
||||
static constexpr int mode = 2;
|
||||
|
||||
struct Context {
|
||||
VisitFilterHeader h;
|
||||
ailego::BitsetHelper bitset;
|
||||
char *buf{nullptr};
|
||||
};
|
||||
|
||||
VisitBitMap() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->bitset.set(idx);
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *c, id_t idx) {
|
||||
return &c->buf[idx >> 3];
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->bitset.test(idx);
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->bitset.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
maxScanNum > c->h.maxScanNum)) {
|
||||
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
|
||||
auto buf = new (std::nothrow) char[len];
|
||||
if (buf == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
c->bitset.clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
delete[] c->buf;
|
||||
c->buf = buf;
|
||||
memset(c->buf, 0, len);
|
||||
c->bitset.mount(c->buf, len);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
(void)tpl; // unsed warning
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
|
||||
c->buf = new (std::nothrow) char[len];
|
||||
if (c->buf == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed, reuse old one");
|
||||
delete c;
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
memset(c->buf, 0, len);
|
||||
c->bitset.mount(c->buf, len);
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete[] c->buf;
|
||||
delete c;
|
||||
}
|
||||
}; // end of VisitBitMap
|
||||
|
||||
class VisitByteMap {
|
||||
public:
|
||||
static constexpr int mode = 3;
|
||||
struct Context {
|
||||
VisitFilterHeader h;
|
||||
uint8_t curNum{0};
|
||||
uint8_t *arr{nullptr};
|
||||
};
|
||||
|
||||
VisitByteMap() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->arr[idx] = c->curNum;
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *c, id_t idx) {
|
||||
return c->arr + idx;
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->arr[idx] == c->curNum;
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->curNum++;
|
||||
if (c->curNum == 0) {
|
||||
memset(c->arr, 0, c->h.maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
maxScanNum > c->h.maxScanNum)) {
|
||||
auto arr = new (std::nothrow) uint8_t[maxDocCnt];
|
||||
if (arr != nullptr) {
|
||||
memset(arr, 0, maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
delete[] c->arr;
|
||||
c->arr = arr;
|
||||
return true;
|
||||
}
|
||||
LOG_ERROR("New memory in initVisitByteMap failed, reuse old one");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
(void)tpl; // unsed warning
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitByteMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
c->arr = new (std::nothrow) uint8_t[maxDocCnt];
|
||||
if (c->arr == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitByteMap failed");
|
||||
delete c;
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
memset(c->arr, 0, maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete[] c->arr;
|
||||
delete c;
|
||||
}
|
||||
}; // end of VisitByteMap
|
||||
|
||||
|
||||
#define PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(cls, impl, ctx, ...) \
|
||||
case cls::mode: \
|
||||
return cls::impl(static_cast<cls::Context *>(ctx), ##__VA_ARGS__);
|
||||
|
||||
#define PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(impl, ...) \
|
||||
switch (mode_) { \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBloomFilter, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBitMap, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitByteMap, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
}
|
||||
|
||||
|
||||
// visit list will be called with high frequency,
|
||||
// so using switch instead of std::function or virtual class
|
||||
// funtion point, lambda, virtual class all cannot be inlined
|
||||
class VisitFilter {
|
||||
public:
|
||||
enum Mode {
|
||||
Default = 0,
|
||||
BloomFilter = VisitBloomFilter::mode,
|
||||
BitMap = VisitBitMap::mode,
|
||||
ByteMap = VisitByteMap::mode
|
||||
};
|
||||
|
||||
VisitFilter() : mode_(0), ctx_(nullptr) {};
|
||||
|
||||
inline bool visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(visited, idx);
|
||||
return true; // place holder
|
||||
}
|
||||
|
||||
inline void set_visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_visited, idx);
|
||||
}
|
||||
|
||||
inline void *get_visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(get_visited, idx);
|
||||
return nullptr; // place holder
|
||||
}
|
||||
|
||||
inline int set_max_scan_num(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_max_scan_num, idx);
|
||||
return 0; // place holder
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(clear);
|
||||
}
|
||||
|
||||
inline bool reset(uint64_t maxDocCnt, uint64_t maxScanNum) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(reset, maxDocCnt, maxScanNum);
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void destroy() {
|
||||
if (ctx_ != nullptr) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(destroy);
|
||||
}
|
||||
}
|
||||
|
||||
int init(int mode, uint64_t maxDocCnt, uint64_t maxScanNum,
|
||||
float negativeProbility) {
|
||||
mode_ = mode;
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(init, &ctx_, maxDocCnt, maxScanNum,
|
||||
std::make_tuple(negativeProbility));
|
||||
return 0; // place holder
|
||||
}
|
||||
|
||||
int get_mode(void) const {
|
||||
return mode_;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
VisitFilter(const VisitFilter &) = delete;
|
||||
VisitFilter &operator=(const VisitFilter &) = delete;
|
||||
|
||||
int mode_{0U}; // custom data for each method
|
||||
void *ctx_{nullptr};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -533,4 +533,4 @@ class HnswContext : public IndexContext {
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
@@ -5,6 +5,6 @@ cc_library(
|
||||
NAME core_interface STATIC STRICT ALWAYS_LINK
|
||||
SRCS *.cc indexes/*.cc
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/ ${PROJECT_ROOT_DIR}/src/core
|
||||
LIBS zvec_ailego core_framework sparsehash magic_enum rabitqlib
|
||||
LIBS zvec_ailego core_framework core_plugin sparsehash magic_enum rabitqlib
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
||||
@@ -47,6 +47,10 @@ Index::Pointer IndexFactory::CreateAndInitIndex(const BaseIndexParam ¶m) {
|
||||
ptr = std::make_shared<IVFIndex>();
|
||||
} else if (param.index_type == IndexType::kHNSWRabitq) {
|
||||
ptr = std::make_shared<HNSWRabitqIndex>();
|
||||
#if DISKANN_SUPPORTED
|
||||
} else if (param.index_type == IndexType::kDiskAnn) {
|
||||
ptr = std::make_shared<DiskAnnIndex>();
|
||||
#endif
|
||||
} else if (param.index_type == IndexType::kVamana) {
|
||||
ptr = std::make_shared<VamanaIndex>();
|
||||
} else {
|
||||
|
||||
@@ -199,6 +199,26 @@ ailego::JsonObject VamanaIndexParam::SerializeToJsonObject(
|
||||
return json_obj;
|
||||
}
|
||||
|
||||
bool DiskAnnIndexParam::DeserializeFromJsonObject(
|
||||
const ailego::JsonObject &json_obj) {
|
||||
if (!BaseIndexParam::DeserializeFromJsonObject(json_obj)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (index_type != IndexType::kDiskAnn) {
|
||||
LOG_ERROR("index_type is not DiskAnn");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
ailego::JsonObject DiskAnnIndexParam::SerializeToJsonObject(
|
||||
bool omit_empty_value) const {
|
||||
auto json_obj = BaseIndexParam::SerializeToJsonObject(omit_empty_value);
|
||||
return json_obj;
|
||||
}
|
||||
|
||||
bool VamanaIndexParam::DeserializeFromJsonObject(
|
||||
const ailego::JsonObject &json_obj) {
|
||||
if (!BaseIndexParam::DeserializeFromJsonObject(json_obj)) {
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <zvec/core/interface/index.h>
|
||||
#include <zvec/plugin/diskann_plugin.h>
|
||||
#include "algorithm/diskann/diskann_params.h"
|
||||
#include "holder_builder.h"
|
||||
|
||||
namespace zvec::core_interface {
|
||||
|
||||
namespace {
|
||||
|
||||
// Implicitly bring the DiskAnn runtime online on first use. This keeps the
|
||||
// DiskAnn index an ordinary public API (users just instantiate a
|
||||
// DiskAnnIndexParam) while still letting the rest of the library — HNSW,
|
||||
// IVF, Flat, Vamana — run on hosts that happen to lack libaio. On such
|
||||
// hosts only DiskAnn fails, with a clear, actionable error message, and
|
||||
// every other index type stays fully functional.
|
||||
int EnsureDiskAnnRuntimeReady() {
|
||||
static std::once_flag once;
|
||||
static int cached_result = 0;
|
||||
std::call_once(once, []() {
|
||||
const int status = ::zvec::LoadDiskAnnPlugin();
|
||||
if (status == kDiskAnnPluginOk) {
|
||||
cached_result = 0;
|
||||
return;
|
||||
}
|
||||
switch (status) {
|
||||
case kDiskAnnPluginLibAioMissing:
|
||||
LOG_ERROR(
|
||||
"DiskAnn requires libaio at runtime, but it was not found on this "
|
||||
"host. Install it (e.g. 'apt-get install libaio1' on "
|
||||
"Debian/Ubuntu, "
|
||||
"or 'libaio1t64' on Ubuntu 24.04+) and retry.");
|
||||
break;
|
||||
case kDiskAnnPluginUnsupportedPlatform:
|
||||
LOG_ERROR("DiskAnn is only supported on Linux x86_64.");
|
||||
break;
|
||||
case kDiskAnnPluginDlopenFailed:
|
||||
default:
|
||||
LOG_ERROR("Failed to initialize the DiskAnn runtime (status=%d).",
|
||||
status);
|
||||
break;
|
||||
}
|
||||
cached_result = core::IndexError_Runtime;
|
||||
});
|
||||
return cached_result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int DiskAnnIndex::CreateAndInitStreamer(const BaseIndexParam ¶m) {
|
||||
// Fail fast and cleanly if the DiskAnn runtime cannot be brought up on
|
||||
// this host (most commonly: libaio is missing). The rest of zvec keeps
|
||||
// running; only DiskAnn is unusable.
|
||||
if (int rc = EnsureDiskAnnRuntimeReady(); rc != 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (is_sparse_) {
|
||||
LOG_ERROR("Failed to create streamer. Sparse is not Supported.");
|
||||
return core::IndexError_Unsupported;
|
||||
}
|
||||
|
||||
param_ = dynamic_cast<const DiskAnnIndexParam &>(param);
|
||||
param_.max_degree = std::min(100, param_.max_degree);
|
||||
param_.list_size = std::min(100, param_.list_size);
|
||||
param_.pq_chunk_num = std::min(1024, param_.pq_chunk_num);
|
||||
|
||||
proxima_index_params_.set(core::PARAM_DISKANN_BUILDER_MAX_DEGREE,
|
||||
param_.max_degree);
|
||||
proxima_index_params_.set(core::PARAM_DISKANN_BUILDER_LIST_SIZE,
|
||||
param_.list_size);
|
||||
proxima_index_params_.set(core::PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM,
|
||||
param_.pq_chunk_num);
|
||||
|
||||
builder_ = core::IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
streamer_ = core::IndexFactory::CreateStreamer("DiskAnnStreamer");
|
||||
|
||||
if (ailego_unlikely(!builder_ || !streamer_)) {
|
||||
LOG_ERROR(
|
||||
"Failed to create DiskAnnBuilder/DiskAnnStreamer: DiskAnn factory "
|
||||
"entries are not registered. This usually means the DiskAnn shared "
|
||||
"module could not be located next to the hosting binary.");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
|
||||
IndexMeta real_meta;
|
||||
if (converter_) {
|
||||
real_meta = converter_->meta();
|
||||
} else {
|
||||
real_meta = proxima_index_meta_;
|
||||
}
|
||||
|
||||
const int builder_ret = builder_->init(real_meta, proxima_index_params_);
|
||||
const int streamer_ret = streamer_->init(real_meta, proxima_index_params_);
|
||||
if (ailego_unlikely(builder_ret != 0 || streamer_ret != 0)) {
|
||||
LOG_ERROR(
|
||||
"Failed to init builder or streamer, builder_ret: %d, "
|
||||
"streamer_ret: %d",
|
||||
builder_ret, streamer_ret);
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnIndex::Open(const std::string &file_path,
|
||||
StorageOptions storage_options) {
|
||||
ailego::Params storage_params;
|
||||
file_path_ = file_path;
|
||||
is_read_only_ = storage_options.read_only;
|
||||
switch (storage_options.type) {
|
||||
case StorageOptions::StorageType::kMMAP:
|
||||
case StorageOptions::StorageType::kBufferPool: {
|
||||
// NOTE: DiskAnn index is dumped via FileDumper (plain binary file), which
|
||||
// is not compatible with BufferStorage's IndexFormat layout. Fall back to
|
||||
// FileReadStorage for both MMAP and BufferPool storage types.
|
||||
storage_ = core::IndexFactory::CreateStorage("FileReadStorage");
|
||||
if (storage_ == nullptr) {
|
||||
LOG_ERROR("Failed to create FileReadStorage");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
int ret = storage_->init(storage_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to init FileReadStorage, path: %s, err: %s",
|
||||
file_path_.c_str(), core::IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported storage type");
|
||||
return core::IndexError_Unsupported;
|
||||
}
|
||||
}
|
||||
|
||||
if (!storage_options.create_new) {
|
||||
int ret = storage_->open(file_path_, false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to open storage, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (streamer_ == nullptr || streamer_->open(storage_) != 0) {
|
||||
LOG_ERROR("Failed to open streamer, path: %s", file_path_.c_str());
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
is_trained_ = true;
|
||||
}
|
||||
is_open_ = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnIndex::GenerateHolder() {
|
||||
return BuildMultiPassHolder(param_.data_type, param_.dimension, doc_cache_,
|
||||
converter_, &holder_);
|
||||
}
|
||||
|
||||
int DiskAnnIndex::Add(const VectorData &vector, uint32_t doc_id) {
|
||||
if (is_trained_) {
|
||||
LOG_ERROR("this diskann index is trained");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (!std::holds_alternative<DenseVector>(vector.vector)) {
|
||||
LOG_ERROR("Invalid vector data");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
const DenseVector &dense_vector = std::get<DenseVector>(vector.vector);
|
||||
std::string out_vector_buffer = std::string(
|
||||
static_cast<const char *>(dense_vector.data),
|
||||
input_vector_meta_.dimension() * input_vector_meta_.unit_size());
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (doc_cache_.size() <= doc_id) {
|
||||
std::string fake_data(
|
||||
input_vector_meta_.dimension() * input_vector_meta_.unit_size(), 0);
|
||||
doc_cache_.resize(doc_id + 1, std::make_pair(kInvalidKey, fake_data));
|
||||
}
|
||||
doc_cache_[doc_id] = std::make_pair(doc_id, out_vector_buffer);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnIndex::Train() {
|
||||
int ret = GenerateHolder();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to generate holder, err: %s",
|
||||
core::IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = builder_->train(holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to train builder, err: %s", core::IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = builder_->build(holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to build index, err: %s", core::IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
auto dumper = core::IndexFactory::CreateDumper("FileDumper");
|
||||
if (dumper == nullptr) {
|
||||
LOG_ERROR("Failed to create FileDumper");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
|
||||
ret = dumper->create(file_path_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to create dumper, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
ret = builder_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump index, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
dumper->close();
|
||||
ret = storage_->open(file_path_, false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to open storage, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (streamer_ == nullptr || streamer_->open(storage_) != 0) {
|
||||
LOG_ERROR("Failed to open streamer, path: %s", file_path_.c_str());
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
is_trained_ = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnIndex::_dense_fetch(const uint32_t doc_id,
|
||||
VectorDataBuffer *vector_data_buffer) {
|
||||
if (is_trained_) {
|
||||
return Index::_dense_fetch(doc_id, vector_data_buffer);
|
||||
} else {
|
||||
DenseVectorBuffer dense_vector_buffer;
|
||||
std::string &out_vector_buffer = dense_vector_buffer.data;
|
||||
out_vector_buffer = doc_cache_[doc_id].second;
|
||||
vector_data_buffer->vector_buffer = std::move(dense_vector_buffer);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int DiskAnnIndex::_prepare_for_search(
|
||||
const VectorData & /*query*/,
|
||||
const BaseIndexQueryParam::Pointer &search_param,
|
||||
core::IndexContext::Pointer &context) {
|
||||
const auto &diskann_search_param =
|
||||
std::dynamic_pointer_cast<DiskAnnQueryParam>(search_param);
|
||||
if (diskann_search_param == nullptr) {
|
||||
LOG_ERROR("Invalid search param: expected DiskAnnQueryParam");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
|
||||
context->set_topk(diskann_search_param->topk);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnIndex::Merge(const std::vector<Index::Pointer> &indexes,
|
||||
const IndexFilter &filter,
|
||||
const MergeOptions &options) {
|
||||
int pre_ret = Index::Merge(indexes, filter, options);
|
||||
if (pre_ret != 0) {
|
||||
return pre_ret;
|
||||
}
|
||||
auto dumper = core::IndexFactory::CreateDumper("FileDumper");
|
||||
|
||||
dumper->create(file_path_);
|
||||
int ret = builder_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump index, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
|
||||
dumper->close();
|
||||
|
||||
ret = storage_->open(file_path_, false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to open storage, path: %s, err: %s", file_path_.c_str(),
|
||||
core::IndexError::What(ret));
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (streamer_ == nullptr || streamer_->open(storage_) != 0) {
|
||||
LOG_ERROR("Failed to open streamer, path: %s", file_path_.c_str());
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
is_trained_ = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace zvec::core_interface
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <zvec/core/framework/index_converter.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
#include <zvec/core/interface/index_param.h>
|
||||
|
||||
namespace zvec::core_interface {
|
||||
|
||||
inline constexpr uint64_t kInvalidKey = std::numeric_limits<uint64_t>::max();
|
||||
|
||||
template <core::IndexMeta::DataType DT, typename T>
|
||||
inline int BuildMultiPassHolderImpl(
|
||||
uint32_t dimension,
|
||||
const std::vector<std::pair<uint64_t, std::string>> &doc_cache,
|
||||
core::IndexHolder::Pointer *holder_out) {
|
||||
auto holder =
|
||||
std::make_shared<zvec::core::MultiPassIndexHolder<DT>>(dimension);
|
||||
for (const auto &doc : doc_cache) {
|
||||
if (doc.first == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
ailego::NumericalVector<T> vec(doc.second);
|
||||
if (!holder->emplace(doc.first, vec)) {
|
||||
LOG_ERROR("Failed to add vector");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
*holder_out = holder;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline int BuildMultiPassHolder(
|
||||
DataType data_type, uint32_t dimension,
|
||||
const std::vector<std::pair<uint64_t, std::string>> &doc_cache,
|
||||
const core::IndexConverter::Pointer &converter,
|
||||
core::IndexHolder::Pointer *holder) {
|
||||
int ret = 0;
|
||||
switch (data_type) {
|
||||
case DataType::DT_FP16:
|
||||
ret = BuildMultiPassHolderImpl<DataType::DT_FP16, uint16_t>(
|
||||
dimension, doc_cache, holder);
|
||||
break;
|
||||
case DataType::DT_FP32:
|
||||
ret = BuildMultiPassHolderImpl<DataType::DT_FP32, float>(
|
||||
dimension, doc_cache, holder);
|
||||
break;
|
||||
case DataType::DT_INT8:
|
||||
ret = BuildMultiPassHolderImpl<DataType::DT_INT8, uint8_t>(
|
||||
dimension, doc_cache, holder);
|
||||
break;
|
||||
default:
|
||||
LOG_ERROR("data_type is not support");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
if (converter) {
|
||||
core::IndexConverter::TrainAndTransform(converter, *holder);
|
||||
*holder = converter->result();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace zvec::core_interface
|
||||
@@ -16,11 +16,10 @@
|
||||
#include <string>
|
||||
#include <zvec/core/interface/index.h>
|
||||
#include "algorithm/ivf/ivf_params.h"
|
||||
#include "holder_builder.h"
|
||||
|
||||
namespace zvec::core_interface {
|
||||
|
||||
static constexpr uint64_t kInvalidKey = std::numeric_limits<uint64_t>::max();
|
||||
|
||||
int IVFIndex::CreateAndInitStreamer(const BaseIndexParam ¶m) {
|
||||
if (is_sparse_) {
|
||||
LOG_ERROR("IVF Index not support sparse vector");
|
||||
@@ -129,60 +128,8 @@ int IVFIndex::Open(const std::string &file_path,
|
||||
}
|
||||
|
||||
int IVFIndex::GenerateHolder() {
|
||||
if (param_.data_type == DataType::DT_FP16) {
|
||||
auto holder =
|
||||
std::make_shared<zvec::core::MultiPassIndexHolder<DataType::DT_FP16>>(
|
||||
param_.dimension);
|
||||
for (auto doc : doc_cache_) {
|
||||
ailego::NumericalVector<uint16_t> vec(doc.second);
|
||||
if (doc.first == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!holder->emplace(doc.first, vec)) {
|
||||
LOG_ERROR("Failed to add vector");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
holder_ = holder;
|
||||
} else if (param_.data_type == DataType::DT_FP32) {
|
||||
auto holder =
|
||||
std::make_shared<zvec::core::MultiPassIndexHolder<DataType::DT_FP32>>(
|
||||
param_.dimension);
|
||||
for (auto doc : doc_cache_) {
|
||||
ailego::NumericalVector<float> vec(doc.second);
|
||||
if (doc.first == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!holder->emplace(doc.first, vec)) {
|
||||
LOG_ERROR("Failed to add vector");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
holder_ = holder;
|
||||
} else if (param_.data_type == DataType::DT_INT8) {
|
||||
auto holder =
|
||||
std::make_shared<zvec::core::MultiPassIndexHolder<DataType::DT_INT8>>(
|
||||
param_.dimension);
|
||||
for (auto doc : doc_cache_) {
|
||||
ailego::NumericalVector<uint8_t> vec(doc.second);
|
||||
if (doc.first == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!holder->emplace(doc.first, vec)) {
|
||||
LOG_ERROR("Failed to add vector");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
holder_ = holder;
|
||||
} else {
|
||||
LOG_ERROR("data_type is not support");
|
||||
return core::IndexError_Runtime;
|
||||
}
|
||||
if (converter_) {
|
||||
core::IndexConverter::TrainAndTransform(converter_, holder_);
|
||||
holder_ = converter_->result();
|
||||
}
|
||||
return 0;
|
||||
return BuildMultiPassHolder(param_.data_type, param_.dimension, doc_cache_,
|
||||
converter_, &holder_);
|
||||
}
|
||||
|
||||
int IVFIndex::Add(const VectorData &vector, uint32_t doc_id) {
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
##
|
||||
## Copyright (C) The Software Authors. All rights reserved.
|
||||
##
|
||||
## \file CMakeLists.txt
|
||||
## \brief Build script for the zvec internal plugin-loading glue library.
|
||||
## Lives inside the main zvec_core artifact and provides the
|
||||
## implicit DiskAnn runtime bring-up used by DiskAnnIndex on first
|
||||
## use. These APIs are NOT part of the public user surface: users
|
||||
## simply instantiate ``DiskAnnIndexParam`` / ``DiskAnnIndex`` and
|
||||
## the runtime (``libzvec_diskann_plugin.so``) is loaded behind
|
||||
## the scenes.
|
||||
##
|
||||
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
set(CORE_PLUGIN_LIBS zvec_ailego core_framework)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
list(APPEND CORE_PLUGIN_LIBS ${CMAKE_DL_LIBS})
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME core_plugin
|
||||
STATIC STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS ${CORE_PLUGIN_LIBS}
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
#include <zvec/plugin/diskann_plugin.h>
|
||||
|
||||
#if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
||||
#include <dlfcn.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
#include <limits.h>
|
||||
#endif
|
||||
|
||||
namespace zvec {
|
||||
|
||||
namespace {
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
constexpr const char *kPluginFileName = "libzvec_diskann_plugin.so";
|
||||
// Candidate soname list. On Ubuntu 24.04 the libaio package was renamed with
|
||||
// the t64 suffix (64-bit time_t transition), so we probe both spellings.
|
||||
constexpr const char *kLibAioSoNames[] = {
|
||||
"libaio.so.1",
|
||||
"libaio.so.1t64",
|
||||
};
|
||||
constexpr bool kPlatformSupportsDiskAnnPlugin = true;
|
||||
#elif defined(__APPLE__)
|
||||
constexpr const char *kPluginFileName = "libzvec_diskann_plugin.dylib";
|
||||
constexpr bool kPlatformSupportsDiskAnnPlugin = false;
|
||||
#else
|
||||
constexpr const char *kPluginFileName = "zvec_diskann_plugin.dll";
|
||||
constexpr bool kPlatformSupportsDiskAnnPlugin = false;
|
||||
#endif
|
||||
|
||||
// Global plugin handle. Nullptr means "not loaded".
|
||||
std::atomic<void *> g_plugin_handle{nullptr};
|
||||
std::mutex g_plugin_mutex;
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
|
||||
// Resolve the directory containing the currently running executable, so we
|
||||
// can look for the plugin next to it regardless of the working directory.
|
||||
std::string GetExecutableDir() {
|
||||
char buf[PATH_MAX];
|
||||
ssize_t n = ::readlink("/proc/self/exe", buf, sizeof(buf) - 1);
|
||||
if (n <= 0) {
|
||||
return {};
|
||||
}
|
||||
buf[n] = '\0';
|
||||
std::string path(buf);
|
||||
auto slash = path.find_last_of('/');
|
||||
if (slash == std::string::npos) {
|
||||
return {};
|
||||
}
|
||||
return path.substr(0, slash);
|
||||
}
|
||||
|
||||
// Resolve the directory containing the shared object that hosts this
|
||||
// function. For Python wheels this is the directory of
|
||||
// ``_zvec.cpython-*.so``; for regular C++ binaries it is the directory of
|
||||
// ``libzvec_core.so``. In either case the DiskAnn plugin is shipped
|
||||
// alongside, so this is the most reliable lookup location.
|
||||
//
|
||||
// NOTE: we pass the address of an *exported* function (LoadDiskAnnPlugin)
|
||||
// rather than one in this anonymous namespace, because dladdr() on a symbol
|
||||
// with internal linkage can report the main executable instead of the
|
||||
// hosting shared object when the translation unit is whole-archived into
|
||||
// another .so.
|
||||
std::string ResolveHostingSoDir() {
|
||||
::Dl_info info{};
|
||||
if (::dladdr(reinterpret_cast<void *>(&::zvec::LoadDiskAnnPlugin), &info) ==
|
||||
0 ||
|
||||
info.dli_fname == nullptr) {
|
||||
return {};
|
||||
}
|
||||
std::string path(info.dli_fname);
|
||||
auto slash = path.find_last_of('/');
|
||||
if (slash == std::string::npos) {
|
||||
return {};
|
||||
}
|
||||
return path.substr(0, slash);
|
||||
}
|
||||
|
||||
// Full path of the shared object that hosts LoadDiskAnnPlugin, or empty
|
||||
// string on failure.
|
||||
std::string ResolveHostingSoPath() {
|
||||
::Dl_info info{};
|
||||
if (::dladdr(reinterpret_cast<void *>(&::zvec::LoadDiskAnnPlugin), &info) ==
|
||||
0 ||
|
||||
info.dli_fname == nullptr) {
|
||||
return {};
|
||||
}
|
||||
return std::string(info.dli_fname);
|
||||
}
|
||||
|
||||
// Promote the hosting shared object (e.g. the Python extension module
|
||||
// ``_zvec.cpython-*.so``) to the global symbol scope. Python loads C
|
||||
// extensions with RTLD_LOCAL by default, which means their C++ symbols are
|
||||
// invisible to subsequently dlopen(RTLD_GLOBAL)ed libraries. Without this
|
||||
// promotion, the DiskAnn plugin's undefined references to zvec:: symbols
|
||||
// cannot be resolved against the already-loaded host module and dlopen
|
||||
// fails with messages like:
|
||||
//
|
||||
// undefined symbol: _ZN4zvec6ailego6Logger10LEVEL_INFOE
|
||||
//
|
||||
// Using RTLD_NOLOAD re-opens the existing image without loading it again,
|
||||
// while RTLD_GLOBAL merges its symbols into the global scope. This is a
|
||||
// no-op for hosts that were already loaded with RTLD_GLOBAL.
|
||||
void PromoteHostingSoToGlobal() {
|
||||
const std::string host = ResolveHostingSoPath();
|
||||
if (host.empty()) {
|
||||
return;
|
||||
}
|
||||
// When LoadDiskAnnPlugin is statically linked into the main executable,
|
||||
// dladdr resolves to the executable itself. The main exe's symbols are
|
||||
// already in the global scope by definition, so skip promotion.
|
||||
const std::string exe_path = GetExecutableDir();
|
||||
if (!exe_path.empty()) {
|
||||
char exe_buf[PATH_MAX];
|
||||
ssize_t n = ::readlink("/proc/self/exe", exe_buf, sizeof(exe_buf) - 1);
|
||||
if (n > 0) {
|
||||
exe_buf[n] = '\0';
|
||||
// Compare resolved real paths to handle relative vs absolute.
|
||||
char host_real[PATH_MAX];
|
||||
char exe_real[PATH_MAX];
|
||||
if (::realpath(host.c_str(), host_real) != nullptr &&
|
||||
::realpath(exe_buf, exe_real) != nullptr &&
|
||||
std::string(host_real) == std::string(exe_real)) {
|
||||
// Host IS the main executable; symbols are already global.
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
void *h = ::dlopen(host.c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD);
|
||||
if (h == nullptr) {
|
||||
const char *err = ::dlerror();
|
||||
LOG_WARN("Could not promote host '%s' to RTLD_GLOBAL: %s", host.c_str(),
|
||||
err ? err : "unknown");
|
||||
return;
|
||||
}
|
||||
// We purposely keep the handle refcount incremented for the life of the
|
||||
// process: there is no safe point at which to dlclose() it, and doing so
|
||||
// would only decrement the count anyway (the image remains mapped because
|
||||
// Python still holds its own reference).
|
||||
(void)h;
|
||||
}
|
||||
|
||||
// Build the list of candidate paths for the plugin.
|
||||
std::vector<std::string> BuildCandidatePaths(const std::string &explicit_path) {
|
||||
std::vector<std::string> candidates;
|
||||
if (!explicit_path.empty()) {
|
||||
candidates.push_back(explicit_path);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// Helper that pushes both ``<dir>/<plugin>`` and ``<dir>/../lib/<plugin>``
|
||||
// to handle the conventional CMake build layout where executables live in
|
||||
// ``bin/`` while shared objects (including this plugin) live in ``lib/``.
|
||||
auto push_dir_candidates = [&candidates](const std::string &dir) {
|
||||
if (dir.empty()) {
|
||||
return;
|
||||
}
|
||||
candidates.push_back(dir + "/" + kPluginFileName);
|
||||
candidates.push_back(dir + "/../lib/" + kPluginFileName);
|
||||
};
|
||||
|
||||
// 1. Directory of the library that hosts LoadDiskAnnPlugin (e.g. the
|
||||
// Python extension module or libzvec_core). This works for Python,
|
||||
// C++ embedding, and most packaging layouts.
|
||||
const std::string own_dir = ResolveHostingSoDir();
|
||||
push_dir_candidates(own_dir);
|
||||
// 2. Directory of the running executable. Useful for self-contained C++
|
||||
// tools that drop the plugin next to their binary, as well as for the
|
||||
// standard CMake bin/lib split (handled by ``../lib/`` above).
|
||||
const std::string exe_dir = GetExecutableDir();
|
||||
if (!exe_dir.empty() && exe_dir != own_dir) {
|
||||
push_dir_candidates(exe_dir);
|
||||
}
|
||||
// 3. Fallback: rely on the dynamic linker's default search path
|
||||
// (RPATH / LD_LIBRARY_PATH / /etc/ld.so.conf).
|
||||
candidates.emplace_back(kPluginFileName);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
#endif // linux
|
||||
|
||||
} // namespace
|
||||
|
||||
bool IsLibAioAvailable() {
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
const char *kRequiredSymbols[] = {"io_setup", "io_submit", "io_getevents",
|
||||
"io_destroy"};
|
||||
for (const char *soname : kLibAioSoNames) {
|
||||
// RTLD_LAZY keeps the cost low; we only need to know whether the library
|
||||
// is resolvable and exposes the symbols DiskAnn actually calls.
|
||||
void *handle = ::dlopen(soname, RTLD_LAZY);
|
||||
if (handle == nullptr) {
|
||||
continue;
|
||||
}
|
||||
bool ok = true;
|
||||
for (const char *sym : kRequiredSymbols) {
|
||||
if (::dlsym(handle, sym) == nullptr) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
::dlclose(handle);
|
||||
if (ok) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool IsDiskAnnPluginLoaded() {
|
||||
return g_plugin_handle.load(std::memory_order_acquire) != nullptr;
|
||||
}
|
||||
|
||||
int LoadDiskAnnPlugin(const std::string &path) {
|
||||
if (!kPlatformSupportsDiskAnnPlugin) {
|
||||
LOG_ERROR(
|
||||
"DiskAnn plugin is not supported on this platform; it is only "
|
||||
"available on Linux x86_64 with libaio.");
|
||||
return kDiskAnnPluginUnsupportedPlatform;
|
||||
}
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
// Fast path: already loaded.
|
||||
if (g_plugin_handle.load(std::memory_order_acquire) != nullptr) {
|
||||
return kDiskAnnPluginOk;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(g_plugin_mutex);
|
||||
if (g_plugin_handle.load(std::memory_order_relaxed) != nullptr) {
|
||||
return kDiskAnnPluginOk;
|
||||
}
|
||||
|
||||
if (!IsLibAioAvailable()) {
|
||||
LOG_ERROR(
|
||||
"libaio is not available on this host; the DiskAnn runtime cannot be "
|
||||
"activated. Install libaio1 (e.g. 'apt-get install libaio1', or "
|
||||
"'libaio1t64' on Ubuntu 24.04+) and retry. This does not affect "
|
||||
"other index types (HNSW, IVF, Flat, Vamana).");
|
||||
return kDiskAnnPluginLibAioMissing;
|
||||
}
|
||||
|
||||
const std::vector<std::string> candidates = BuildCandidatePaths(path);
|
||||
// Ensure the hosting module's C++ symbols (zvec::*) are visible to the
|
||||
// plugin at dlopen time. See PromoteHostingSoToGlobal() for the rationale.
|
||||
PromoteHostingSoToGlobal();
|
||||
void *handle = nullptr;
|
||||
std::string last_error;
|
||||
for (const std::string &candidate : candidates) {
|
||||
// RTLD_GLOBAL so the plugin's factory registrations (which live in the
|
||||
// plugin's own static-init code) can reference symbols from the main
|
||||
// library, and any callers that later dlsym against the process can see
|
||||
// the plugin's symbols.
|
||||
handle = ::dlopen(candidate.c_str(), RTLD_NOW | RTLD_GLOBAL);
|
||||
if (handle != nullptr) {
|
||||
LOG_INFO("Loaded DiskAnn plugin from: %s", candidate.c_str());
|
||||
break;
|
||||
}
|
||||
const char *err = ::dlerror();
|
||||
last_error = err ? err : "unknown dlopen error";
|
||||
LOG_DEBUG("dlopen(%s) failed: %s", candidate.c_str(), last_error.c_str());
|
||||
}
|
||||
|
||||
if (handle == nullptr) {
|
||||
LOG_ERROR("Failed to load DiskAnn plugin; last error: %s",
|
||||
last_error.c_str());
|
||||
return kDiskAnnPluginDlopenFailed;
|
||||
}
|
||||
|
||||
g_plugin_handle.store(handle, std::memory_order_release);
|
||||
return kDiskAnnPluginOk;
|
||||
#else
|
||||
(void)path;
|
||||
return kDiskAnnPluginUnsupportedPlatform;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool UnloadDiskAnnPlugin() {
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
std::lock_guard<std::mutex> lock(g_plugin_mutex);
|
||||
void *handle = g_plugin_handle.exchange(nullptr, std::memory_order_acq_rel);
|
||||
if (handle == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (::dlclose(handle) != 0) {
|
||||
const char *err = ::dlerror();
|
||||
LOG_WARN("dlclose for DiskAnn plugin returned non-zero: %s",
|
||||
err ? err : "unknown");
|
||||
}
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
@@ -99,6 +99,11 @@ struct FileDumper : public IndexDumper {
|
||||
return packer_.magic();
|
||||
}
|
||||
|
||||
//! Retrieve size of index
|
||||
size_t size(void) const override {
|
||||
return file_.size();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Close index file
|
||||
bool close_index(void) {
|
||||
|
||||
@@ -82,6 +82,11 @@ class FileReadStorage : public IndexStorage {
|
||||
return region_size_;
|
||||
}
|
||||
|
||||
//! Retrieve offset of data
|
||||
size_t data_offset(void) const override {
|
||||
return data_offset_;
|
||||
}
|
||||
|
||||
//! Fetch data from segment (with own buffer)
|
||||
size_t fetch(size_t offset, void *buf, size_t len) const override {
|
||||
if (ailego_unlikely(offset + len > region_size_)) {
|
||||
@@ -285,7 +290,10 @@ class FileReadStorage : public IndexStorage {
|
||||
}
|
||||
|
||||
int flush(void) override {
|
||||
return IndexError_NotImplemented;
|
||||
// Read-only storage — nothing to flush. Return success so that
|
||||
// generic Index::Flush() works on read-only-backed indexes (e.g.
|
||||
// DiskAnn after build/dump). Mirrors MMapFileReadStorage::flush().
|
||||
return 0;
|
||||
}
|
||||
|
||||
int append(const std::string & /*id*/, size_t /*size*/) override {
|
||||
@@ -385,6 +393,15 @@ class FileReadStorage : public IndexStorage {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Retrieve file ptr if has
|
||||
std::shared_ptr<ailego::File> file(void) const override {
|
||||
return file_ptr_;
|
||||
}
|
||||
|
||||
std::string file_path(void) const override {
|
||||
return file_path_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Open a index file
|
||||
static inline std::shared_ptr<ailego::File> OpenFile(const std::string &path,
|
||||
|
||||
@@ -39,6 +39,7 @@ class MMapFileReadStorage : public IndexStorage {
|
||||
: data_ptr_(reinterpret_cast<uint8_t *>(file_ptr->region()) + offset +
|
||||
segment.data_offset()),
|
||||
data_size_(segment.data_size()),
|
||||
data_offset_(offset + segment.data_offset()),
|
||||
padding_size_(segment.padding_size()),
|
||||
region_size_(segment.data_size() + segment.padding_size()),
|
||||
data_crc_(segment.data_crc()),
|
||||
@@ -66,6 +67,11 @@ class MMapFileReadStorage : public IndexStorage {
|
||||
return region_size_;
|
||||
}
|
||||
|
||||
//! Retrieve offset of data
|
||||
size_t data_offset(void) const override {
|
||||
return data_offset_;
|
||||
}
|
||||
|
||||
//! Fetch data from segment (with own buffer)
|
||||
size_t fetch(size_t offset, void *buf, size_t len) const override {
|
||||
if (ailego_unlikely(offset + len > region_size_)) {
|
||||
@@ -135,6 +141,7 @@ class MMapFileReadStorage : public IndexStorage {
|
||||
private:
|
||||
const uint8_t *data_ptr_{nullptr};
|
||||
size_t data_size_{0u};
|
||||
size_t data_offset_{0};
|
||||
size_t padding_size_{0u};
|
||||
size_t region_size_{0u};
|
||||
uint32_t data_crc_{0u};
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! CRTP base that builds search results from top-k heaps.
|
||||
//!
|
||||
//! The single/group result construction logic is identical across the
|
||||
//! graph-based index contexts. They differ only in the heap element value
|
||||
//! type and in how a single result document is materialized. Concrete
|
||||
//! contexts inherit from this base and supply the small set of accessors and
|
||||
//! hooks listed below (make this base a `friend` if any of them are private):
|
||||
//! - HeapT &topk_heap();
|
||||
//! - std::map<std::string, HeapT> &group_topk_heaps();
|
||||
//! - ResultList &mutable_results(); // std::vector<IndexDocumentList>
|
||||
//! - GroupList &mutable_group_results(); // std::vector<IndexGroupDocList>
|
||||
//! - uint32_t topk() const;
|
||||
//! - uint32_t group_topk() const;
|
||||
//! - uint32_t group_num() const;
|
||||
//! - float threshold() const;
|
||||
//! - bool group_by_search();
|
||||
//! - float result_score(const HeapValue &value) const;
|
||||
//! - void emplace_result_doc(IndexDocumentList &docs, IdType id,
|
||||
//! float score, const HeapValue &value);
|
||||
template <typename Derived, typename HeapT>
|
||||
class TopkResultBuilder {
|
||||
public:
|
||||
//! Heap element value type (e.g. VectorInfo / dist_t / ResultRecord).
|
||||
using HeapValue =
|
||||
typename std::decay<decltype(std::declval<HeapT &>()[0].second)>::type;
|
||||
|
||||
//! Dispatch to single or group result construction.
|
||||
inline void topk_to_result(uint32_t idx) {
|
||||
if (derived().group_by_search()) {
|
||||
topk_to_group_result(idx);
|
||||
} else {
|
||||
topk_to_single_result(idx);
|
||||
}
|
||||
}
|
||||
|
||||
//! Construct a flat result list from the top-k heap.
|
||||
inline void topk_to_single_result(uint32_t idx) {
|
||||
HeapT &heap = derived().topk_heap();
|
||||
if (ailego_unlikely(heap.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto &results = derived().mutable_results();
|
||||
ailego_assert_with(idx < results.size(), "invalid idx");
|
||||
uint32_t size =
|
||||
std::min(derived().topk(), static_cast<uint32_t>(heap.size()));
|
||||
heap.sort();
|
||||
results[idx].clear();
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
float score = derived().result_score(heap[i].second);
|
||||
if (score > derived().threshold()) {
|
||||
break;
|
||||
}
|
||||
derived().emplace_result_doc(results[idx], heap[i].first, score,
|
||||
heap[i].second);
|
||||
}
|
||||
}
|
||||
|
||||
//! Construct grouped result lists from the per-group top-k heaps.
|
||||
inline void topk_to_group_result(uint32_t idx) {
|
||||
auto &group_results = derived().mutable_group_results();
|
||||
ailego_assert_with(idx < group_results.size(), "invalid idx");
|
||||
|
||||
auto &group_heaps = derived().group_topk_heaps();
|
||||
group_results[idx].clear();
|
||||
|
||||
std::vector<std::pair<std::string, HeapT>> group_topk_list;
|
||||
std::vector<std::pair<std::string, float>> best_score_in_groups;
|
||||
for (auto &entry : group_heaps) {
|
||||
HeapT &heap = entry.second;
|
||||
heap.sort();
|
||||
if (heap.size() > 0) {
|
||||
best_score_in_groups.emplace_back(
|
||||
entry.first, derived().result_score(heap[0].second));
|
||||
}
|
||||
}
|
||||
|
||||
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
|
||||
[](const std::pair<std::string, float> &a,
|
||||
const std::pair<std::string, float> &b) {
|
||||
return a.second < b.second;
|
||||
});
|
||||
|
||||
// truncate to group num
|
||||
for (uint32_t i = 0;
|
||||
i < derived().group_num() && i < best_score_in_groups.size(); ++i) {
|
||||
const std::string &group_id = best_score_in_groups[i].first;
|
||||
group_topk_list.emplace_back(group_id, group_heaps[group_id]);
|
||||
}
|
||||
|
||||
group_results[idx].resize(group_topk_list.size());
|
||||
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
|
||||
const std::string &group_id = group_topk_list[i].first;
|
||||
group_results[idx][i].set_group_id(group_id);
|
||||
|
||||
HeapT &heap = group_topk_list[i].second;
|
||||
uint32_t size =
|
||||
std::min(derived().group_topk(), static_cast<uint32_t>(heap.size()));
|
||||
for (uint32_t j = 0; j < size; ++j) {
|
||||
float score = derived().result_score(heap[j].second);
|
||||
if (score > derived().threshold()) {
|
||||
break;
|
||||
}
|
||||
derived().emplace_result_doc(*group_results[idx][i].mutable_docs(),
|
||||
heap[j].first, score, heap[j].second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
inline Derived &derived() {
|
||||
return *static_cast<Derived *>(this);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -203,6 +203,18 @@ class ProximaEngineHelper {
|
||||
return std::move(ivf_query_param);
|
||||
}
|
||||
|
||||
case IndexType::DISKANN: {
|
||||
auto diskann_query_param_result =
|
||||
_build_common_query_param<core_interface::DiskAnnQueryParam>(
|
||||
query_params);
|
||||
if (!diskann_query_param_result.has_value()) {
|
||||
return tl::make_unexpected(Status::InvalidArgument(
|
||||
"failed to build query param: " +
|
||||
diskann_query_param_result.error().message()));
|
||||
}
|
||||
return std::move(diskann_query_param_result.value());
|
||||
}
|
||||
|
||||
case IndexType::VAMANA: {
|
||||
auto vamana_query_param_result =
|
||||
_build_common_query_param<core_interface::VamanaQueryParam>(
|
||||
@@ -417,6 +429,27 @@ class ProximaEngineHelper {
|
||||
return index_param_builder->Build();
|
||||
}
|
||||
|
||||
case IndexType::DISKANN: {
|
||||
auto index_param_builder_result =
|
||||
_build_common_index_param<DiskAnnIndexParams,
|
||||
core_interface::DiskAnnIndexParamBuilder>(
|
||||
field_schema);
|
||||
if (!index_param_builder_result.has_value()) {
|
||||
return tl::make_unexpected(Status::InvalidArgument(
|
||||
"failed to build index param: " +
|
||||
index_param_builder_result.error().message()));
|
||||
}
|
||||
auto index_param_builder = index_param_builder_result.value();
|
||||
|
||||
auto db_index_params = dynamic_cast<const DiskAnnIndexParams *>(
|
||||
field_schema.index_params().get());
|
||||
index_param_builder->WithMaxDegree(db_index_params->max_degree());
|
||||
index_param_builder->WithListSize(db_index_params->list_size());
|
||||
index_param_builder->WithPqChunkNum(db_index_params->pq_chunk_num());
|
||||
|
||||
return index_param_builder->Build();
|
||||
}
|
||||
|
||||
case IndexType::VAMANA: {
|
||||
auto index_param_builder_result =
|
||||
_build_common_index_param<VamanaIndexParams,
|
||||
|
||||
@@ -144,6 +144,28 @@ proto::InvertIndexParams ProtoConverter::ToPb(const InvertIndexParams *params) {
|
||||
return params_pb;
|
||||
}
|
||||
|
||||
// DiskAnnIndexParams
|
||||
DiskAnnIndexParams::OPtr ProtoConverter::FromPb(
|
||||
const proto::DiskAnnIndexParams ¶ms_pb) {
|
||||
return std::make_shared<DiskAnnIndexParams>(
|
||||
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
|
||||
params_pb.max_degree(), params_pb.list_size(), params_pb.pq_chunk_num(),
|
||||
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()));
|
||||
}
|
||||
|
||||
proto::DiskAnnIndexParams ProtoConverter::ToPb(
|
||||
const DiskAnnIndexParams *params) {
|
||||
proto::DiskAnnIndexParams params_pb;
|
||||
params_pb.mutable_base()->set_metric_type(
|
||||
MetricTypeCodeBook::Get(params->metric_type()));
|
||||
params_pb.mutable_base()->set_quantize_type(
|
||||
QuantizeTypeCodeBook::Get(params->quantize_type()));
|
||||
params_pb.set_max_degree(params->max_degree());
|
||||
params_pb.set_list_size(params->list_size());
|
||||
params_pb.set_pq_chunk_num(params->pq_chunk_num());
|
||||
return params_pb;
|
||||
}
|
||||
|
||||
// FtsIndexParams
|
||||
FtsIndexParams::Ptr ProtoConverter::FromPb(
|
||||
const proto::FtsIndexParams ¶ms_pb) {
|
||||
@@ -235,6 +257,8 @@ IndexParams::Ptr ProtoConverter::FromPb(const proto::IndexParams ¶ms_pb) {
|
||||
return ProtoConverter::FromPb(params_pb.flat());
|
||||
} else if (params_pb.has_hnsw_rabitq()) {
|
||||
return ProtoConverter::FromPb(params_pb.hnsw_rabitq());
|
||||
} else if (params_pb.has_diskann()) {
|
||||
return ProtoConverter::FromPb(params_pb.diskann());
|
||||
} else if (params_pb.has_vamana()) {
|
||||
return ProtoConverter::FromPb(params_pb.vamana());
|
||||
} else if (params_pb.has_fts()) {
|
||||
@@ -302,6 +326,14 @@ proto::IndexParams ProtoConverter::ToPb(const IndexParams *params) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case IndexType::DISKANN: {
|
||||
auto diskann_params = dynamic_cast<const DiskAnnIndexParams *>(params);
|
||||
if (diskann_params) {
|
||||
params_pb.mutable_diskann()->CopyFrom(
|
||||
ProtoConverter::ToPb(diskann_params));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case IndexType::VAMANA: {
|
||||
auto vamana_params = dynamic_cast<const VamanaIndexParams *>(params);
|
||||
if (vamana_params) {
|
||||
|
||||
@@ -48,6 +48,11 @@ struct ProtoConverter {
|
||||
const proto::InvertIndexParams ¶ms_pb);
|
||||
static proto::InvertIndexParams ToPb(const InvertIndexParams *params);
|
||||
|
||||
// DiskAnnIndexParams
|
||||
static DiskAnnIndexParams::OPtr FromPb(
|
||||
const proto::DiskAnnIndexParams ¶ms_pb);
|
||||
static proto::DiskAnnIndexParams ToPb(const DiskAnnIndexParams *params);
|
||||
|
||||
// FtsIndexParams
|
||||
static FtsIndexParams::Ptr FromPb(const proto::FtsIndexParams ¶ms_pb);
|
||||
static proto::FtsIndexParams ToPb(const FtsIndexParams *params);
|
||||
|
||||
@@ -54,8 +54,8 @@ std::unordered_set<DataType> support_sparse_vector_type = {
|
||||
};
|
||||
|
||||
std::unordered_set<IndexType> support_dense_vector_index = {
|
||||
IndexType::FLAT, IndexType::HNSW, IndexType::HNSW_RABITQ, IndexType::IVF,
|
||||
IndexType::VAMANA};
|
||||
IndexType::FLAT, IndexType::HNSW, IndexType::HNSW_RABITQ,
|
||||
IndexType::IVF, IndexType::DISKANN, IndexType::VAMANA};
|
||||
|
||||
std::unordered_set<IndexType> support_sparse_vector_index = {IndexType::FLAT,
|
||||
IndexType::HNSW};
|
||||
|
||||
@@ -44,6 +44,8 @@ struct IndexTypeCodeBook {
|
||||
return IndexType::VAMANA;
|
||||
case proto::IT_INVERT:
|
||||
return IndexType::INVERT;
|
||||
case proto::IT_DISKANN:
|
||||
return IndexType::DISKANN;
|
||||
case proto::IT_FTS:
|
||||
return IndexType::FTS;
|
||||
default:
|
||||
@@ -67,6 +69,8 @@ struct IndexTypeCodeBook {
|
||||
return proto::IT_VAMANA;
|
||||
case IndexType::INVERT:
|
||||
return proto::IT_INVERT;
|
||||
case IndexType::DISKANN:
|
||||
return proto::IT_DISKANN;
|
||||
case IndexType::FTS:
|
||||
return proto::IT_FTS;
|
||||
default:
|
||||
@@ -86,6 +90,8 @@ struct IndexTypeCodeBook {
|
||||
return "FLAT";
|
||||
case IndexType::IVF:
|
||||
return "IVF";
|
||||
case IndexType::DISKANN:
|
||||
return "DISKANN";
|
||||
case IndexType::VAMANA:
|
||||
return "VAMANA";
|
||||
case IndexType::INVERT:
|
||||
|
||||
+12
-2
@@ -58,8 +58,10 @@ enum IndexType {
|
||||
IT_FLAT = 3;
|
||||
// Proxima HNSW RABITQ Index
|
||||
IT_HNSW_RABITQ = 4;
|
||||
// Proxima Vamana (DiskANN) Index
|
||||
// Proxima Vamana Index
|
||||
IT_VAMANA = 5;
|
||||
// Proxima DiskAnn Index
|
||||
IT_DISKANN = 6;
|
||||
// Invert Index
|
||||
IT_INVERT = 10;
|
||||
// Full-Text Search Index
|
||||
@@ -120,6 +122,13 @@ message IVFIndexParams {
|
||||
bool use_soar = 4;
|
||||
}
|
||||
|
||||
message DiskAnnIndexParams {
|
||||
BaseIndexParams base = 1;
|
||||
int32 max_degree = 2;
|
||||
int32 list_size = 3;
|
||||
int32 pq_chunk_num = 4;
|
||||
}
|
||||
|
||||
message VamanaIndexParams {
|
||||
BaseIndexParams base = 1;
|
||||
int32 max_degree = 2;
|
||||
@@ -148,6 +157,7 @@ message IndexParams {
|
||||
HnswRabitqIndexParams hnsw_rabitq = 5;
|
||||
VamanaIndexParams vamana = 6;
|
||||
FtsIndexParams fts = 7;
|
||||
DiskAnnIndexParams diskann = 8;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -216,4 +226,4 @@ message Manifest {
|
||||
uint32 delete_snapshot_path_suffix = 7;
|
||||
|
||||
uint32 next_segment_id = 8;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstring>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
@@ -109,10 +110,43 @@ class Factory {
|
||||
//! Constructor
|
||||
Factory(void) : map_() {}
|
||||
|
||||
//! Retrieve the singleton factory
|
||||
//! Retrieve the singleton factory.
|
||||
//!
|
||||
//! We deliberately avoid the canonical
|
||||
//!
|
||||
//! static Factory factory; return &factory;
|
||||
//!
|
||||
//! "magic static" pattern here. That pattern requires the compiler to emit
|
||||
//! a guard variable (`_ZGVZN...E7factory`) alongside the object, and both
|
||||
//! must be unified across DSOs for the singleton to be shared.
|
||||
//!
|
||||
//! In the Python extension build the _zvec.so version script exports the
|
||||
//! storage (`zvec::*` matches its demangled name) while the guard variable,
|
||||
//! whose demangled form is `guard variable for zvec::...`, ends up hidden
|
||||
//! (compilers emit the guard in a COMDAT group whose visibility is not
|
||||
//! upgraded by our version script). When libzvec_diskann_plugin.so is
|
||||
//! loaded, _zvec.so and the plugin then share the `factory` storage but
|
||||
//! each have their own guard; the plugin's still-zero guard triggers a
|
||||
//! second run of the Factory constructor on the shared storage, wiping
|
||||
//! all registrations performed during _zvec.so import (e.g. FlatStreamer).
|
||||
//!
|
||||
//! A constant-initialized static std::atomic<T*> has NO guard variable
|
||||
//! (its zero init is compile-time), so we use a leaked heap singleton with
|
||||
//! atomic double-checked locking. Only the atomic pointer itself needs to
|
||||
//! be unified across DSOs, and it merges cleanly as STT_GNU_UNIQUE.
|
||||
static Factory *Instance(void) {
|
||||
static Factory factory;
|
||||
return (&factory);
|
||||
static std::atomic<Factory *> ptr{nullptr};
|
||||
Factory *cur = ptr.load(std::memory_order_acquire);
|
||||
if (cur) {
|
||||
return cur;
|
||||
}
|
||||
Factory *created = new Factory();
|
||||
if (ptr.compare_exchange_strong(cur, created, std::memory_order_acq_rel,
|
||||
std::memory_order_acquire)) {
|
||||
return created;
|
||||
}
|
||||
delete created;
|
||||
return cur;
|
||||
}
|
||||
|
||||
//! Inserts a new class into map
|
||||
@@ -163,10 +197,34 @@ class Factory {
|
||||
std::map<const char *, std::function<TBase *()>, KeyComparer> map_;
|
||||
};
|
||||
|
||||
// Force compiler/linker retention of the factory-registration static object.
|
||||
// Under clang -O3 + --gc-sections + install.strip (CI Release), these
|
||||
// file-scope static objects have no external references and can be silently
|
||||
// dropped along with their .init_array entries, causing runtime
|
||||
// 'Create vector column indexer failed' failures for indexers like
|
||||
// VamanaStreamer. `used` defeats compiler dead-code elimination and
|
||||
// `retain` (GCC>=11 / Clang>=13) defeats linker `--gc-sections`. On MSVC
|
||||
// the equivalent is achieved elsewhere via /WHOLEARCHIVE. Fall back to just
|
||||
// `used` on older toolchains that don't know `retain`.
|
||||
#if defined(__has_attribute)
|
||||
#if __has_attribute(retain) && __has_attribute(used)
|
||||
#define AILEGO_FACTORY_REGISTER_ATTR __attribute__((used, retain))
|
||||
#elif __has_attribute(used)
|
||||
#define AILEGO_FACTORY_REGISTER_ATTR __attribute__((used))
|
||||
#else
|
||||
#define AILEGO_FACTORY_REGISTER_ATTR
|
||||
#endif
|
||||
#elif defined(__GNUC__)
|
||||
#define AILEGO_FACTORY_REGISTER_ATTR __attribute__((used))
|
||||
#else
|
||||
#define AILEGO_FACTORY_REGISTER_ATTR
|
||||
#endif
|
||||
|
||||
//! Factory Register
|
||||
#define AILEGO_FACTORY_REGISTER(__NAME__, __BASE__, __IMPL__, ...) \
|
||||
static ailego::Factory<__BASE__>::Register<__IMPL__> \
|
||||
__ailegoFactoryRegister_##__NAME__(#__NAME__, ##__VA_ARGS__)
|
||||
static AILEGO_FACTORY_REGISTER_ATTR \
|
||||
ailego::Factory<__BASE__>::Register<__IMPL__> \
|
||||
__ailegoFactoryRegister_##__NAME__(#__NAME__, ##__VA_ARGS__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
|
||||
@@ -98,6 +98,23 @@ class IndexDocument {
|
||||
vector_(vector),
|
||||
sparse_doc_(std::move(sparse_doc)) {}
|
||||
|
||||
//! Constructor
|
||||
IndexDocument(uint64_t k, float v, uint32_t i, std::string vector_string)
|
||||
: key_(k),
|
||||
score_(v),
|
||||
index_(i),
|
||||
vector_string_(std::move(vector_string)) {}
|
||||
|
||||
//! Constructor
|
||||
IndexDocument(uint64_t k, float v, uint32_t i, std::string vector_string,
|
||||
IndexSparseDocument sparse_doc)
|
||||
: key_(k),
|
||||
score_(v),
|
||||
index_(i),
|
||||
vector_{nullptr},
|
||||
vector_string_(std::move(vector_string)),
|
||||
sparse_doc_(std::move(sparse_doc)) {}
|
||||
|
||||
IndexDocument(uint64_t k, float v, uint32_t i,
|
||||
IndexStorage::MemoryBlock vec_block,
|
||||
IndexSparseDocument sparse_doc)
|
||||
@@ -116,6 +133,7 @@ class IndexDocument {
|
||||
score_(rhs.score_),
|
||||
index_(rhs.index_),
|
||||
vector_(rhs.vector_),
|
||||
vector_string_(std::move(rhs.vector_string_)),
|
||||
sparse_doc_{rhs.sparse_doc_} {
|
||||
if (rhs.has_vec_mem_block_) {
|
||||
vec_mem_block_ = rhs.vec_mem_block_;
|
||||
@@ -130,6 +148,7 @@ class IndexDocument {
|
||||
score_ = rhs.score_;
|
||||
index_ = rhs.index_;
|
||||
vector_ = rhs.vector_;
|
||||
vector_string_ = rhs.vector_string_;
|
||||
if (rhs.has_vec_mem_block_) {
|
||||
vec_mem_block_ = rhs.vec_mem_block_;
|
||||
has_vec_mem_block_ = true;
|
||||
@@ -169,6 +188,11 @@ class IndexDocument {
|
||||
return vector_;
|
||||
}
|
||||
|
||||
//! Retrieve vec string
|
||||
const std::string &vector_string() const {
|
||||
return vector_string_;
|
||||
}
|
||||
|
||||
//! Retrieve vec
|
||||
const IndexSparseDocument &sparse_doc() const {
|
||||
return sparse_doc_;
|
||||
@@ -210,6 +234,7 @@ class IndexDocument {
|
||||
float score_{0.0f};
|
||||
uint32_t index_{0u};
|
||||
const void *vector_{nullptr};
|
||||
std::string vector_string_{};
|
||||
bool has_vec_mem_block_{false};
|
||||
mutable IndexStorage::MemoryBlock vec_mem_block_{};
|
||||
IndexSparseDocument sparse_doc_{};
|
||||
|
||||
@@ -51,6 +51,11 @@ class IndexDumper : public IndexModule {
|
||||
|
||||
//! Retrieve magic number of index
|
||||
virtual uint32_t magic(void) const = 0;
|
||||
|
||||
//! Retrieve size
|
||||
virtual size_t size(void) const {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/*! Index Segment Dumper
|
||||
|
||||
@@ -447,6 +447,12 @@ class IndexRunner : public IndexModule {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//! Get vector by key
|
||||
virtual int get_vector(uint64_t /*key*/, Context::Pointer & /*context*/,
|
||||
std::string & /*vector*/) const {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(const uint32_t /*id*/,
|
||||
IndexStorage::MemoryBlock & /*block*/) const {
|
||||
return IndexError_NotImplemented;
|
||||
|
||||
@@ -105,6 +105,11 @@ class IndexSegmentStorage : public IndexStorage {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Retrieve offset of data
|
||||
virtual size_t data_offset(void) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void update_data_crc(uint32_t) override {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <cstring>
|
||||
#include <zvec/ailego/buffer/vector_page_table.h>
|
||||
#include <zvec/ailego/container/params.h>
|
||||
#include <zvec/ailego/io/file.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_module.h>
|
||||
|
||||
@@ -285,6 +286,11 @@ class IndexStorage : public IndexModule {
|
||||
//! Retrieve size of data
|
||||
virtual size_t data_size(void) const = 0;
|
||||
|
||||
//! Retrieve offset of data
|
||||
virtual size_t data_offset(void) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve crc of data
|
||||
virtual uint32_t data_crc(void) const = 0;
|
||||
|
||||
@@ -379,6 +385,15 @@ class IndexStorage : public IndexModule {
|
||||
virtual MemoryBlock::MemoryBlockType memory_block_type(void) const {
|
||||
return MemoryBlock::MBT_MMAP;
|
||||
}
|
||||
|
||||
//! Retrieve file ptr if has
|
||||
virtual std::shared_ptr<ailego::File> file(void) const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual std::string file_path(void) const {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
|
||||
@@ -33,5 +33,8 @@ constexpr static bool kDefaultVamanaSaturateGraph = false;
|
||||
constexpr const uint32_t kDefaultRabitqTotalBits = 7;
|
||||
constexpr const uint32_t kDefaultRabitqNumClusters = 16;
|
||||
|
||||
constexpr const uint32_t kDefaultDiskAnnMaxDegree = 100;
|
||||
constexpr const uint32_t kDefaultDiskAnnListSize = 200;
|
||||
constexpr const uint32_t kDefaultDiskAnnPqChunkNum = 16;
|
||||
|
||||
} // namespace zvec::core_interface
|
||||
@@ -338,5 +338,36 @@ class HNSWRabitqIndex : public Index {
|
||||
HNSWRabitqIndexParam param_{};
|
||||
};
|
||||
|
||||
class DiskAnnIndex : public Index {
|
||||
public:
|
||||
DiskAnnIndex() = default;
|
||||
|
||||
protected:
|
||||
virtual int CreateAndInitStreamer(const BaseIndexParam ¶m) override;
|
||||
|
||||
virtual int _prepare_for_search(
|
||||
const VectorData &query, const BaseIndexQueryParam::Pointer &search_param,
|
||||
core::IndexContext::Pointer &context) override;
|
||||
|
||||
int Add(const VectorData &vector, uint32_t doc_id) override;
|
||||
|
||||
int Train() override;
|
||||
|
||||
int Open(const std::string &file_path,
|
||||
StorageOptions storage_options) override;
|
||||
|
||||
int _dense_fetch(const uint32_t doc_id,
|
||||
VectorDataBuffer *vector_data_buffer) override;
|
||||
int Merge(const std::vector<Index::Pointer> &indexes,
|
||||
const IndexFilter &filter, const MergeOptions &options) override;
|
||||
int GenerateHolder();
|
||||
|
||||
private:
|
||||
DiskAnnIndexParam param_{};
|
||||
std::mutex mutex_{};
|
||||
std::vector<std::pair<uint64_t, std::string>> doc_cache_;
|
||||
core::IndexHolder::Pointer holder_{};
|
||||
std::string file_path_;
|
||||
};
|
||||
|
||||
} // namespace zvec::core_interface
|
||||
|
||||
@@ -63,6 +63,7 @@ enum class IndexType {
|
||||
kIVF, // it's actual a two-layer index
|
||||
kHNSW,
|
||||
kHNSWRabitq,
|
||||
kDiskAnn,
|
||||
kVamana,
|
||||
};
|
||||
|
||||
@@ -214,6 +215,14 @@ struct IVFQueryParam : public BaseIndexQueryParam {
|
||||
}
|
||||
};
|
||||
|
||||
struct DiskAnnQueryParam : public BaseIndexQueryParam {
|
||||
using Pointer = std::shared_ptr<DiskAnnQueryParam>;
|
||||
|
||||
BaseIndexQueryParam::Pointer Clone() const override {
|
||||
return std::make_shared<DiskAnnQueryParam>(*this);
|
||||
}
|
||||
};
|
||||
|
||||
// --- Construction Parameters ---
|
||||
// template<typename IndexQueryParamType>
|
||||
class BaseIndexParam : public SerializableBase {
|
||||
@@ -402,4 +411,27 @@ struct HNSWRabitqIndexParam : public BaseIndexParam {
|
||||
bool omit_empty_value = false) const override;
|
||||
};
|
||||
|
||||
struct DiskAnnIndexParam : public BaseIndexParam {
|
||||
using Pointer = std::shared_ptr<DiskAnnIndexParam>;
|
||||
|
||||
int max_degree = kDefaultDiskAnnMaxDegree;
|
||||
int list_size = kDefaultDiskAnnListSize;
|
||||
int pq_chunk_num = kDefaultDiskAnnPqChunkNum;
|
||||
|
||||
// Constructors with delegation
|
||||
DiskAnnIndexParam() : BaseIndexParam(IndexType::kDiskAnn) {}
|
||||
|
||||
DiskAnnIndexParam(MetricType metric, int dim, int max_degree, int list_size,
|
||||
int pq_chunk_num)
|
||||
: BaseIndexParam(IndexType::kDiskAnn, metric, dim),
|
||||
max_degree(max_degree),
|
||||
list_size(list_size),
|
||||
pq_chunk_num(pq_chunk_num) {}
|
||||
|
||||
protected:
|
||||
bool DeserializeFromJsonObject(const ailego::JsonObject &json_obj) override;
|
||||
ailego::JsonObject SerializeToJsonObject(
|
||||
bool omit_empty_value = false) const override;
|
||||
};
|
||||
|
||||
} // namespace zvec::core_interface
|
||||
@@ -195,6 +195,28 @@ class HNSWRabitqIndexParamBuilder
|
||||
}
|
||||
};
|
||||
|
||||
class DiskAnnIndexParamBuilder
|
||||
: public BaseIndexParamBuilder<DiskAnnIndexParamBuilder,
|
||||
DiskAnnIndexParam> {
|
||||
public:
|
||||
DiskAnnIndexParamBuilder() = default;
|
||||
DiskAnnIndexParamBuilder &WithMaxDegree(int max_degree) {
|
||||
param->max_degree = max_degree;
|
||||
return *this;
|
||||
}
|
||||
DiskAnnIndexParamBuilder &WithListSize(int list_size) {
|
||||
param->list_size = list_size;
|
||||
return *this;
|
||||
}
|
||||
DiskAnnIndexParamBuilder &WithPqChunkNum(int pq_chunk_num) {
|
||||
param->pq_chunk_num = pq_chunk_num;
|
||||
return *this;
|
||||
}
|
||||
std::shared_ptr<DiskAnnIndexParam> Build() override {
|
||||
return param;
|
||||
}
|
||||
};
|
||||
|
||||
class VamanaIndexParamBuilder
|
||||
: public BaseIndexParamBuilder<VamanaIndexParamBuilder, VamanaIndexParam> {
|
||||
public:
|
||||
|
||||
@@ -54,7 +54,7 @@ class IndexParams {
|
||||
bool is_vector_index_type() const {
|
||||
return type_ == IndexType::FLAT || type_ == IndexType::HNSW ||
|
||||
type_ == IndexType::HNSW_RABITQ || type_ == IndexType::IVF ||
|
||||
type_ == IndexType::VAMANA;
|
||||
type_ == IndexType::DISKANN || type_ == IndexType::VAMANA;
|
||||
}
|
||||
|
||||
IndexType type() const {
|
||||
@@ -454,6 +454,78 @@ class IVFIndexParams : public VectorIndexParams {
|
||||
bool use_soar_;
|
||||
};
|
||||
|
||||
class DiskAnnIndexParams : public VectorIndexParams {
|
||||
public:
|
||||
DiskAnnIndexParams(MetricType metric_type, int max_degree = 100,
|
||||
int list_size = 50, int pq_chunk_num = 0,
|
||||
QuantizeType quantize_type = QuantizeType::UNDEFINED)
|
||||
: VectorIndexParams(IndexType::DISKANN, metric_type, quantize_type),
|
||||
max_degree_{max_degree},
|
||||
list_size_{list_size},
|
||||
pq_chunk_num_{pq_chunk_num} {}
|
||||
|
||||
using OPtr = std::shared_ptr<DiskAnnIndexParams>;
|
||||
|
||||
public:
|
||||
Ptr clone() const override {
|
||||
return std::make_shared<DiskAnnIndexParams>(
|
||||
metric_type_, max_degree_, list_size_, pq_chunk_num_, quantize_type_);
|
||||
}
|
||||
|
||||
std::string to_string() const override {
|
||||
auto base_str = vector_index_params_to_string("DiskAnnIndexParams",
|
||||
metric_type_, quantize_type_);
|
||||
std::ostringstream oss;
|
||||
oss << base_str << ",max_degree:" << max_degree_
|
||||
<< ",list_size:" << list_size_ << ", pq_chunk_num:" << pq_chunk_num_
|
||||
<< "}";
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
int max_degree() const {
|
||||
return max_degree_;
|
||||
}
|
||||
|
||||
void set_max_degree(int max_degree) {
|
||||
max_degree_ = max_degree;
|
||||
}
|
||||
|
||||
int list_size() const {
|
||||
return list_size_;
|
||||
}
|
||||
|
||||
void set_list_size(int list_size) {
|
||||
list_size_ = list_size;
|
||||
}
|
||||
|
||||
int pq_chunk_num() const {
|
||||
return pq_chunk_num_;
|
||||
}
|
||||
|
||||
void set_pq_chunk_num(int pq_chunk_num) {
|
||||
pq_chunk_num_ = pq_chunk_num;
|
||||
}
|
||||
|
||||
bool operator==(const IndexParams &other) const override {
|
||||
return type() == other.type() &&
|
||||
metric_type() ==
|
||||
static_cast<const DiskAnnIndexParams &>(other).metric_type() &&
|
||||
max_degree_ ==
|
||||
static_cast<const DiskAnnIndexParams &>(other).max_degree_ &&
|
||||
list_size_ ==
|
||||
static_cast<const DiskAnnIndexParams &>(other).list_size_ &&
|
||||
pq_chunk_num_ ==
|
||||
static_cast<const DiskAnnIndexParams &>(other).pq_chunk_num_ &&
|
||||
quantize_type() ==
|
||||
static_cast<const DiskAnnIndexParams &>(other).quantize_type();
|
||||
}
|
||||
|
||||
private:
|
||||
int max_degree_;
|
||||
int list_size_;
|
||||
int pq_chunk_num_;
|
||||
};
|
||||
|
||||
/*
|
||||
* Vector: Vamana index params
|
||||
*/
|
||||
@@ -514,6 +586,7 @@ class VamanaIndexParams : public VectorIndexParams {
|
||||
int max_degree() const {
|
||||
return max_degree_;
|
||||
}
|
||||
|
||||
void set_max_degree(int max_degree) {
|
||||
max_degree_ = max_degree;
|
||||
}
|
||||
|
||||
@@ -173,6 +173,28 @@ class FlatQueryParams : public QueryParams {
|
||||
float scale_factor_{10};
|
||||
};
|
||||
|
||||
class DiskAnnQueryParams : public QueryParams {
|
||||
public:
|
||||
DiskAnnQueryParams(int list_size = 300) : QueryParams(IndexType::DISKANN) {
|
||||
set_list_size(list_size);
|
||||
}
|
||||
|
||||
virtual ~DiskAnnQueryParams() = default;
|
||||
|
||||
int list_size() const {
|
||||
return list_size_;
|
||||
}
|
||||
|
||||
void set_list_size(int list_size) {
|
||||
list_size_ = list_size;
|
||||
}
|
||||
|
||||
private:
|
||||
// list size: controls the size of the search frontier during graph traversal
|
||||
// — larger values trade query latency for recall
|
||||
int list_size_;
|
||||
};
|
||||
|
||||
class VamanaQueryParams : public QueryParams {
|
||||
public:
|
||||
VamanaQueryParams(int ef_search = core_interface::kDefaultVamanaEfSearch,
|
||||
|
||||
@@ -26,7 +26,8 @@ enum class IndexType : uint32_t {
|
||||
IVF = 2,
|
||||
FLAT = 3,
|
||||
HNSW_RABITQ = 4,
|
||||
VAMANA = 5,
|
||||
DISKANN = 5,
|
||||
VAMANA = 6,
|
||||
INVERT = 10,
|
||||
FTS = 11,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
// NOTE: The APIs declared in this header are INTERNAL to zvec. They are
|
||||
// invoked implicitly by ``DiskAnnIndex`` on first use so that DiskAnn works
|
||||
// out of the box, without users ever calling a ``load_diskann_plugin()`` /
|
||||
// ``is_libaio_available()`` entry point. External callers should not depend
|
||||
// on these symbols; they may change or be removed in future releases. On
|
||||
// hosts missing libaio the bring-up fails cleanly and only DiskAnn becomes
|
||||
// unavailable — other index types (HNSW / IVF / Flat / Vamana) keep working.
|
||||
|
||||
#if defined(_WIN32) || defined(__CYGWIN__)
|
||||
#ifdef ZVEC_BUILD_SHARED
|
||||
#define ZVEC_PLUGIN_EXPORT __declspec(dllexport)
|
||||
#elif defined(ZVEC_USE_SHARED)
|
||||
#define ZVEC_PLUGIN_EXPORT __declspec(dllimport)
|
||||
#else
|
||||
#define ZVEC_PLUGIN_EXPORT
|
||||
#endif
|
||||
#else
|
||||
#if __GNUC__ >= 4
|
||||
#define ZVEC_PLUGIN_EXPORT __attribute__((visibility("default")))
|
||||
#else
|
||||
#define ZVEC_PLUGIN_EXPORT
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace zvec {
|
||||
|
||||
// Return codes for LoadDiskAnnPlugin().
|
||||
enum DiskAnnPluginStatus {
|
||||
kDiskAnnPluginOk = 0,
|
||||
kDiskAnnPluginUnsupportedPlatform = -1,
|
||||
kDiskAnnPluginLibAioMissing = -2,
|
||||
kDiskAnnPluginDlopenFailed = -3,
|
||||
};
|
||||
|
||||
// Returns true if libaio is present on the host and the minimum set of symbols
|
||||
// required by the DiskAnn runtime (io_setup / io_submit / io_getevents /
|
||||
// io_destroy) can be resolved at runtime.
|
||||
//
|
||||
// Internal probe used by ``LoadDiskAnnPlugin`` before attempting dlopen. Not
|
||||
// part of the user-facing API.
|
||||
ZVEC_PLUGIN_EXPORT bool IsLibAioAvailable();
|
||||
|
||||
// Load the DiskAnn runtime shared library (libzvec_diskann_plugin.so) via
|
||||
// dlopen(). Invoked implicitly by ``DiskAnnIndex::CreateAndInitStreamer`` on
|
||||
// first use; callers should not invoke it directly. The call is idempotent
|
||||
// and returns ``kDiskAnnPluginOk`` when the runtime is already active.
|
||||
//
|
||||
// Search order when ``path`` is empty:
|
||||
// 1. next to the currently running executable (``/proc/self/exe`` on Linux);
|
||||
// 2. next to the hosting shared object (e.g. ``_zvec.cpython-*.so``);
|
||||
// 3. the platform default dynamic-linker search path (RPATH,
|
||||
// ``LD_LIBRARY_PATH``, ``/etc/ld.so.conf``, …).
|
||||
ZVEC_PLUGIN_EXPORT int LoadDiskAnnPlugin(const std::string &path = "");
|
||||
|
||||
// Returns true if the DiskAnn runtime is currently loaded in this process.
|
||||
// Internal diagnostic; not a user-facing API.
|
||||
ZVEC_PLUGIN_EXPORT bool IsDiskAnnPluginLoaded();
|
||||
|
||||
// Unload the DiskAnn runtime. Internal only — unloading a library that has
|
||||
// registered itself into global factory singletons is inherently racy;
|
||||
// callers must guarantee that no DiskAnn objects are still alive and no
|
||||
// background threads are executing DiskAnn code before calling this. Returns
|
||||
// true when a live handle was released.
|
||||
ZVEC_PLUGIN_EXPORT bool UnloadDiskAnnPlugin();
|
||||
|
||||
} // namespace zvec
|
||||
@@ -8,6 +8,14 @@ cc_directories(ivf)
|
||||
cc_directories(hnsw)
|
||||
cc_directories(hnsw_sparse)
|
||||
cc_directories(vamana)
|
||||
if(RABITQ_SUPPORTED)
|
||||
cc_directories(hnsw_rabitq)
|
||||
|
||||
if(DISKANN_SUPPORTED)
|
||||
message(STATUS "build diskann tests")
|
||||
cc_directory(diskann)
|
||||
else()
|
||||
message(STATUS "skip diskann tests (unsupported platform)")
|
||||
endif()
|
||||
|
||||
if(RABITQ_SUPPORTED)
|
||||
cc_directories(hnsw_rabitq)
|
||||
endif()
|
||||
|
||||
@@ -52,8 +52,8 @@ TEST(KmeansCluster, General) {
|
||||
ASSERT_TRUE(!!cluster);
|
||||
|
||||
zvec::ailego::Params params;
|
||||
params.set("proxima.general.cluster.count", 1);
|
||||
params.set("proxima.kmeans.cluster.count", 56);
|
||||
params.set("zvec.general.cluster.count", 1);
|
||||
params.set("zvec.kmeans.cluster.count", 56);
|
||||
|
||||
ASSERT_EQ(0, cluster->init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster->mount(features));
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "cluster/multi_chunk_cluster.h"
|
||||
#include <cmath>
|
||||
#include <random>
|
||||
#include <ailego/algorithm/kmeans.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <zvec/ailego/container/params.h>
|
||||
#include "zvec/core/framework/index_framework.h"
|
||||
|
||||
using namespace zvec::core;
|
||||
using namespace zvec::ailego;
|
||||
using namespace zvec::ailego;
|
||||
TEST(MultiChunkCluster, General) {
|
||||
const uint32_t count = 10000u;
|
||||
const uint32_t dimension = 960u;
|
||||
const uint32_t chunk_count = 480u;
|
||||
const uint32_t cluster_count = 256u;
|
||||
// const uint32_t thread_count = 4;
|
||||
const uint32_t thread_count = 16;
|
||||
|
||||
IndexMeta index_meta;
|
||||
|
||||
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
index_meta.set_metric("SquaredEuclidean", 0, Params());
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> features(
|
||||
new CompactIndexFeatures(index_meta));
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
std::uniform_real_distribution<float> dist(0.0, 5.0);
|
||||
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
std::vector<float> vec(dimension);
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] = dist(gen);
|
||||
}
|
||||
features->emplace(vec.data());
|
||||
}
|
||||
|
||||
// Create a Kmeans cluster
|
||||
MultiChunkCluster cluster = MultiChunkCluster();
|
||||
|
||||
Params params;
|
||||
params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
|
||||
|
||||
ASSERT_EQ(0, cluster.init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster.mount(features));
|
||||
|
||||
IndexCluster::CentroidList centroids;
|
||||
std::vector<uint32_t> labels;
|
||||
|
||||
ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
|
||||
for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
|
||||
size_t idx = chunk * cluster_count + cluster;
|
||||
const auto ¢ = centroids[idx];
|
||||
const auto &vec = cent.vector<float>();
|
||||
|
||||
std::cout << "chunk: " << chunk << ", cluster: " << cluster
|
||||
<< ", dim: " << vec.size() << ", count: " << cent.follows()
|
||||
<< " (" << cent.score() << ") { " << vec[0] << "," << vec[1]
|
||||
<< " }" << std::endl;
|
||||
ASSERT_EQ(0u, cent.similars().size());
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
|
||||
}
|
||||
|
||||
TEST(MultiChunkCluster, TestChunk) {
|
||||
const uint32_t count = 10000u;
|
||||
const uint32_t dimension = 95;
|
||||
const uint32_t chunk_count = 20u;
|
||||
const uint32_t cluster_count = 256u;
|
||||
// const uint32_t thread_count = 4;
|
||||
const uint32_t thread_count = 16;
|
||||
|
||||
IndexMeta index_meta;
|
||||
|
||||
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
index_meta.set_metric("SquaredEuclidean", 0, Params());
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> features(
|
||||
new CompactIndexFeatures(index_meta));
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
std::uniform_real_distribution<float> dist(0.0, 5.0);
|
||||
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
std::vector<float> vec(dimension);
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] = dist(gen);
|
||||
}
|
||||
features->emplace(vec.data());
|
||||
}
|
||||
|
||||
// Create a Kmeans cluster
|
||||
MultiChunkCluster cluster = MultiChunkCluster();
|
||||
|
||||
Params params;
|
||||
params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
|
||||
|
||||
ASSERT_EQ(0, cluster.init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster.mount(features));
|
||||
|
||||
IndexCluster::CentroidList centroids;
|
||||
std::vector<uint32_t> labels;
|
||||
|
||||
ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
|
||||
for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
|
||||
size_t idx = chunk * cluster_count + cluster;
|
||||
const auto ¢ = centroids[idx];
|
||||
const auto &vec = cent.vector<float>();
|
||||
|
||||
std::cout << "chunk: " << chunk << ", cluster: " << cluster
|
||||
<< ", dim: " << vec.size() << ", count: " << cent.follows()
|
||||
<< " (" << cent.score() << ") { " << vec[0] << "," << vec[1]
|
||||
<< " }" << std::endl;
|
||||
ASSERT_EQ(0u, cent.similars().size());
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
|
||||
}
|
||||
|
||||
TEST(MultiChunkCluster, General_InnerProduct) {
|
||||
const uint32_t count = 50000u;
|
||||
const uint32_t dimension = 96u;
|
||||
const uint32_t chunk_count = 12u;
|
||||
const uint32_t cluster_count = 16u;
|
||||
const uint32_t chain_length = 0;
|
||||
const uint32_t thread_count = 16;
|
||||
|
||||
IndexMeta index_meta;
|
||||
|
||||
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
index_meta.set_metric("InnerProduct", 0, Params());
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> features(
|
||||
new CompactIndexFeatures(index_meta));
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
std::uniform_real_distribution<float> dist(-1.0, 1.0);
|
||||
|
||||
// do normalize
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
std::vector<float> vec(dimension);
|
||||
|
||||
float norm = 0;
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] = dist(gen);
|
||||
|
||||
norm += vec[j] * vec[j];
|
||||
}
|
||||
norm = sqrt(norm);
|
||||
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] /= norm;
|
||||
}
|
||||
|
||||
features->emplace(vec.data());
|
||||
}
|
||||
|
||||
// Create a Kmeans cluster
|
||||
MultiChunkCluster cluster = MultiChunkCluster();
|
||||
|
||||
Params params;
|
||||
params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
|
||||
params.set("zvec.cluster.multi_chunk_cluster.markov_chain_length",
|
||||
chain_length);
|
||||
|
||||
ASSERT_EQ(0, cluster.init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster.mount(features));
|
||||
|
||||
IndexCluster::CentroidList centroids;
|
||||
std::vector<uint32_t> labels;
|
||||
|
||||
ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
|
||||
for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
|
||||
size_t idx = chunk * cluster_count + cluster;
|
||||
const auto ¢ = centroids[idx];
|
||||
const auto &vec = cent.vector<float>();
|
||||
|
||||
std::cout << "chunk: " << chunk << ", cluster: " << cluster
|
||||
<< ", dim: " << vec.size() << ", count: " << cent.follows()
|
||||
<< " (" << cent.score() << ") { " << vec[0] << ", " << vec[1]
|
||||
<< ", " << vec[2] << ", ... , " << vec[vec.size() - 2] << ", "
|
||||
<< vec[vec.size() - 1] << " }" << std::endl;
|
||||
ASSERT_EQ(0u, cent.similars().size());
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
|
||||
}
|
||||
@@ -52,8 +52,8 @@ TEST(OptKmeansCluster, General) {
|
||||
ASSERT_TRUE(!!cluster);
|
||||
|
||||
Params params;
|
||||
params.set("proxima.general.cluster.count", 1);
|
||||
params.set("proxima.optkmeans.cluster.count", 56);
|
||||
params.set("zvec.general.cluster.count", 1);
|
||||
params.set("zvec.optkmeans.cluster.count", 56);
|
||||
|
||||
ASSERT_EQ(0, cluster->init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster->mount(features));
|
||||
@@ -191,7 +191,7 @@ TEST(OptKmeansCluster, General) {
|
||||
// << ", " << vec[vec.size() - 1] << " }" << std::endl;
|
||||
// }
|
||||
|
||||
// params.set("proxima.optkmeans.cluster.purge_empty", true);
|
||||
// params.set("zvec.optkmeans.cluster.purge_empty", true);
|
||||
// cluster->update(params);
|
||||
|
||||
// ASSERT_EQ(12u, centroids.size());
|
||||
@@ -254,8 +254,8 @@ TEST(OptKmeansCluster, IN4General) {
|
||||
ASSERT_EQ(0, cluster->init(index_meta_wrong, params));
|
||||
ASSERT_NE(0, cluster->mount(features_wrong));
|
||||
|
||||
params.set("proxima.general.cluster.count", 1);
|
||||
params.set("proxima.optkmeans.cluster.count", 56);
|
||||
params.set("zvec.general.cluster.count", 1);
|
||||
params.set("zvec.optkmeans.cluster.count", 56);
|
||||
|
||||
ASSERT_EQ(0, cluster->init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster->mount(features));
|
||||
@@ -351,11 +351,11 @@ TEST(OptKmeansCluster, IN4Correctness) {
|
||||
ASSERT_TRUE(!!cluster_once);
|
||||
|
||||
Params params_once;
|
||||
params_once.set("proxima.general.cluster.count", 65);
|
||||
params_once.set("proxima.optkmeans.cluster.count", 63);
|
||||
params_once.set("proxima.optkmeans.cluster.max_iterations", 1);
|
||||
params_once.set("zvec.general.cluster.count", 65);
|
||||
params_once.set("zvec.optkmeans.cluster.count", 63);
|
||||
params_once.set("zvec.optkmeans.cluster.max_iterations", 1);
|
||||
// Use KMC2 to init centroids
|
||||
params_once.set("proxima.optkmeans.cluster.markov_chain_length", 20);
|
||||
params_once.set("zvec.optkmeans.cluster.markov_chain_length", 20);
|
||||
|
||||
ASSERT_EQ(0, cluster_once->init(index_meta1, params_once));
|
||||
ASSERT_EQ(0, cluster_once->mount(features1));
|
||||
@@ -374,8 +374,8 @@ TEST(OptKmeansCluster, IN4Correctness) {
|
||||
ASSERT_TRUE(!!cluster_int8);
|
||||
|
||||
Params params_int8;
|
||||
params_int8.set("proxima.general.cluster.count", 65);
|
||||
params_int8.set("proxima.optkmeans.cluster.count", 63);
|
||||
params_int8.set("zvec.general.cluster.count", 65);
|
||||
params_int8.set("zvec.optkmeans.cluster.count", 63);
|
||||
|
||||
ASSERT_EQ(0, cluster_int8->init(index_meta1, params_int8));
|
||||
ASSERT_EQ(0, cluster_int8->mount(features1));
|
||||
@@ -387,8 +387,8 @@ TEST(OptKmeansCluster, IN4Correctness) {
|
||||
ASSERT_TRUE(!!cluster_int4);
|
||||
|
||||
Params params_int4;
|
||||
params_int4.set("proxima.general.cluster.count", 65);
|
||||
params_int4.set("proxima.optkmeans.cluster.count", 63);
|
||||
params_int4.set("zvec.general.cluster.count", 65);
|
||||
params_int4.set("zvec.optkmeans.cluster.count", 63);
|
||||
|
||||
ASSERT_EQ(0, cluster_int4->init(index_meta2, params_int4));
|
||||
ASSERT_EQ(0, cluster_int4->mount(features2));
|
||||
@@ -448,8 +448,8 @@ TEST(OptKmeansCluster, InnerProduct) {
|
||||
ASSERT_TRUE(!!cluster);
|
||||
|
||||
Params params;
|
||||
params.set("proxima.general.cluster.count", 1);
|
||||
params.set("proxima.optkmeans.cluster.count", 56);
|
||||
params.set("zvec.general.cluster.count", 1);
|
||||
params.set("zvec.optkmeans.cluster.count", 56);
|
||||
|
||||
ASSERT_EQ(0, cluster->init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster->mount(features));
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
|
||||
|
||||
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
|
||||
|
||||
foreach(CC_SRCS ${ALL_TEST_SRCS})
|
||||
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
|
||||
cc_gtest(
|
||||
NAME ${CC_TARGET}
|
||||
STRICT
|
||||
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_cluster core_plugin core_knn_diskann
|
||||
SRCS ${CC_SRCS}
|
||||
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm/diskann
|
||||
)
|
||||
endforeach()
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_builder.h"
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <fcntl.h>
|
||||
#include <future>
|
||||
#include <gtest/gtest.h>
|
||||
#include <zvec/ailego/container/vector.h>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_holder.h"
|
||||
|
||||
using namespace zvec::core;
|
||||
using namespace zvec::ailego;
|
||||
using namespace std;
|
||||
|
||||
constexpr size_t static dim = 64;
|
||||
|
||||
class DiskAnnBuilderTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp(void) override;
|
||||
void TearDown(void) override;
|
||||
|
||||
static std::string _dir;
|
||||
static shared_ptr<IndexMeta> _index_meta_ptr;
|
||||
};
|
||||
|
||||
std::string DiskAnnBuilderTest::_dir("DiskAnnBuilderTest");
|
||||
shared_ptr<IndexMeta> DiskAnnBuilderTest::_index_meta_ptr;
|
||||
|
||||
void DiskAnnBuilderTest::SetUp(void) {
|
||||
LoggerBroker::SetLevel(Logger::LEVEL_INFO);
|
||||
|
||||
_index_meta_ptr.reset(new (nothrow)
|
||||
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
|
||||
_index_meta_ptr->set_metric("SquaredEuclidean", 0, Params());
|
||||
}
|
||||
|
||||
void DiskAnnBuilderTest::TearDown(void) {
|
||||
char cmdBuf[100];
|
||||
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
|
||||
system(cmdBuf);
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnBuilderTest, TestGeneral) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 50);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestGeneral";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
}
|
||||
|
||||
// DiskAnn is now exposed implicitly: no caller ever invokes a
|
||||
// ``LoadDiskAnnPlugin`` / ``IsLibAioAvailable`` API (those were removed from
|
||||
// the public surface together with ``zvec.load_diskann_plugin()`` in Python).
|
||||
// The only contract this test validates is the UX guarantee: once the DiskAnn
|
||||
// module has been linked into the hosting binary (here, directly into the
|
||||
// test via the ``core_knn_diskann`` target), its factory entries are
|
||||
// registered automatically and the global ``IndexFactory`` can hand out a
|
||||
// ``DiskAnnBuilder`` without any explicit setup step. On hosts missing
|
||||
// libaio, DiskAnn would fail at the index-creation layer with a clear error
|
||||
// while other index types (HNSW/IVF/Flat/Vamana) remain unaffected; that
|
||||
// runtime branch lives in ``DiskAnnIndex::CreateAndInitStreamer`` and is
|
||||
// covered by the higher-level interface tests.
|
||||
TEST_F(DiskAnnBuilderTest, TestImplicitFactoryRegistration) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr)
|
||||
<< "DiskAnnBuilder factory entry missing: DiskAnn must be available "
|
||||
"without any manual plugin load step.";
|
||||
|
||||
IndexStreamer::Pointer streamer =
|
||||
IndexFactory::CreateStreamer("DiskAnnStreamer");
|
||||
ASSERT_NE(streamer, nullptr)
|
||||
<< "DiskAnnStreamer factory entry missing: DiskAnn must be available "
|
||||
"without any manual plugin load step.";
|
||||
}
|
||||
@@ -0,0 +1,816 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_searcher.h"
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <fcntl.h>
|
||||
#include <ailego/math/distance.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <zvec/ailego/container/vector.h>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_holder.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
using namespace zvec::core;
|
||||
using namespace zvec::ailego;
|
||||
using namespace std;
|
||||
|
||||
constexpr size_t static dim = 64;
|
||||
|
||||
class DiskAnnSearcherTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp(void) override;
|
||||
void TearDown(void) override;
|
||||
|
||||
static std::string _dir;
|
||||
static shared_ptr<IndexMeta> _index_meta_ptr;
|
||||
};
|
||||
|
||||
std::string DiskAnnSearcherTest::_dir("DiskAnnSearcherTest/");
|
||||
shared_ptr<IndexMeta> DiskAnnSearcherTest::_index_meta_ptr;
|
||||
|
||||
void DiskAnnSearcherTest::SetUp(void) {
|
||||
LoggerBroker::SetLevel(Logger::LEVEL_INFO);
|
||||
|
||||
_index_meta_ptr.reset(new (nothrow)
|
||||
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
|
||||
_index_meta_ptr->set_metric("SquaredEuclidean", 0, Params());
|
||||
}
|
||||
|
||||
void DiskAnnSearcherTest::TearDown(void) {
|
||||
char cmdBuf[100];
|
||||
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
|
||||
system(cmdBuf);
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestGeneral) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestGeneral";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
auto ctx = searcher->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
auto linearCtx = searcher->create_context();
|
||||
auto linearByPKeysCtx = searcher->create_context();
|
||||
auto knnCtx = searcher->create_context();
|
||||
|
||||
ASSERT_TRUE(!!linearCtx);
|
||||
ASSERT_TRUE(!!linearByPKeysCtx);
|
||||
ASSERT_TRUE(!!knnCtx);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
size_t topk = 200;
|
||||
uint64_t knnTotalTime = 0;
|
||||
uint64_t linearTotalTime = 0;
|
||||
int totalHits = 0;
|
||||
int totalCnts = 0;
|
||||
int topk1Hits = 0;
|
||||
linearCtx->set_topk(topk);
|
||||
linearByPKeysCtx->set_topk(topk);
|
||||
knnCtx->set_topk(topk);
|
||||
|
||||
// do linear search test
|
||||
{
|
||||
float query[dim];
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
query[i] = 3.1f;
|
||||
}
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(query, qmeta, linearCtx));
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(3UL, linearResult[0].key());
|
||||
ASSERT_EQ(4UL, linearResult[1].key());
|
||||
ASSERT_EQ(2UL, linearResult[2].key());
|
||||
ASSERT_EQ(5UL, linearResult[3].key());
|
||||
ASSERT_EQ(1UL, linearResult[4].key());
|
||||
ASSERT_EQ(6UL, linearResult[5].key());
|
||||
ASSERT_EQ(0UL, linearResult[6].key());
|
||||
ASSERT_EQ(7UL, linearResult[7].key());
|
||||
for (size_t i = 8; i < topk; ++i) {
|
||||
ASSERT_EQ(i, linearResult[i].key());
|
||||
}
|
||||
}
|
||||
|
||||
// do linear search by p_keys test
|
||||
std::vector<std::vector<uint64_t>> p_keys;
|
||||
p_keys.resize(1);
|
||||
p_keys[0] = {8, 9, 10, 11, 3, 2, 1, 0};
|
||||
{
|
||||
float query[dim];
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
query[i] = 3.1f;
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(query, p_keys, qmeta,
|
||||
linearByPKeysCtx));
|
||||
auto &linearByPKeysResult = linearByPKeysCtx->result();
|
||||
ASSERT_EQ(8, linearByPKeysResult.size());
|
||||
ASSERT_EQ(3UL, linearByPKeysResult[0].key());
|
||||
ASSERT_EQ(2UL, linearByPKeysResult[1].key());
|
||||
ASSERT_EQ(1UL, linearByPKeysResult[2].key());
|
||||
ASSERT_EQ(0UL, linearByPKeysResult[3].key());
|
||||
ASSERT_EQ(8UL, linearByPKeysResult[4].key());
|
||||
ASSERT_EQ(9UL, linearByPKeysResult[5].key());
|
||||
ASSERT_EQ(10UL, linearByPKeysResult[6].key());
|
||||
ASSERT_EQ(11UL, linearByPKeysResult[7].key());
|
||||
}
|
||||
|
||||
size_t step = 500;
|
||||
for (size_t i = 0; i < doc_cnt; i += step) {
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
auto t1 = Realtime::MicroSeconds();
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
|
||||
auto t2 = Realtime::MicroSeconds();
|
||||
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
|
||||
auto t3 = Realtime::MicroSeconds();
|
||||
knnTotalTime += t2 - t1;
|
||||
linearTotalTime += t3 - t2;
|
||||
|
||||
auto &knnResult = knnCtx->result();
|
||||
// TODO: check
|
||||
topk1Hits += i == knnResult[0].key();
|
||||
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(topk, linearResult.size());
|
||||
ASSERT_EQ(i, linearResult[0].key());
|
||||
|
||||
for (size_t k = 0; k < topk; ++k) {
|
||||
totalCnts++;
|
||||
for (size_t j = 0; j < topk; ++j) {
|
||||
if (linearResult[j].key() == knnResult[k].key()) {
|
||||
totalHits++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float recall = totalHits * step * step * 1.0f / totalCnts;
|
||||
float topk1Recall = topk1Hits * step * 1.0f / doc_cnt;
|
||||
float cost = linearTotalTime * 1.0f / knnTotalTime;
|
||||
|
||||
EXPECT_GT(recall, 0.90f);
|
||||
EXPECT_GT(topk1Recall, 0.80f);
|
||||
EXPECT_GT(cost, 2.0f);
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestNodeCache) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestNodeCache";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.cache_node_num", 32);
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
auto ctx = searcher->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
auto linearCtx = searcher->create_context();
|
||||
auto linearByPKeysCtx = searcher->create_context();
|
||||
auto knnCtx = searcher->create_context();
|
||||
|
||||
ASSERT_TRUE(!!linearCtx);
|
||||
ASSERT_TRUE(!!linearByPKeysCtx);
|
||||
ASSERT_TRUE(!!knnCtx);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
size_t topk = 200;
|
||||
uint64_t knnTotalTime = 0;
|
||||
uint64_t linearTotalTime = 0;
|
||||
int totalHits = 0;
|
||||
int totalCnts = 0;
|
||||
int topk1Hits = 0;
|
||||
linearCtx->set_topk(topk);
|
||||
linearByPKeysCtx->set_topk(topk);
|
||||
knnCtx->set_topk(topk);
|
||||
|
||||
size_t step = 500;
|
||||
for (size_t i = 0; i < doc_cnt; i += step) {
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
auto t1 = Realtime::MicroSeconds();
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
|
||||
auto t2 = Realtime::MicroSeconds();
|
||||
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
|
||||
auto t3 = Realtime::MicroSeconds();
|
||||
knnTotalTime += t2 - t1;
|
||||
linearTotalTime += t3 - t2;
|
||||
|
||||
auto &knnResult = knnCtx->result();
|
||||
// TODO: check
|
||||
topk1Hits += i == knnResult[0].key();
|
||||
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(topk, linearResult.size());
|
||||
ASSERT_EQ(i, linearResult[0].key());
|
||||
|
||||
for (size_t k = 0; k < topk; ++k) {
|
||||
totalCnts++;
|
||||
for (size_t j = 0; j < topk; ++j) {
|
||||
if (linearResult[j].key() == knnResult[k].key()) {
|
||||
totalHits++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float recall = totalHits * step * step * 1.0f / totalCnts;
|
||||
float topk1Recall = topk1Hits * step * 1.0f / doc_cnt;
|
||||
float cost = linearTotalTime * 1.0f / knnTotalTime;
|
||||
|
||||
EXPECT_GT(recall, 0.90f);
|
||||
EXPECT_GT(topk1Recall, 0.80f);
|
||||
EXPECT_GT(cost, 2.0f);
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestFilter) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestFilter";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.cache_node_num", 32);
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
auto ctx = searcher->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
auto linearCtx = searcher->create_context();
|
||||
auto linearByPKeysCtx = searcher->create_context();
|
||||
auto knnCtx = searcher->create_context();
|
||||
|
||||
ASSERT_TRUE(!!linearCtx);
|
||||
ASSERT_TRUE(!!linearByPKeysCtx);
|
||||
ASSERT_TRUE(!!knnCtx);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
|
||||
size_t topk = 200;
|
||||
linearCtx->set_topk(topk);
|
||||
linearByPKeysCtx->set_topk(topk);
|
||||
knnCtx->set_topk(topk);
|
||||
|
||||
size_t key = 50;
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = key + 0.1f;
|
||||
}
|
||||
|
||||
// no filter
|
||||
{
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
|
||||
|
||||
auto &knnResult = knnCtx->result();
|
||||
ASSERT_EQ(topk, knnResult.size());
|
||||
ASSERT_EQ(50UL, knnResult[0].key());
|
||||
ASSERT_EQ(51UL, knnResult[1].key());
|
||||
ASSERT_EQ(49UL, knnResult[2].key());
|
||||
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
|
||||
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(topk, linearResult.size());
|
||||
ASSERT_EQ(50UL, linearResult[0].key());
|
||||
ASSERT_EQ(51UL, linearResult[1].key());
|
||||
ASSERT_EQ(49UL, linearResult[2].key());
|
||||
}
|
||||
|
||||
// with filter
|
||||
{
|
||||
auto filterFunc = [](uint64_t key) {
|
||||
if (key == 50UL || key == 51UL || key == 49UL) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
|
||||
knnCtx->set_filter(filterFunc);
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
|
||||
|
||||
auto &knnResult = knnCtx->result();
|
||||
ASSERT_EQ(topk, knnResult.size());
|
||||
ASSERT_EQ(52UL, knnResult[0].key());
|
||||
ASSERT_EQ(48UL, knnResult[1].key());
|
||||
ASSERT_EQ(53UL, knnResult[2].key());
|
||||
|
||||
linearCtx->set_filter(filterFunc);
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
|
||||
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(topk, linearResult.size());
|
||||
ASSERT_EQ(52UL, linearResult[0].key());
|
||||
ASSERT_EQ(48UL, linearResult[1].key());
|
||||
ASSERT_EQ(53UL, linearResult[2].key());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestGroup) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i / 10.0;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestGroup";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
auto ctx = searcher->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
size_t group_topk = 20;
|
||||
uint64_t total_time = 0;
|
||||
|
||||
auto groupbyFunc = [](uint64_t key) {
|
||||
uint32_t group_id = key / 10 % 10;
|
||||
|
||||
// std::cout << "key: " << key << ", group id: " << group_id << std::endl;
|
||||
|
||||
return std::string("g_") + std::to_string(group_id);
|
||||
};
|
||||
|
||||
size_t group_num = 5;
|
||||
|
||||
ctx->set_group_params(group_num, group_topk);
|
||||
ctx->set_group_by(groupbyFunc);
|
||||
|
||||
size_t query_value = doc_cnt / 2;
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = query_value / 10 + 0.1f;
|
||||
}
|
||||
|
||||
auto t1 = Realtime::MicroSeconds();
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
|
||||
auto t2 = Realtime::MicroSeconds();
|
||||
|
||||
total_time += t2 - t1;
|
||||
|
||||
auto &group_result = ctx->group_result();
|
||||
|
||||
for (uint32_t i = 0; i < group_result.size(); ++i) {
|
||||
const std::string &group_id = group_result[i].group_id();
|
||||
auto &result = group_result[i].docs();
|
||||
|
||||
ASSERT_GT(result.size(), 0);
|
||||
std::cout << "Group ID: " << group_id << std::endl;
|
||||
|
||||
for (uint32_t j = 0; j < result.size(); ++j) {
|
||||
std::cout << "\tKey: " << result[j].key() << std::fixed
|
||||
<< std::setprecision(3) << ", Score: " << result[j].score()
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
// do linear search by p_keys test
|
||||
auto groupbyFuncLinear = [](uint64_t key) {
|
||||
uint32_t group_id = key % 10;
|
||||
|
||||
return std::string("g_") + std::to_string(group_id);
|
||||
};
|
||||
|
||||
auto linear_pk_ctx = searcher->create_context();
|
||||
|
||||
linear_pk_ctx->set_group_params(group_num, group_topk);
|
||||
linear_pk_ctx->set_group_by(groupbyFuncLinear);
|
||||
|
||||
std::vector<std::vector<uint64_t>> p_keys;
|
||||
p_keys.resize(1);
|
||||
p_keys[0] = {4, 3, 2, 1, 5, 6, 7, 8, 9, 10};
|
||||
|
||||
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(vec.data(), p_keys, qmeta,
|
||||
linear_pk_ctx));
|
||||
auto &linear_by_pkeys_group_result = linear_pk_ctx->group_result();
|
||||
ASSERT_EQ(linear_by_pkeys_group_result.size(), group_num);
|
||||
|
||||
for (uint32_t i = 0; i < linear_by_pkeys_group_result.size(); ++i) {
|
||||
const std::string &group_id = linear_by_pkeys_group_result[i].group_id();
|
||||
auto &result = linear_by_pkeys_group_result[i].docs();
|
||||
|
||||
ASSERT_GT(result.size(), 0);
|
||||
std::cout << "Group ID: " << group_id << std::endl;
|
||||
|
||||
for (uint32_t j = 0; j < result.size(); ++j) {
|
||||
std::cout << "\tKey: " << result[j].key() << std::fixed
|
||||
<< std::setprecision(3) << ", Score: " << result[j].score()
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
ASSERT_EQ(10 - i, result[0].key());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestFetchVector) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestFetchVector";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
|
||||
size_t query_cnt = 20U;
|
||||
auto linearCtx = searcher->create_context();
|
||||
auto knnCtx = searcher->create_context();
|
||||
auto linearByPKeysCtx = searcher->create_context();
|
||||
knnCtx->set_fetch_vector(true);
|
||||
|
||||
for (size_t i = 0; i < doc_cnt; i += doc_cnt / 10) {
|
||||
std::string vec_value;
|
||||
ASSERT_EQ(0, searcher->get_vector(i, linearCtx, vec_value));
|
||||
|
||||
float vector_value = *(const float *)(vec_value.data());
|
||||
ASSERT_EQ(vector_value, i);
|
||||
}
|
||||
|
||||
size_t topk = 200;
|
||||
linearCtx->set_topk(topk);
|
||||
knnCtx->set_topk(topk);
|
||||
uint64_t knnTotalTime = 0;
|
||||
uint64_t linearTotalTime = 0;
|
||||
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t i = 0; i < query_cnt; i++) {
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
|
||||
auto t1 = Realtime::MicroSeconds();
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
|
||||
auto t2 = Realtime::MicroSeconds();
|
||||
ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
|
||||
auto t3 = Realtime::MicroSeconds();
|
||||
knnTotalTime += t2 - t1;
|
||||
linearTotalTime += t3 - t2;
|
||||
|
||||
auto &knnResult = knnCtx->result();
|
||||
ASSERT_EQ(topk, knnResult.size());
|
||||
|
||||
auto &linearResult = linearCtx->result();
|
||||
ASSERT_EQ(topk, linearResult.size());
|
||||
ASSERT_EQ(i, linearResult[0].key());
|
||||
|
||||
ASSERT_NE(knnResult[0].vector_string(), "");
|
||||
float vector_value = *((float *)(knnResult[0].vector_string().data()));
|
||||
ASSERT_EQ(vector_value, i);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DiskAnnSearcherTest, TestRnnSearch) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
|
||||
auto holder =
|
||||
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
|
||||
size_t doc_cnt = 10000UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
Params params;
|
||||
|
||||
params.set("zvec.diskann.builder.max_degree", 32);
|
||||
params.set("zvec.diskann.builder.list_size", 300);
|
||||
params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
|
||||
params.set("zvec.diskann.builder.threads", 4);
|
||||
|
||||
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
|
||||
|
||||
ASSERT_EQ(0, builder->train(holder));
|
||||
|
||||
ASSERT_EQ(0, builder->build(holder));
|
||||
|
||||
auto dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ASSERT_NE(dumper, nullptr);
|
||||
|
||||
string path = _dir + "/TestRnnSearch";
|
||||
ASSERT_EQ(0, dumper->create(path));
|
||||
ASSERT_EQ(0, builder->dump(dumper));
|
||||
ASSERT_EQ(0, dumper->close());
|
||||
|
||||
auto &stats = builder->stats();
|
||||
ASSERT_EQ(doc_cnt, stats.trained_count());
|
||||
ASSERT_EQ(doc_cnt, stats.built_count());
|
||||
ASSERT_EQ(doc_cnt, stats.dumped_count());
|
||||
ASSERT_EQ(0UL, stats.discarded_count());
|
||||
ASSERT_GT(stats.trained_costtime(), 0UL);
|
||||
ASSERT_GT(stats.built_costtime(), 0UL);
|
||||
|
||||
// test searcher
|
||||
IndexSearcher::Pointer searcher =
|
||||
IndexFactory::CreateSearcher("DiskAnnSearcher");
|
||||
ASSERT_TRUE(searcher != nullptr);
|
||||
|
||||
Params search_params;
|
||||
search_params.set("zvec.diskann.searcher.list_size", 500);
|
||||
|
||||
ASSERT_EQ(0, searcher->init(search_params));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("FileReadStorage");
|
||||
ASSERT_EQ(0, storage->open(path, false));
|
||||
ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
|
||||
|
||||
auto ctx = searcher->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = 0.0;
|
||||
}
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
|
||||
size_t topk = 50;
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &results = ctx->result();
|
||||
ASSERT_EQ(topk, results.size());
|
||||
|
||||
float radius = results[topk / 2].score();
|
||||
ctx->set_threshold(radius);
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
|
||||
ASSERT_GT(topk, results.size());
|
||||
for (size_t k = 0; k < results.size(); ++k) {
|
||||
ASSERT_GE(radius, results[k].score());
|
||||
}
|
||||
|
||||
// Test Reset Threshold
|
||||
ctx->reset_threshold();
|
||||
ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
|
||||
ASSERT_EQ(topk, results.size());
|
||||
ASSERT_LT(radius, results[topk - 1].score());
|
||||
}
|
||||
@@ -132,7 +132,7 @@ TEST_F(IVFBuilderTest, TestTrainWithTrainer2Level) {
|
||||
prepare_index_holder(0, 1000);
|
||||
|
||||
Params params;
|
||||
params.set("proxima.stratified.trainer.cluster_count", "4*2");
|
||||
params.set("zvec.stratified.trainer.cluster_count", "4*2");
|
||||
ASSERT_EQ(0, trainer->init(index_meta_, params));
|
||||
ASSERT_EQ(0, trainer->train(threads_, holder_));
|
||||
|
||||
@@ -165,7 +165,7 @@ TEST_F(IVFBuilderTest, TestTrainWithTrainer1Level) {
|
||||
prepare_index_holder(0, 1000);
|
||||
|
||||
Params params1;
|
||||
params1.set("proxima.stratified.trainer.cluster_count", "4");
|
||||
params1.set("zvec.stratified.trainer.cluster_count", "4");
|
||||
ASSERT_EQ(0, trainer->init(index_meta_, params1));
|
||||
ASSERT_EQ(0, trainer->train(threads_, holder_));
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
|
||||
core_knn_hnsw_rabitq
|
||||
core_knn_hnsw_sparse
|
||||
core_knn_ivf
|
||||
core_knn_diskann
|
||||
core_mix_reducer
|
||||
core_metric
|
||||
core_utility
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user