feat: add hnsw-rabitq support (#69)

* feat: add hnsw-rabitq

* undefine transform

* support config rotator type

* support sample_count

* fix ut

* refactor: update interface

* refactor: update interface

* update rabitq index params

* fix interface update

* fix searcher test

* fix streamer test

* streamer support bf and add more ut

* rm env check

* add collection ut

* add schema check

* add rabitq query param binding

* add integration test

* fix local_builder

* cleanup dist calculator

* add RaBitQ-Library submodule

* cleanup rabitq converter/reformer

* add files

* disable build on mac

* disable Feature_Optimize_HNSW_RABITQ

* disable python/tests/test_collection_hnsw_rabitq.py:13

* check avx2/avx512

* fix refine

* fix mac ci

* add dimension check

* fix mac ci

* fix ci

* add missing lib

* fix centroids selection for Cosine/InnerProduct

* rename hnsw-rabitq to hnsw_rabitq

* address comments

* fix compile

* fix rabitqlib name

* fix mac compile

* check avx2/avx512 support

* runtime check again compile-time

* check AUTO_DETECT_ARCH

* rm debug log

* fix

* rabitq use avx2 by default

* add missing file

* fix search_bf/group_by dist

* address comments

* fix typo

* address comments

* rabitq support disable id_map

* address comments
This commit is contained in:
egolearner
2026-03-19 17:21:42 +08:00
committed by GitHub
parent 432de52c5f
commit e5ba11b6fe
96 changed files with 14500 additions and 92 deletions
+3
View File
@@ -40,3 +40,6 @@
path = thirdparty/magic_enum/magic_enum-0.9.7
url = https://github.com/Neargye/magic_enum.git
ignore = all
[submodule "thirdparty/RaBitQ-Library/RaBitQ-Library-0.1"]
path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.1
url = https://github.com/VectorDB-NTU/RaBitQ-Library.git
+36 -1
View File
@@ -18,6 +18,12 @@ endif()
message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
if(NOT ANDROID AND AUTO_DETECT_ARCH AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64")
setup_compiler_march_for_x86(MATH_MARCH_FLAG_SSE MATH_MARCH_FLAG_AVX2 MATH_MARCH_FLAG_AVX512)
message(STATUS "best compiler march, sse: " ${MATH_MARCH_FLAG_SSE} ", avx2: " ${MATH_MARCH_FLAG_AVX2} ", avx512: " ${MATH_MARCH_FLAG_AVX512})
endif()
include_directories(${PROJECT_ROOT_DIR}/src/include)
include_directories(${PROJECT_ROOT_DIR}/src)
@@ -28,6 +34,35 @@ message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}")
option(BUILD_TOOLS "Build tools" ON)
message(STATUS "BUILD_TOOLS:${BUILD_TOOLS}")
option(RABITQ_ENABLE_AVX512 "Compile RaBitQ with AVX-512 support" OFF)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64" AND NOT ANDROID)
include(CheckCCompilerFlag)
check_c_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
check_c_compiler_flag("-mavx512f -mavx512bw -mavx512vl" COMPILER_SUPPORTS_AVX512)
if(COMPILER_SUPPORTS_AVX2 OR COMPILER_SUPPORTS_AVX512)
set(RABITQ_SUPPORTED ON)
add_definitions(-DRABITQ_SUPPORTED=1)
if(RABITQ_ENABLE_AVX512 AND COMPILER_SUPPORTS_AVX512)
add_definitions(-DRABITQ_COMPILED_AVX512=1)
set(RABITQ_ARCH_FLAG "${MATH_MARCH_FLAG_AVX512}")
else()
set(RABITQ_ARCH_FLAG "${MATH_MARCH_FLAG_AVX2}")
endif()
else()
set(RABITQ_SUPPORTED OFF)
add_definitions(-DRABITQ_SUPPORTED=0)
message(STATUS "RaBitQ support disabled - compiler does not support AVX2 or AVX-512")
endif()
else()
set(RABITQ_SUPPORTED OFF)
add_definitions(-DRABITQ_SUPPORTED=0)
message(STATUS "RaBitQ support disabled - only supported on Linux x86_64")
endif()
message(STATUS "RABITQ_ARCH_FLAG: ${RABITQ_ARCH_FLAG}")
option(USE_OSS_MIRROR "Use OSS mirror for faster third-party downloads" OFF)
if(DEFINED ENV{USE_OSS_MIRROR} AND NOT "$ENV{USE_OSS_MIRROR}" STREQUAL "")
set(USE_OSS_MIRROR "$ENV{USE_OSS_MIRROR}" CACHE BOOL "Use OSS mirror for faster third-party downloads" FORCE)
@@ -64,4 +99,4 @@ if(BUILD_PYTHON_BINDINGS)
message(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
endif()
endif()
+9 -8
View File
@@ -110,10 +110,10 @@ Doc create_doc(const uint64_t doc_id, const CollectionSchema &schema,
std::vector<double>(field->dimension(), double(doc_id + 0.1)));
break;
case DataType::VECTOR_FP16:
new_doc.set<std::vector<float16_t>>(
field->name(), std::vector<float16_t>(
field->dimension(),
static_cast<float16_t>(float(doc_id + 0.1))));
new_doc.set<std::vector<zvec::float16_t>>(
field->name(), std::vector<zvec::float16_t>(
field->dimension(), static_cast<zvec::float16_t>(
float(doc_id + 0.1))));
break;
case DataType::VECTOR_INT8:
new_doc.set<std::vector<int8_t>>(
@@ -127,16 +127,17 @@ Doc create_doc(const uint64_t doc_id, const CollectionSchema &schema,
break;
case DataType::SPARSE_VECTOR_FP16: {
std::vector<uint32_t> indices;
std::vector<float16_t> values;
std::vector<zvec::float16_t> values;
for (uint32_t i = 0; i < 100; i++) {
indices.push_back(i);
values.push_back(float16_t(float(doc_id + 0.1)));
values.push_back(zvec::float16_t(float(doc_id + 0.1)));
}
std::pair<std::vector<uint32_t>, std::vector<float16_t>>
std::pair<std::vector<uint32_t>, std::vector<zvec::float16_t>>
sparse_float_vec;
sparse_float_vec.first = indices;
sparse_float_vec.second = values;
new_doc.set<std::pair<std::vector<uint32_t>, std::vector<float16_t>>>(
new_doc.set<
std::pair<std::vector<uint32_t>, std::vector<zvec::float16_t>>>(
field->name(), sparse_float_vec);
break;
}
+574
View File
@@ -0,0 +1,574 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import platform
import sys
import pytest
import math
import zvec
pytestmark = pytest.mark.skipif(
not (sys.platform == "linux" and platform.machine() in ("x86_64", "AMD64")),
reason="HNSW RaBitQ only supported on Linux x86_64",
)
from zvec import (
Collection,
CollectionOption,
DataType,
Doc,
FieldSchema,
HnswRabitqIndexParam,
HnswRabitqQueryParam,
MetricType,
VectorSchema,
VectorQuery,
)
# ==================== Fixtures ====================
@pytest.fixture(scope="session")
def hnsw_rabitq_collection_schema():
"""Create a collection schema with HNSW RaBitQ index."""
return zvec.CollectionSchema(
name="test_hnsw_rabitq_collection",
fields=[
FieldSchema("id", DataType.INT64, nullable=False),
FieldSchema("name", DataType.STRING, nullable=False),
],
vectors=[
VectorSchema(
"embedding",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswRabitqIndexParam(
metric_type=MetricType.L2,
m=16,
ef_construction=200,
total_bits=7,
num_clusters=64,
),
),
],
)
@pytest.fixture(scope="session")
def collection_option():
"""Create collection options."""
return CollectionOption(read_only=False, enable_mmap=True)
@pytest.fixture
def single_doc():
"""Create a single document for testing."""
return Doc(
id="0",
fields={"id": 0, "name": "test_doc_0"},
vectors={"embedding": [0.1 + i * 0.01 for i in range(128)]},
)
@pytest.fixture
def multiple_docs():
"""Create multiple documents for testing."""
return [
Doc(
id=f"{i}",
fields={"id": i, "name": f"test_doc_{i}"},
vectors={"embedding": [i * 0.1 + j * 0.01 for j in range(128)]},
)
for i in range(1, 101)
]
@pytest.fixture(scope="function")
def hnsw_rabitq_collection(
tmp_path_factory, hnsw_rabitq_collection_schema, collection_option
) -> Collection:
"""
Function-scoped fixture: creates and opens a collection with HNSW RaBitQ index.
"""
temp_dir = tmp_path_factory.mktemp("zvec_hnsw_rabitq")
collection_path = temp_dir / "test_hnsw_rabitq_collection"
coll = zvec.create_and_open(
path=str(collection_path),
schema=hnsw_rabitq_collection_schema,
option=collection_option,
)
assert coll is not None, "Failed to create and open HNSW RaBitQ collection"
assert coll.path == str(collection_path)
assert coll.schema.name == hnsw_rabitq_collection_schema.name
try:
yield coll
finally:
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
@pytest.fixture
def collection_with_single_doc(
hnsw_rabitq_collection: Collection, single_doc: Doc
) -> Collection:
"""Setup: insert single doc into collection."""
assert hnsw_rabitq_collection.stats.doc_count == 0
result = hnsw_rabitq_collection.insert(single_doc)
assert bool(result)
assert result.ok()
assert hnsw_rabitq_collection.stats.doc_count == 1
yield hnsw_rabitq_collection
# Teardown: delete single doc
hnsw_rabitq_collection.delete(single_doc.id)
assert hnsw_rabitq_collection.stats.doc_count == 0
@pytest.fixture
def collection_with_multiple_docs(
hnsw_rabitq_collection: Collection, multiple_docs: list[Doc]
) -> Collection:
"""Setup: insert multiple docs into collection."""
assert hnsw_rabitq_collection.stats.doc_count == 0
result = hnsw_rabitq_collection.insert(multiple_docs)
assert len(result) == len(multiple_docs)
for item in result:
assert item.ok()
assert hnsw_rabitq_collection.stats.doc_count == len(multiple_docs)
yield hnsw_rabitq_collection
# Teardown: delete multiple docs
hnsw_rabitq_collection.delete([doc.id for doc in multiple_docs])
# ==================== Tests ====================
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionCreation:
"""Test HNSW RaBitQ collection creation and schema validation."""
def test_collection_creation(
self, hnsw_rabitq_collection: Collection, hnsw_rabitq_collection_schema
):
"""Test that collection is created with correct schema."""
assert hnsw_rabitq_collection is not None
assert hnsw_rabitq_collection.schema.name == hnsw_rabitq_collection_schema.name
assert len(hnsw_rabitq_collection.schema.fields) == len(
hnsw_rabitq_collection_schema.fields
)
assert len(hnsw_rabitq_collection.schema.vectors) == len(
hnsw_rabitq_collection_schema.vectors
)
def test_vector_schema_validation(self, hnsw_rabitq_collection: Collection):
"""Test that vector schema has correct HNSW RaBitQ configuration."""
vector_schema = hnsw_rabitq_collection.schema.vector("embedding")
assert vector_schema is not None
assert vector_schema.name == "embedding"
assert vector_schema.data_type == DataType.VECTOR_FP32
assert vector_schema.dimension == 128
index_param = vector_schema.index_param
assert index_param is not None
assert index_param.metric_type == MetricType.L2
assert index_param.m == 16
assert index_param.ef_construction == 200
assert index_param.total_bits == 7
assert index_param.num_clusters == 64
def test_collection_stats(self, hnsw_rabitq_collection: Collection):
"""Test initial collection statistics."""
stats = hnsw_rabitq_collection.stats
assert stats is not None
assert stats.doc_count == 0
assert len(stats.index_completeness) == 1
assert stats.index_completeness["embedding"] == 1
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionInsert:
"""Test document insertion into HNSW RaBitQ collection."""
def test_insert_single_doc(
self, hnsw_rabitq_collection: Collection, single_doc: Doc
):
"""Test inserting a single document."""
result = hnsw_rabitq_collection.insert(single_doc)
assert bool(result)
assert result.ok()
stats = hnsw_rabitq_collection.stats
assert stats is not None
assert stats.doc_count == 1
def test_insert_multiple_docs(
self, hnsw_rabitq_collection: Collection, multiple_docs: list[Doc]
):
"""Test inserting multiple documents."""
result = hnsw_rabitq_collection.insert(multiple_docs)
assert len(result) == len(multiple_docs)
for item in result:
assert item.ok()
stats = hnsw_rabitq_collection.stats
assert stats is not None
assert stats.doc_count == len(multiple_docs)
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionFetch:
"""Test document fetching from HNSW RaBitQ collection."""
def test_fetch_single_doc(
self, collection_with_single_doc: Collection, single_doc: Doc
):
"""Test fetching a single document by ID."""
result = collection_with_single_doc.fetch(ids=[single_doc.id])
assert bool(result)
assert single_doc.id in result.keys()
doc = result[single_doc.id]
assert doc is not None
assert doc.id == single_doc.id
assert doc.field("id") == single_doc.field("id")
assert doc.field("name") == single_doc.field("name")
def test_fetch_multiple_docs(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test fetching multiple documents by IDs."""
ids = [doc.id for doc in multiple_docs[:10]]
result = collection_with_multiple_docs.fetch(ids=ids)
assert bool(result)
assert len(result) == len(ids)
for doc_id in ids:
assert doc_id in result
doc = result[doc_id]
assert doc is not None
assert doc.id == doc_id
def test_fetch_nonexistent_doc(self, collection_with_single_doc: Collection):
"""Test fetching a non-existent document."""
result = collection_with_single_doc.fetch(ids=["nonexistent_id"])
assert len(result) == 0
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionQuery:
"""Test vector search queries on HNSW RaBitQ collection."""
def test_query_by_vector(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying by vector with HNSW RaBitQ index."""
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
result = collection_with_multiple_docs.query(vectors=query, topk=10)
assert len(result) > 0
assert len(result) <= 10
# First result should be the query document itself (or very close)
first_doc = result[0]
assert first_doc is not None
assert first_doc.id is not None
def test_query_by_id(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying by document ID with HNSW RaBitQ index."""
query = VectorQuery(
field_name="embedding",
id=multiple_docs[0].id,
param=HnswRabitqQueryParam(ef=300),
)
result = collection_with_multiple_docs.query(vectors=query, topk=10)
assert len(result) > 0
assert len(result) <= 10
def test_query_with_different_ef_values(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying with different ef parameter values."""
query_vector = multiple_docs[0].vector("embedding")
# Test with ef=100
query_100 = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=100),
)
result_100 = collection_with_multiple_docs.query(vectors=query_100, topk=10)
assert len(result_100) > 0
# Test with ef=500
query_500 = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=500),
)
result_500 = collection_with_multiple_docs.query(vectors=query_500, topk=10)
assert len(result_500) > 0
def test_query_with_topk(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying with different topk values."""
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
# Test topk=5
result_5 = collection_with_multiple_docs.query(vectors=query, topk=5)
assert len(result_5) <= 5
# Test topk=20
result_20 = collection_with_multiple_docs.query(vectors=query, topk=20)
assert len(result_20) <= 20
def test_query_with_filter(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying with filter conditions."""
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
# Query with id filter
result = collection_with_multiple_docs.query(
vectors=query, topk=10, filter="id < 50"
)
assert len(result) > 0
for doc in result:
assert doc.field("id") < 50
def test_query_with_output_fields(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying with specific output fields."""
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
result = collection_with_multiple_docs.query(
vectors=query, topk=10, output_fields=["id", "name"]
)
assert len(result) > 0
first_doc = result[0]
assert "id" in first_doc.field_names()
assert "name" in first_doc.field_names()
def test_query_with_include_vector(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test querying with vector data included in results."""
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
result = collection_with_multiple_docs.query(
vectors=query, topk=10, include_vector=True
)
assert len(result) > 0
first_doc = result[0]
assert first_doc.vector("embedding") is not None
assert len(first_doc.vector("embedding")) == 128
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionUpdate:
"""Test document update in HNSW RaBitQ collection."""
def test_update_doc_fields(
self, collection_with_single_doc: Collection, single_doc: Doc
):
"""Test updating document fields."""
updated_doc = Doc(
id=single_doc.id,
fields={"id": single_doc.field("id"), "name": "updated_name"},
)
result = collection_with_single_doc.update(updated_doc)
assert bool(result)
assert result.ok()
# Verify update
fetched = collection_with_single_doc.fetch(ids=[single_doc.id])
assert single_doc.id in fetched
doc = fetched[single_doc.id]
assert doc.field("name") == "updated_name"
def test_update_doc_vector(
self, collection_with_single_doc: Collection, single_doc: Doc
):
"""Test updating document vector."""
new_vector = [0.5 + i * 0.01 for i in range(128)]
updated_doc = Doc(
id=single_doc.id,
vectors={"embedding": new_vector},
)
result = collection_with_single_doc.update(updated_doc)
assert bool(result)
assert result.ok()
# Verify update
fetched = collection_with_single_doc.fetch(
ids=[single_doc.id],
)
assert single_doc.id in fetched
doc = fetched[single_doc.id]
assert doc.vector("embedding") is not None
embedding = doc.vector("embedding")
assert len(embedding) == 128
# Verify vector values are approximately equal (float comparison)
for i in range(128):
assert math.isclose(embedding[i], new_vector[i], rel_tol=1e-5)
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionDelete:
"""Test document deletion from HNSW RaBitQ collection."""
def test_delete_single_doc(
self, collection_with_single_doc: Collection, single_doc: Doc
):
"""Test deleting a single document."""
result = collection_with_single_doc.delete(single_doc.id)
assert bool(result)
assert result.ok()
stats = collection_with_single_doc.stats
assert stats.doc_count == 0
def test_delete_multiple_docs(
self, collection_with_multiple_docs: Collection, multiple_docs: list[Doc]
):
"""Test deleting multiple documents."""
ids_to_delete = [doc.id for doc in multiple_docs[:10]]
result = collection_with_multiple_docs.delete(ids_to_delete)
assert len(result) == len(ids_to_delete)
for item in result:
assert item.ok()
stats = collection_with_multiple_docs.stats
assert stats.doc_count == len(multiple_docs) - len(ids_to_delete)
@pytest.mark.usefixtures("hnsw_rabitq_collection")
class TestHnswRabitqCollectionOptimizeAndReopen:
"""Test collection optimize and reopen functionality."""
def test_optimize_close_reopen_and_query(
self,
tmp_path_factory,
hnsw_rabitq_collection_schema,
collection_option,
multiple_docs: list[Doc],
):
"""Test inserting 100 docs, optimize, close, reopen and query."""
# Create collection and insert 100 documents
temp_dir = tmp_path_factory.mktemp("zvec_hnsw_rabitq_optimize")
collection_path = temp_dir / "test_optimize_collection"
coll = zvec.create_and_open(
path=str(collection_path),
schema=hnsw_rabitq_collection_schema,
option=collection_option,
)
assert coll is not None
assert coll.stats.doc_count == 0
# Insert 100 documents
result = coll.insert(multiple_docs)
assert len(result) == len(multiple_docs)
for item in result:
assert item.ok()
assert coll.stats.doc_count == len(multiple_docs)
# Call optimize
from zvec import OptimizeOption
coll.optimize(option=OptimizeOption())
# Verify data is still accessible after optimize
query_vector = multiple_docs[0].vector("embedding")
query = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
result_before_close = coll.query(vectors=query, topk=10)
assert len(result_before_close) > 0
# Close collection (destroy will close it)
collection_path_str = str(collection_path)
del coll
# Reopen collection
reopened_coll = zvec.open(path=collection_path_str, option=collection_option)
assert reopened_coll is not None
assert reopened_coll.stats.doc_count == len(multiple_docs)
# Execute query on reopened collection
query_after_reopen = VectorQuery(
field_name="embedding",
vector=query_vector,
param=HnswRabitqQueryParam(ef=300),
)
result_after_reopen = reopened_coll.query(vectors=query_after_reopen, topk=10)
assert len(result_after_reopen) > 0
assert len(result_after_reopen) <= 10
# Verify query results are valid
first_doc = result_after_reopen[0]
assert first_doc is not None
assert first_doc.id is not None
assert first_doc.field("id") is not None
assert first_doc.field("name") is not None
# Cleanup
reopened_coll.destroy()
+4
View File
@@ -63,6 +63,8 @@ from .model.param import (
FlatIndexParam,
HnswIndexParam,
HnswQueryParam,
HnswRabitqIndexParam,
HnswRabitqQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
@@ -109,6 +111,8 @@ __all__ = [
"VectorQuery",
"InvertIndexParam",
"HnswIndexParam",
"HnswRabitqIndexParam",
"HnswRabitqQueryParam",
"FlatIndexParam",
"IVFIndexParam",
"CollectionOption",
+4
View File
@@ -20,6 +20,8 @@ from _zvec.param import (
FlatIndexParam,
HnswIndexParam,
HnswQueryParam,
HnswRabitqIndexParam,
HnswRabitqQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
@@ -34,6 +36,8 @@ __all__ = [
"FlatIndexParam",
"HnswIndexParam",
"HnswQueryParam",
"HnswRabitqIndexParam",
"HnswRabitqQueryParam",
"IVFIndexParam",
"IVFQueryParam",
"IndexOption",
+131
View File
@@ -16,6 +16,8 @@ __all__: list[str] = [
"FlatIndexParam",
"HnswIndexParam",
"HnswQueryParam",
"HnswRabitqIndexParam",
"HnswRabitqQueryParam",
"IVFIndexParam",
"IVFQueryParam",
"IndexOption",
@@ -285,6 +287,135 @@ class HnswQueryParam(QueryParam):
int: Size of the dynamic candidate list during HNSW search.
"""
class HnswRabitqIndexParam(VectorIndexParam):
"""
Parameters for configuring an HNSW (Hierarchical Navigable Small World) index with RabitQ quantization.
HNSW is a graph-based approximate nearest neighbor search index. RabitQ is a
quantization method that provides high compression with minimal accuracy loss.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
total_bits (int): Total bits for RabitQ quantization. Default is 7.
num_clusters (int): Number of clusters for RabitQ. Default is 16.
m (int): Number of bi-directional links created for every new element
during construction. Higher values improve accuracy but increase
memory usage and construction time. Default is 50.
ef_construction (int): Size of the dynamic candidate list for nearest
neighbors during index construction. Larger values yield better
graph quality at the cost of slower build time. Default is 500.
sample_count (int): Sample count for RabitQ training. Default is 0.
Examples:
>>> from zvec.typing import MetricType
>>> params = HnswRabitqIndexParam(
... metric_type=MetricType.COSINE,
... total_bits=8,
... num_clusters=256,
... m=16,
... ef_construction=200,
... sample_count=10000
... )
>>> print(params)
{'metric_type': 'COSINE', 'total_bits': 8, 'num_clusters': 256, 'm': 16, 'ef_construction': 200, 'sample_count': 10000}
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
metric_type: _zvec.typing.MetricType = ...,
total_bits: typing.SupportsInt = 7,
num_clusters: typing.SupportsInt = 16,
m: typing.SupportsInt = 50,
ef_construction: typing.SupportsInt = 500,
sample_count: typing.SupportsInt = 0,
) -> None: ...
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def ef_construction(self) -> int:
"""
int: Candidate list size during index construction.
"""
@property
def m(self) -> int:
"""
int: Maximum number of neighbors per node.
"""
@property
def total_bits(self) -> int:
"""
int: Total bits for RabitQ quantization.
"""
@property
def num_clusters(self) -> int:
"""
int: Number of clusters for RabitQ.
"""
@property
def sample_count(self) -> int:
"""
int: Sample count for RabitQ training.
"""
class HnswRabitqQueryParam(QueryParam):
"""
Query parameters for HNSW index with RabitQ quantization.
Controls the trade-off between search speed and accuracy via the `ef` parameter.
Attributes:
type (IndexType): Always ``IndexType.HNSW_RABITQ``.
ef (int): Size of the dynamic candidate list during search.
Larger values improve recall but slow down search.
Default is 300.
radius (float): Search radius for range queries. Default is 0.0.
is_linear (bool): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
Examples:
>>> params = HnswRabitqQueryParam(ef=300)
>>> print(params.ef)
300
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
ef: typing.SupportsInt = 300,
radius: typing.SupportsFloat = 0.0,
is_linear: bool = False,
is_using_refiner: bool = False,
) -> None:
"""
Constructs an HnswRabitqQueryParam instance.
Args:
ef (int, optional): Search-time candidate list size.
Higher values improve accuracy. Defaults to 300.
radius (float, optional): Search radius for range queries. Default is 0.0.
is_linear (bool, optional): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def ef(self) -> int:
"""
int: Size of the dynamic candidate list during HNSW search.
"""
class IVFIndexParam(VectorIndexParam):
"""
+2
View File
@@ -24,6 +24,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
$<TARGET_FILE:core_knn_flat_static>
$<TARGET_FILE:core_knn_flat_sparse_static>
$<TARGET_FILE:core_knn_hnsw_static>
$<TARGET_FILE:core_knn_hnsw_rabitq_static>
$<TARGET_FILE:core_knn_hnsw_sparse_static>
$<TARGET_FILE:core_knn_ivf_static>
$<TARGET_FILE:core_knn_cluster_static>
@@ -42,6 +43,7 @@ elseif (APPLE)
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_sparse_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_rabitq_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_sparse_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_ivf_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_cluster_static>
+176 -2
View File
@@ -31,6 +31,8 @@ static std::string index_type_to_string(const IndexType type) {
return "IVF";
case IndexType::HNSW:
return "HNSW";
case IndexType::HNSW_RABITQ:
return "HNSW_RABITQ";
default:
return "UNDEFINED";
}
@@ -59,6 +61,8 @@ static std::string quantize_type_to_string(const QuantizeType type) {
return "INT4";
case QuantizeType::FP16:
return "FP16";
case QuantizeType::RABITQ:
return "RABITQ";
default:
return "UNDEFINED";
}
@@ -376,6 +380,105 @@ Examples:
t[3].cast<QuantizeType>());
}));
// binding hnsw rabitq index params
py::class_<HnswRabitqIndexParams, VectorIndexParams,
std::shared_ptr<HnswRabitqIndexParams>>
hnsw_rabitq_params(m, "HnswRabitqIndexParam", R"pbdoc(
Parameters for configuring an HNSW (Hierarchical Navigable Small World) index with RabitQ quantization.
HNSW is a graph-based approximate nearest neighbor search index. RabitQ is a
quantization method that provides high compression with minimal accuracy loss.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
m (int): Number of bi-directional links created for every new element
during construction. Higher values improve accuracy but increase
memory usage and construction time. Default is 50.
ef_construction (int): Size of the dynamic candidate list for nearest
neighbors during index construction. Larger values yield better
graph quality at the cost of slower build time. Default is 500.
Examples:
>>> from zvec.typing import MetricType
>>> params = HnswRabitqIndexParam(
... metric_type=MetricType.COSINE,
... m=16,
... ef_construction=200
... )
>>> print(params)
{'metric_type': 'COSINE', 'm': 16, 'ef_construction': 200}
)pbdoc");
hnsw_rabitq_params
.def(py::init<MetricType, int, int, int, int, int>(),
py::arg("metric_type") = MetricType::IP,
py::arg("total_bits") = core_interface::kDefaultRabitqTotalBits,
py::arg("num_clusters") = core_interface::kDefaultRabitqNumClusters,
py::arg("m") = core_interface::kDefaultHnswNeighborCnt,
py::arg("ef_construction") =
core_interface::kDefaultHnswEfConstruction,
py::arg("sample_count") = 0)
.def_property_readonly("m", &HnswRabitqIndexParams::m,
"int: Maximum number of neighbors per node.")
.def_property_readonly(
"ef_construction", &HnswRabitqIndexParams::ef_construction,
"int: Candidate list size during index construction.")
.def_property_readonly("total_bits", &HnswRabitqIndexParams::total_bits,
"int: Total bits for RabitQ quantization.")
.def_property_readonly("num_clusters",
&HnswRabitqIndexParams::num_clusters,
"int: Number of clusters for RabitQ.")
.def_property_readonly("sample_count",
&HnswRabitqIndexParams::sample_count,
"int: Sample count for RabitQ training.")
.def(
"to_dict",
[](const HnswRabitqIndexParams &self) -> py::dict {
py::dict dict;
dict["type"] = index_type_to_string(self.type());
dict["metric_type"] = metric_type_to_string(self.metric_type());
dict["quantize_type"] =
quantize_type_to_string(self.quantize_type());
dict["total_bits"] = self.total_bits();
dict["num_clusters"] = self.num_clusters();
dict["sample_count"] = self.sample_count();
dict["m"] = self.m();
dict["ef_construction"] = self.ef_construction();
return dict;
},
"Convert to dictionary with all fields")
.def(
"__repr__",
[](const HnswRabitqIndexParams &self) -> std::string {
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"metric_type\":\"" +
metric_type_to_string(self.metric_type()) +
"\", \"total_bits\":" + std::to_string(self.total_bits()) +
", \"num_clusters\":" + std::to_string(self.num_clusters()) +
", \"sample_count\":" + std::to_string(self.sample_count()) +
", \"m\":" + std::to_string(self.m()) +
", \"ef_construction\":" +
std::to_string(self.ef_construction()) +
", \"quantize_type\":\"" +
quantize_type_to_string(self.quantize_type()) + "\"}";
})
.def(py::pickle(
[](const HnswRabitqIndexParams &self) {
return py::make_tuple(self.metric_type(), self.total_bits(),
self.num_clusters(), self.m(),
self.ef_construction(), self.sample_count());
},
[](py::tuple t) {
if (t.size() != 6)
throw std::runtime_error(
"Invalid state for HnswRabitqIndexParams");
return std::make_shared<HnswRabitqIndexParams>(
t[0].cast<MetricType>(), t[1].cast<int>(), t[2].cast<int>(),
t[3].cast<int>(), t[4].cast<int>(), t[5].cast<int>());
}));
// FlatIndexParams
py::class_<FlatIndexParams, VectorIndexParams,
std::shared_ptr<FlatIndexParams>>
@@ -709,10 +812,81 @@ Args:
obj->set_is_linear(t[2].cast<bool>());
return obj;
}));
// binding hnsw rabitq query params
py::class_<HnswRabitqQueryParams, QueryParams,
std::shared_ptr<HnswRabitqQueryParams>>
hnsw_rabitq_query_params(m, "HnswRabitqQueryParam", R"pbdoc(
Query parameters for HNSW RaBitQ (Hierarchical Navigable Small World with RaBitQ quantization) index.
Controls the trade-off between search speed and accuracy via the `ef` parameter.
RaBitQ provides efficient quantization while maintaining high search quality.
Attributes:
type (IndexType): Always ``IndexType.HNSW_RABITQ``.
ef (int): Size of the dynamic candidate list during search.
Larger values improve recall but slow down search.
Default is 300.
radius (float): Search radius for range queries. Default is 0.0.
is_linear (bool): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
Examples:
>>> params = HnswRabitqQueryParam(ef=300)
>>> print(params.ef)
300
>>> print(params.to_dict() if hasattr(params, 'to_dict') else params)
{"type":"HNSW_RABITQ", "ef":300}
)pbdoc");
hnsw_rabitq_query_params
.def(py::init<int, float, bool, bool>(),
py::arg("ef") = core_interface::kDefaultHnswEfSearch,
py::arg("radius") = 0.0f, py::arg("is_linear") = false,
py::arg("is_using_refiner") = false,
R"pbdoc(
Constructs an HnswRabitqQueryParam instance.
Args:
ef (int, optional): Search-time candidate list size.
Higher values improve accuracy. Defaults to 300.
radius (float, optional): Search radius for range queries. Default is 0.0.
is_linear (bool, optional): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
)pbdoc")
.def_property_readonly(
"ef",
[](const HnswRabitqQueryParams &self) -> int { return self.ef(); },
"int: Size of the dynamic candidate list during HNSW RaBitQ search.")
.def("__repr__",
[](const HnswRabitqQueryParams &self) -> std::string {
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"ef\":" + std::to_string(self.ef()) +
", \"radius\":" + std::to_string(self.radius()) +
", \"is_linear\":" + std::to_string(self.is_linear()) +
", \"is_using_refiner\":" +
std::to_string(self.is_using_refiner()) + "}";
})
.def(py::pickle(
[](const HnswRabitqQueryParams &self) {
return py::make_tuple(self.ef(), self.radius(), self.is_linear(),
self.is_using_refiner());
},
[](py::tuple t) {
if (t.size() != 4)
throw std::runtime_error(
"Invalid state for HnswRabitqQueryParams");
auto obj =
std::make_shared<HnswRabitqQueryParams>(t[0].cast<int>());
obj->set_radius(t[1].cast<float>());
obj->set_is_linear(t[2].cast<bool>());
obj->set_is_using_refiner(t[3].cast<bool>());
return obj;
}));
}
void ZVecPyParams::bind_options(py::module_ &m) {
// binding collection options
void ZVecPyParams::bind_options(py::module_ &m) { // binding collection options
py::class_<CollectionOptions>(m, "CollectionOption", R"pbdoc(
Options for opening or creating a collection.
+3 -1
View File
@@ -96,6 +96,7 @@ Examples:
)pbdoc")
.value("UNDEFINED", IndexType::UNDEFINED)
.value("HNSW", IndexType::HNSW)
.value("HNSW_RABITQ", IndexType::HNSW_RABITQ)
.value("IVF", IndexType::IVF)
.value("FLAT", IndexType::FLAT)
.value("INVERT", IndexType::INVERT);
@@ -131,7 +132,8 @@ Examples:
.value("UNDEFINED", QuantizeType::UNDEFINED)
.value("FP16", QuantizeType::FP16)
.value("INT8", QuantizeType::INT8)
.value("INT4", QuantizeType::INT4);
.value("INT4", QuantizeType::INT4)
.value("RABITQ", QuantizeType::RABITQ);
}
void ZVecPyTyping::bind_status(py::module_ &m) {
+29 -2
View File
@@ -1,6 +1,26 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
if(RABITQ_SUPPORTED AND AUTO_DETECT_ARCH)
set(HNSW_RABITQ_FILES
hnsw_rabitq_query_algorithm.cc
hnsw_rabitq_streamer.cc
hnsw_rabitq_searcher.cc
hnsw_rabitq_entity.cc
rabitq_reformer.cc
rabitq_converter.cc
)
set(HNSW_RABITQ_FILES_FULL ${HNSW_RABITQ_FILES})
list(TRANSFORM HNSW_RABITQ_FILES_FULL PREPEND "algorithm/hnsw_rabitq/")
foreach(FILE ${HNSW_RABITQ_FILES_FULL})
set_source_files_properties(
${FILE}
PROPERTIES
COMPILE_FLAGS "${RABITQ_ARCH_FLAG}"
)
endforeach()
endif()
cc_directory(framework)
cc_directory(algorithm)
cc_directory(metric)
@@ -12,10 +32,17 @@ cc_directory(mixed_reducer)
git_version(GIT_SRCS_VER ${CMAKE_CURRENT_SOURCE_DIR})
file(GLOB_RECURSE ALL_CORE_SRCS *.cc *.c *.h)
# Remove algorithm/hnsw_rabitq implementation files if not supported.
# interface/indexes/hnsw_rabitq_index.cc is kept because it provides the vtable
# for HNSWRabitqIndex and guards rabitqlib usage with #if RABITQ_SUPPORTED.
if(NOT RABITQ_SUPPORTED)
list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/algorithm/hnsw_rabitq/.*")
endif()
cc_library(
NAME zvec_core STATIC STRICT PACKED
SRCS ${ALL_CORE_SRCS}
LIBS zvec_ailego zvec_turbo sparsehash magic_enum
LIBS zvec_ailego zvec_turbo sparsehash magic_enum rabitqlib
INCS . ${PROJECT_ROOT_DIR}/src/core
VERSION "${GIT_SRCS_VER}"
)
)
+22 -1
View File
@@ -6,4 +6,25 @@ cc_directory(flat)
cc_directory(flat_sparse)
cc_directory(ivf)
cc_directory(hnsw)
cc_directory(hnsw_sparse)
cc_directory(hnsw_sparse)
if(RABITQ_SUPPORTED)
message(STATUS "BUILD RABITQ")
cc_directory(hnsw_rabitq)
else()
message(STATUS "NOT BUILD RABITQ")
# Empty stub library for unsupported platforms
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
"// Stub implementation for unsupported platforms\n"
"// RaBitQ only supports Linux x86_64\n"
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
)
cc_library(
NAME core_knn_hnsw_rabitq
STATIC SHARED STRICT ALWAYS_LINK
SRCS ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
LIBS core_framework
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
endif()
@@ -1250,7 +1250,7 @@ int OptKmeansCluster::init(const IndexMeta &meta,
const ailego::Params &params) {
auto type_ = meta.data_type();
if (meta.metric_name() == "InnerProduct") {
if (meta.metric_name() == "InnerProduct" || meta.metric_name() == "Cosine") {
switch (type_) {
case IndexMeta::DataType::DT_FP16: {
algorithm_.reset(
-1
View File
@@ -127,7 +127,6 @@ int FlatBuilder<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
return error_code;
}
holder_ = nullptr;
stats_.set_dumped_count(keys.size());
stats_.set_dumped_costtime(stamp.milli_seconds());
return 0;
@@ -0,0 +1,21 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
if(AUTO_DETECT_ARCH)
foreach(FILE ${HNSW_RABITQ_FILES})
set_source_files_properties(
${FILE}
PROPERTIES
COMPILE_FLAGS "${RABITQ_ARCH_FLAG}"
)
endforeach()
endif()
cc_library(
NAME core_knn_hnsw_rabitq
STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS core_framework rabitqlib sparsehash
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
@@ -0,0 +1,382 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_algorithm.h"
#include <chrono>
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
HnswRabitqAlgorithm::HnswRabitqAlgorithm(HnswRabitqEntity &entity)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt) {}
int HnswRabitqAlgorithm::cleanup() {
return 0;
}
int HnswRabitqAlgorithm::add_node(node_id_t id, level_t level,
HnswRabitqContext *ctx) {
spin_lock_.lock();
// std::cout << "id: " << id << ", level: " << level << std::endl;
auto cur_max_level = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
entity_.update_ep_and_level(id, level);
spin_lock_.unlock();
return 0;
}
spin_lock_.unlock();
if (ailego_unlikely(level > cur_max_level)) {
mutex_.lock();
// re-check max level
cur_max_level = entity_.cur_max_level();
entry_point = entity_.entry_point();
if (level <= cur_max_level) {
mutex_.unlock();
}
}
level_t cur_level = cur_max_level;
ResultRecord dist = ctx->dist_calculator()(entry_point);
for (; cur_level > level; --cur_level) {
select_entry_point(cur_level, &entry_point, &dist, ctx);
}
for (; cur_level >= 0; --cur_level) {
search_neighbors(cur_level, &entry_point, &dist, ctx->level_topk(cur_level),
ctx);
}
// add neighbors from down level to top level, to avoid upper level visible
// to knn_search but the under layer level not ready
for (cur_level = 0; cur_level <= level; ++cur_level) {
add_neighbors(id, cur_level, ctx->level_topk(cur_level), ctx);
ctx->level_topk(cur_level).clear();
}
if (ailego_unlikely(level > cur_max_level)) {
spin_lock_.lock();
entity_.update_ep_and_level(id, level);
spin_lock_.unlock();
mutex_.unlock();
}
return 0;
}
//! select_entry_point on hnsw level, ef = 1
void HnswRabitqAlgorithm::select_entry_point(level_t level,
node_id_t *entry_point,
ResultRecord *dist,
HnswRabitqContext *ctx) const {
auto &entity = ctx->get_entity();
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
while (true) {
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
uint32_t size = neighbors.size();
if (size == 0) {
break;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = dc.get_vector(&neighbors[0], size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
if (ailego_unlikely(ret != 0)) {
break;
}
bool find_closer = false;
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
ResultRecord cur_dist = dists[i];
if (cur_dist < *dist) {
*entry_point = neighbors[i];
*dist = cur_dist;
find_closer = true;
}
}
if (!find_closer) {
break;
}
}
return;
}
void HnswRabitqAlgorithm::add_neighbors(node_id_t id, level_t level,
TopkHeap &topk_heap,
HnswRabitqContext *ctx) {
if (ailego_unlikely(topk_heap.size() == 0)) {
return;
}
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
update_neighbors(dc, id, level, topk_heap);
// reverse update neighbors
for (size_t i = 0; i < topk_heap.size(); ++i) {
reverse_update_neighbors(dc, topk_heap[i].first, level, id,
topk_heap[i].second, ctx->update_heap());
}
return;
}
void HnswRabitqAlgorithm::search_neighbors(level_t level,
node_id_t *entry_point,
ResultRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx) const {
const auto &entity = ctx->get_entity();
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
candidates.clear();
visit.clear();
visit.set_visited(*entry_point);
if (!filter(*entry_point)) {
topk.emplace(*entry_point, *dist);
}
candidates.emplace(*entry_point, *dist);
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
ResultRecord main_dist = top->second;
if (topk.full() && main_dist > topk[0].second) {
break;
}
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(level, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = dc.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
if (ailego_unlikely(ret != 0)) {
break;
}
// do prefetch
static constexpr node_id_t BATCH_SIZE = 12;
static constexpr node_id_t PREFETCH_STEP = 2;
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
}
// done
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
ResultRecord cur_dist = dists[i];
if ((!topk.full()) || cur_dist < topk[0].second) {
candidates.emplace(node, cur_dist);
// update entry_point for next level scan
if (cur_dist < *dist) {
*entry_point = node;
*dist = cur_dist;
}
if (!filter(node)) {
topk.emplace(node, cur_dist);
}
} // end if
} // end for
} // while
return;
}
void HnswRabitqAlgorithm::update_neighbors(HnswRabitqAddDistCalculator &dc,
node_id_t id, level_t level,
TopkHeap &topk_heap) {
topk_heap.sort();
uint32_t max_neighbor_cnt = entity_.neighbor_cnt(level);
if (topk_heap.size() <= static_cast<size_t>(entity_.prune_cnt())) {
if (topk_heap.size() <= static_cast<size_t>(max_neighbor_cnt)) {
entity_.update_neighbors(level, id, topk_heap);
return;
}
}
uint32_t cur_size = 0;
for (size_t i = 0; i < topk_heap.size(); ++i) {
node_id_t cur_node = topk_heap[i].first;
ResultRecord cur_node_dist = topk_heap[i].second;
bool good = true;
for (uint32_t j = 0; j < cur_size; ++j) {
ResultRecord tmp_dist = dc.dist(cur_node, topk_heap[j].first);
if (tmp_dist <= cur_node_dist) {
good = false;
break;
}
}
if (good) {
topk_heap[cur_size].first = cur_node;
topk_heap[cur_size].second = cur_node_dist;
cur_size++;
if (cur_size >= max_neighbor_cnt) {
break;
}
}
}
// when after-prune neighbor count is too seldom,
// we use this strategy to make-up enough edges
// not only just make-up out-degrees
// we also make-up enough in-degrees
uint32_t min_neighbors = entity_.min_neighbor_cnt();
for (size_t k = cur_size; cur_size < min_neighbors && k < topk_heap.size();
++k) {
bool exist = false;
for (size_t j = 0; j < cur_size; ++j) {
if (topk_heap[j].first == topk_heap[k].first) {
exist = true;
break;
}
}
if (!exist) {
topk_heap[cur_size].first = topk_heap[k].first;
topk_heap[cur_size].second = topk_heap[k].second;
cur_size++;
}
}
topk_heap.resize(cur_size);
entity_.update_neighbors(level, id, topk_heap);
return;
}
void HnswRabitqAlgorithm::reverse_update_neighbors(
HnswRabitqAddDistCalculator &dc, node_id_t id, level_t level,
node_id_t link_id, ResultRecord dist, TopkHeap &update_heap) {
const size_t max_neighbor_cnt = entity_.neighbor_cnt(level);
uint32_t lock_idx = id & kLockMask;
lock_pool_[lock_idx].lock();
const Neighbors neighbors = entity_.get_neighbors(level, id);
size_t size = neighbors.size();
ailego_assert_with(size <= max_neighbor_cnt, "invalid neighbor size");
if (size < max_neighbor_cnt) {
entity_.add_neighbor(level, id, size, link_id);
lock_pool_[lock_idx].unlock();
return;
}
update_heap.emplace(link_id, dist);
for (size_t i = 0; i < size; ++i) {
node_id_t node = neighbors[i];
ResultRecord cur_dist = dc.dist(id, node);
update_heap.emplace(node, cur_dist);
}
//! TODO: optimize prune
//! prune edges
update_heap.sort();
size_t cur_size = 0;
for (size_t i = 0; i < update_heap.size(); ++i) {
node_id_t cur_node = update_heap[i].first;
ResultRecord cur_node_dist = update_heap[i].second;
bool good = true;
for (size_t j = 0; j < cur_size; ++j) {
ResultRecord tmp_dist = dc.dist(cur_node, update_heap[j].first);
if (tmp_dist <= cur_node_dist) {
good = false;
break;
}
}
if (good) {
update_heap[cur_size].first = cur_node;
update_heap[cur_size].second = cur_node_dist;
cur_size++;
if (cur_size >= max_neighbor_cnt) {
break;
}
}
}
update_heap.resize(cur_size);
entity_.update_neighbors(level, id, update_heap);
lock_pool_[lock_idx].unlock();
update_heap.clear();
return;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,124 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <ailego/parallel/lock.h>
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
//! hnsw graph algorithm implement
class HnswRabitqAlgorithm {
public:
typedef std::unique_ptr<HnswRabitqAlgorithm> UPointer;
public:
//! Constructor
explicit HnswRabitqAlgorithm(HnswRabitqEntity &entity);
//! Destructor
~HnswRabitqAlgorithm() = default;
//! Cleanup HnswRabitqAlgorithm
int cleanup();
//! Add a node to hnsw graph
//! @id: the node unique id
//! @level: a node will be add to graph in each level [0, level]
//! return 0 on success, or errCode in failure
int add_node(node_id_t id, level_t level, HnswRabitqContext *ctx);
//! Initiate HnswRabitqAlgorithm
int init() {
level_probas_.clear();
double level_mult =
1 / std::log(static_cast<double>(entity_.scaling_factor()));
for (int level = 0;; level++) {
// refers faiss get_random_level alg
double proba =
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
if (proba < 1e-9) {
break;
}
level_probas_.push_back(proba);
}
return 0;
}
//! Generate a random level
//! return graph level
uint32_t get_random_level() const {
// gen rand float (0, 1)
double f = mt_() / static_cast<float>(mt_.max());
for (size_t level = 0; level < level_probas_.size(); level++) {
if (f < level_probas_[level]) {
return level;
}
f -= level_probas_[level];
}
return level_probas_.size() - 1;
}
private:
//! Select in upper layer to get entry point for next layer search
void select_entry_point(level_t level, node_id_t *entry_point,
ResultRecord *dist, HnswRabitqContext *ctx) const;
//! update node id neighbors from topkHeap, and reverse link is also updated
void add_neighbors(node_id_t id, level_t level, TopkHeap &topk_heap,
HnswRabitqContext *ctx);
//! Given a node id and level, search the nearest neighbors in graph
//! Note: the nearest neighbors result keeps in topk, and entry_point and
//! dist will be updated to current level nearest node id and distance
void search_neighbors(level_t level, node_id_t *entry_point,
ResultRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx) const;
//! Update the node's neighbors
void update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
level_t level, TopkHeap &topk_heap);
//! Checking linkId could be id's new neighbor, and add as neighbor if true
//! @dc distance calculator
//! @updateHeap temporary heap in updating neighbors
void reverse_update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
level_t level, node_id_t link_id,
ResultRecord dist, TopkHeap &update_heap);
private:
HnswRabitqAlgorithm(const HnswRabitqAlgorithm &) = delete;
HnswRabitqAlgorithm &operator=(const HnswRabitqAlgorithm &) = delete;
private:
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
HnswRabitqEntity &entity_;
mutable std::mt19937 mt_{};
std::vector<double> level_probas_{};
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
std::mutex mutex_{}; // global mutex
// TODO: spin lock?
std::vector<std::mutex> lock_pool_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,554 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_builder.h"
#include <cstdlib>
#include <iostream>
#include <thread>
#include <ailego/pattern/defer.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/utility/string_helper.h>
#include <zvec/ailego/utility/time_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_memory.h"
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_params.h"
#include "rabitq_converter.h"
#include "rabitq_params.h"
#include "rabitq_reformer.h"
namespace zvec {
namespace core {
HnswRabitqBuilder::HnswRabitqBuilder() {}
int HnswRabitqBuilder::init(const IndexMeta &meta,
const ailego::Params &params) {
LOG_INFO("Begin HnswRabitqBuilder::init");
meta_ = meta;
auto params_copy = params;
meta_.set_builder("HnswRabitqBuilder", HnswRabitqEntity::kRevision,
std::move(params_copy));
size_t memory_quota = 0UL;
params.get(PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA, &memory_quota);
params.get(PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT, &thread_cnt_);
params.get(PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
params.get(PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION, &ef_construction_);
params.get(PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS,
&check_interval_secs_);
params.get(PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT,
&upper_max_neighbor_cnt_);
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
params.get(PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
&multiplier);
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
scaling_factor_ = upper_max_neighbor_cnt_;
params.get(PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR, &scaling_factor_);
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
params.get(PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
if (ef_construction_ == 0) {
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
}
if (upper_max_neighbor_cnt_ == 0) {
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (upper_max_neighbor_cnt_ > kMaxNeighborCnt) {
LOG_ERROR("[%s] must be in range (0,%d]",
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
LOG_ERROR("[%s]-[%d] must be <= [%s]-[%d]",
PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT.c_str(),
min_neighbor_cnt_,
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
upper_max_neighbor_cnt_);
return IndexError_InvalidArgument;
}
if (l0_max_neighbor_cnt_ == 0) {
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("L0MaxNeighborCnt must be in range (0,%d)",
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (scaling_factor_ == 0U) {
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
}
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
LOG_ERROR("[%s] must be in range [5,1000]",
PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR.c_str());
return IndexError_InvalidArgument;
}
if (thread_cnt_ == 0) {
thread_cnt_ = std::thread::hardware_concurrency();
}
if (thread_cnt_ > std::thread::hardware_concurrency()) {
LOG_WARN("[%s] greater than cpu cores %zu",
PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT.c_str(),
static_cast<size_t>(std::thread::hardware_concurrency()));
}
if (prune_cnt == 0UL) {
prune_cnt = upper_max_neighbor_cnt_;
}
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
int ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
uint8_t ex_bits = total_bits - 1;
entity_.set_ex_bits(ex_bits);
uint32_t dimension = 0;
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
if (dimension == 0) {
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
return IndexError_InvalidArgument;
}
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
kMinRabitqDimSize, kMaxRabitqDimSize);
return IndexError_InvalidArgument;
}
entity_.update_rabitq_params_and_vector_size(dimension);
entity_.set_ef_construction(ef_construction_);
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_.set_scaling_factor(scaling_factor_);
entity_.set_memory_quota(memory_quota);
entity_.set_prune_cnt(prune_cnt);
ret = entity_.init();
if (ret != 0) {
return ret;
}
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
ret = alg_->init();
if (ret != 0) {
return ret;
}
// Create and initialize RaBitQ converter
converter_ = std::make_shared<RabitqConverter>();
IndexMeta converter_meta = meta_;
converter_meta.set_dimension(dimension);
ret = converter_->init(converter_meta, params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqConverter: %d", ret);
return ret;
}
state_ = BUILD_STATE_INITED;
LOG_INFO(
"End HnswRabitqBuilder::init, params: rawVectorSize=%u vectorSize=%zu "
"efConstruction=%u "
"l0NeighborCnt=%u upperNeighborCnt=%u scalingFactor=%u "
"memoryQuota=%zu neighborPruneCnt=%zu metricName=%s ",
meta_.element_size(), entity_.vector_size(), ef_construction_,
l0_max_neighbor_cnt_, upper_max_neighbor_cnt_, scaling_factor_,
memory_quota, prune_cnt, meta_.metric_name().c_str());
return 0;
}
int HnswRabitqBuilder::cleanup(void) {
LOG_INFO("Begin HnswRabitqBuilder::cleanup");
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
min_neighbor_cnt_ = 0;
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
check_interval_secs_ = kDefaultLogIntervalSecs;
errcode_ = 0;
error_ = false;
entity_.cleanup();
if (alg_) {
alg_->cleanup();
}
meta_.clear();
metric_.reset();
stats_.clear_attributes();
stats_.set_trained_count(0UL);
stats_.set_built_count(0UL);
stats_.set_dumped_count(0UL);
stats_.set_discarded_count(0UL);
stats_.set_trained_costtime(0UL);
stats_.set_built_costtime(0UL);
stats_.set_dumped_costtime(0UL);
state_ = BUILD_STATE_INIT;
LOG_INFO("End HnswRabitqBuilder::cleanup");
return 0;
}
int HnswRabitqBuilder::train(IndexThreads::Pointer,
IndexHolder::Pointer holder) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
return IndexError_NoReady;
}
if (!holder) {
LOG_ERROR("Input holder is nullptr while training index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while training index");
return IndexError_Mismatch;
}
LOG_INFO("Begin HnswRabitqBuilder::train");
size_t trained_cost_time = 0;
size_t trained_count = 0;
int ret = train_converter_and_load_reformer(holder);
if (ret != 0) {
return ret;
}
if (metric_->support_train()) {
auto start_time = ailego::Monotime::MilliSeconds();
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
while (iter->is_valid()) {
ret = metric_->train(iter->data(), meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw build measure train failed, ret=%d", ret);
return ret;
}
iter->next();
++trained_count;
}
trained_cost_time = ailego::Monotime::MilliSeconds() - start_time;
}
stats_.set_trained_count(trained_count);
stats_.set_trained_costtime(trained_cost_time);
state_ = BUILD_STATE_TRAINED;
LOG_INFO("End HnswRabitqBuilder::train");
return 0;
}
int HnswRabitqBuilder::train_converter_and_load_reformer(
IndexHolder::Pointer holder) {
// Train converter (KMeans clustering)
int ret = converter_->train(holder);
if (ret != 0) {
LOG_ERROR("Failed to train RabitqConverter: %d", ret);
return ret;
}
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
memory_dumper->init(ailego::Params());
std::string file_id = ailego::StringHelper::Concat(
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
ret = memory_dumper->create(file_id);
if (ret != 0) {
LOG_ERROR("Failed to create memory dumper: %d", ret);
return ret;
}
// Release memory
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
ret = converter_->dump(memory_dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
ret = memory_dumper->close();
if (ret != 0) {
LOG_ERROR("Failed to close memory dumper: %d", ret);
return ret;
}
reformer_ = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
ret = reformer_->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
ret = memory_storage->open(file_id, false);
if (ret != 0) {
LOG_ERROR("Failed to open memory storage: %d", ret);
return ret;
}
ret = reformer_->load(memory_storage);
if (ret != 0) {
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
return ret;
}
return 0;
}
int HnswRabitqBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
return IndexError_NoReady;
}
LOG_INFO("Begin HnswRabitqBuilder::train by trainer");
stats_.set_trained_count(0UL);
stats_.set_trained_costtime(0UL);
state_ = BUILD_STATE_TRAINED;
LOG_INFO("End HnswRabitqBuilder::train by trainer");
return 0;
}
int HnswRabitqBuilder::build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
if (state_ != BUILD_STATE_TRAINED) {
LOG_ERROR("Train the index before HnswRabitqBuilder::build");
return IndexError_NoReady;
}
if (!holder) {
LOG_ERROR("Input holder is nullptr while building index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while building index");
return IndexError_Mismatch;
}
IndexProvider::Pointer provider =
std::dynamic_pointer_cast<IndexProvider>(holder);
if (!provider) {
LOG_ERROR("Rabitq builder expect IndexProvider");
return IndexError_InvalidArgument;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_cnt_, false);
}
auto start_time = ailego::Monotime::MilliSeconds();
LOG_INFO("Begin HnswRabitqBuilder::build");
if (holder->count() != static_cast<size_t>(-1)) {
LOG_DEBUG("HnswRabitqBuilder holder documents count %lu", holder->count());
int ret = entity_.reserve_space(holder->count());
if (ret != 0) {
LOG_ERROR("HnswBuilde reserver space failed");
return ret;
}
}
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
int ret;
error_ = false;
IndexQueryMeta ometa;
ometa.set_meta(holder->data_type(), holder->dimension());
while (iter->is_valid()) {
const void *vec = iter->data();
// quantize vector
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(vec, ometa, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
node_id_t id;
if (converted_vector.size() != entity_.vector_size()) {
LOG_ERROR(
"Converted vector size %zu is not equal to entity vector size %zu",
converted_vector.size(), entity_.vector_size());
return IndexError_InvalidArgument;
}
ret = entity_.add_vector(level, iter->key(), converted_vector.data(), &id);
if (ailego_unlikely(ret != 0)) {
return ret;
}
iter->next();
}
LOG_INFO("Finished save vector, start build graph...");
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
std::atomic<node_id_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(ailego::Closure ::New(this, &HnswRabitqBuilder::do_build,
i, threads->count(), provider,
&finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
LOG_INFO("Built cnt %zu, finished percent %.3f%%",
static_cast<size_t>(finished.load()),
finished.load() * 100.0f / entity_.doc_cnt());
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
task_group->wait_finish();
stats_.set_built_count(finished.load());
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = BUILD_STATE_BUILT;
LOG_INFO("End HnswRabitqBuilder::build with RaBitQ quantization");
return 0;
}
void HnswRabitqBuilder::do_build(node_id_t idx, size_t step_size,
IndexProvider::Pointer provider,
std::atomic<node_id_t> *finished) {
AILEGO_DEFER([&]() {
std::lock_guard<std::mutex> latch(mutex_);
cond_.notify_one();
});
HnswRabitqContext *ctx = new (std::nothrow) HnswRabitqContext(
meta_.dimension(), metric_,
std::shared_ptr<HnswRabitqEntity>(&entity_, [](HnswRabitqEntity *) {}));
if (ailego_unlikely(ctx == nullptr)) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to create context");
errcode_ = IndexError_NoMemory;
}
return;
}
HnswRabitqContext::Pointer auto_ptr(ctx);
ctx->set_provider(std::move(provider));
ctx->set_max_scan_num(entity_.doc_cnt());
int ret = ctx->init(HnswRabitqContext::kBuilderContext);
if (ret != 0) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to init context");
errcode_ = IndexError_Runtime;
}
return;
}
for (node_id_t id = idx; id < entity_.doc_cnt(); id += step_size) {
ctx->reset_query(ctx->dist_calculator().get_vector(id));
ret = alg_->add_node(id, entity_.get_level(id), ctx);
if (ailego_unlikely(ret != 0)) {
if (!error_.exchange(true)) {
LOG_ERROR("Hnsw graph add node failed");
errcode_ = ret;
}
return;
}
ctx->clear();
(*finished)++;
}
}
int HnswRabitqBuilder::dump(const IndexDumper::Pointer &dumper) {
if (state_ != BUILD_STATE_BUILT) {
LOG_INFO("Build the index before HnswRabitqBuilder::dump");
return IndexError_NoReady;
}
LOG_INFO("Begin HnswRabitqBuilder::dump");
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
ailego::Params());
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
// Dump RaBitQ centroids first
if (converter_) {
ret = converter_->dump(dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
LOG_INFO("RaBitQ centroids dumped: %zu bytes, cost %zu ms",
converter_->stats().dumped_size(),
static_cast<size_t>(converter_->stats().dumped_costtime()));
}
ret = entity_.dump(dumper);
if (ret != 0) {
LOG_ERROR("HnswRabitqBuilder dump index failed");
return ret;
}
stats_.set_dumped_count(entity_.doc_cnt());
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
LOG_INFO("End HnswRabitqBuilder::dump");
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,101 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/parallel/thread_pool.h>
#include "zvec/core/framework/index_builder.h"
#include "zvec/core/framework/index_converter.h"
#include "zvec/core/framework/index_reformer.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_builder_entity.h"
namespace zvec {
namespace core {
class HnswRabitqBuilder : public IndexBuilder {
public:
//! Constructor
HnswRabitqBuilder();
//! Initialize the builder
virtual int init(const IndexMeta &meta,
const ailego::Params &params) override;
//! Cleanup the builder
virtual int cleanup(void) override;
//! Train the data
virtual int train(IndexThreads::Pointer,
IndexHolder::Pointer holder) override;
//! Train the data
virtual int train(const IndexTrainer::Pointer &trainer) override;
//! Build the index
virtual int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
private:
void do_build(node_id_t idx, size_t step_size,
IndexProvider::Pointer provider,
std::atomic<node_id_t> *finished);
int train_converter_and_load_reformer(IndexHolder::Pointer holder);
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
constexpr static uint32_t kMaxNeighborCnt = 65535;
private:
enum BUILD_STATE {
BUILD_STATE_INIT = 0,
BUILD_STATE_INITED = 1,
BUILD_STATE_TRAINED = 2,
BUILD_STATE_BUILT = 3
};
HnswRabitqBuilderEntity entity_{};
HnswRabitqAlgorithm::UPointer alg_; // impl graph algorithm
uint32_t thread_cnt_{0};
uint32_t min_neighbor_cnt_{0};
uint32_t upper_max_neighbor_cnt_{
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
int errcode_{0};
std::atomic_bool error_{false};
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexConverter::Pointer converter_{}; // RaBitQ converter
IndexReformer::Pointer reformer_{}; // RaBitQ reformer
std::mutex mutex_{};
std::condition_variable cond_{};
Stats stats_{};
BUILD_STATE state_{BUILD_STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,199 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_builder_entity.h"
#include <iostream>
#include <zvec/ailego/hash/crc32c.h>
#include "utility/sparse_utility.h"
namespace zvec {
namespace core {
HnswRabitqBuilderEntity::HnswRabitqBuilderEntity() {
update_ep_and_level(kInvalidNodeId, 0U);
}
int HnswRabitqBuilderEntity::cleanup() {
memory_quota_ = 0UL;
neighbors_size_ = 0U;
upper_neighbors_size_ = 0U;
padding_size_ = 0U;
vectors_buffer_.clear();
keys_buffer_.clear();
neighbors_buffer_.clear();
upper_neighbors_buffer_.clear();
neighbors_index_.clear();
vectors_buffer_.shrink_to_fit();
keys_buffer_.shrink_to_fit();
neighbors_buffer_.shrink_to_fit();
upper_neighbors_buffer_.shrink_to_fit();
neighbors_index_.shrink_to_fit();
this->HnswRabitqEntity::cleanup();
return 0;
}
int HnswRabitqBuilderEntity::init() {
size_t size = vector_size();
//! aligned size to 32
set_node_size(AlignSize(size));
//! if node size is aligned to 1k, the build performance will downgrade
if (node_size() % 1024 == 0) {
set_node_size(AlignSize(node_size() + 1));
}
padding_size_ = node_size() - size;
neighbors_size_ = neighbors_size();
upper_neighbors_size_ = upper_neighbors_size();
return 0;
}
int HnswRabitqBuilderEntity::reserve_space(size_t docs) {
if (memory_quota_ > 0 && (node_size() * docs + neighbors_size_ * docs +
sizeof(NeighborIndex) * docs >
memory_quota_)) {
return IndexError_NoMemory;
}
vectors_buffer_.reserve(node_size() * docs);
keys_buffer_.reserve(sizeof(key_t) * docs);
neighbors_buffer_.reserve(neighbors_size_ * docs);
neighbors_index_.reserve(docs);
return 0;
}
int HnswRabitqBuilderEntity::add_vector(level_t level, key_t key,
const void *vec, node_id_t *id) {
if (memory_quota_ > 0 &&
(vectors_buffer_.capacity() + keys_buffer_.capacity() +
neighbors_buffer_.capacity() + upper_neighbors_buffer_.capacity() +
neighbors_index_.capacity() * sizeof(NeighborIndex)) > memory_quota_) {
LOG_ERROR("Add vector failed, used memory exceed quota, cur_doc=%zu",
static_cast<size_t>(doc_cnt()));
return IndexError_NoMemory;
}
vectors_buffer_.append(reinterpret_cast<const char *>(vec), vector_size());
vectors_buffer_.append(padding_size_, '\0');
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
// init level 0 neighbors
neighbors_buffer_.append(neighbors_size_, '\0');
neighbors_index_.emplace_back(upper_neighbors_buffer_.size(), level);
// init upper layer neighbors
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
upper_neighbors_buffer_.append(upper_neighbors_size_, '\0');
}
*id = (*mutable_doc_cnt())++;
return 0;
}
key_t HnswRabitqBuilderEntity::get_key(node_id_t id) const {
return *(reinterpret_cast<const key_t *>(keys_buffer_.data() +
id * sizeof(key_t)));
}
const void *HnswRabitqBuilderEntity::get_vector(node_id_t id) const {
return vectors_buffer_.data() + id * node_size();
}
int HnswRabitqBuilderEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
const void *vec = get_vector(id);
block.reset((void *)vec);
return 0;
}
int HnswRabitqBuilderEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
for (uint32_t i = 0; i < count; ++i) {
vecs[i] = vectors_buffer_.data() + ids[i] * node_size();
}
return 0;
}
int HnswRabitqBuilderEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
std::vector<const void *> vecs(count);
get_vector(ids, count, vecs.data());
for (uint32_t i = 0; i < count; ++i) {
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
}
return 0;
}
const Neighbors HnswRabitqBuilderEntity::get_neighbors(level_t level,
node_id_t id) const {
const NeighborsHeader *hd = get_neighbor_header(level, id);
return {hd->neighbor_cnt, hd->neighbors};
}
int HnswRabitqBuilderEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
NeighborsHeader *hd =
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
for (size_t i = 0; i < neighbors.size(); ++i) {
hd->neighbors[i] = neighbors[i].first;
}
hd->neighbor_cnt = neighbors.size();
// std::cout << "id: " << id << ", neighbour, id: ";
// for (size_t i = 0; i < neighbors.size(); ++i) {
// if (i == neighbors.size()-1)
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
// std::endl;
// else
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
// ", id: ";
// }
return 0;
}
void HnswRabitqBuilderEntity::add_neighbor(level_t level, node_id_t id,
uint32_t /*size*/,
node_id_t neighbor_id) {
NeighborsHeader *hd =
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
hd->neighbors[hd->neighbor_cnt++] = neighbor_id;
return;
}
int HnswRabitqBuilderEntity::dump(const IndexDumper::Pointer &dumper) {
key_t *keys =
reinterpret_cast<key_t *>(const_cast<char *>(keys_buffer_.data()));
auto ret =
dump_segments(dumper, keys, [&](node_id_t id) { return get_level(id); });
if (ailego_unlikely(ret < 0)) {
return ret;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,139 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/internal/platform.h>
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqBuilderEntity : public HnswRabitqEntity {
public:
//! Add vector and key to hnsw entity, and local id will be saved to id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
//! Batch get vectors feature data by keys
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
const NeighborsHeader *get_neighbor_header(level_t level,
node_id_t id) const {
if (level == 0) {
return reinterpret_cast<const NeighborsHeader *>(
neighbors_buffer_.data() + neighbors_size_ * id);
} else {
size_t offset = neighbors_index_[id].offset;
return reinterpret_cast<const NeighborsHeader *>(
upper_neighbors_buffer_.data() + offset +
(level - 1) * upper_neighbors_size_);
}
}
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
//! Replace node id in level's neighbors
virtual int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
override;
//! add a neighbor to id in graph level
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump the hnsw graph to dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Cleanup the entity
virtual int cleanup(void) override;
public:
//! Constructor
HnswRabitqBuilderEntity();
//! Get the node graph level by id
level_t get_level(node_id_t id) const {
return neighbors_index_[id].level;
}
//! Init builerEntity
int init();
//! reserve buffer space for documents
//! @param docs number of documents
int reserve_space(size_t docs);
//! Set memory quota params
inline void set_memory_quota(size_t memory_quota) {
memory_quota_ = memory_quota;
}
//! Get neighbors size
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
//! Get upper neighbors size
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
public:
HnswRabitqBuilderEntity(const HnswRabitqBuilderEntity &) = delete;
HnswRabitqBuilderEntity &operator=(const HnswRabitqBuilderEntity &) = delete;
private:
friend class HnswRabitqSearcherEntity;
//! class internal used only
struct NeighborIndex {
NeighborIndex(size_t off, level_t l) : offset(off), level(l) {}
uint64_t offset : 48;
uint64_t level : 16;
};
std::string vectors_buffer_{}; // aligned vectors
std::string keys_buffer_{}; // aligned vectors
std::string neighbors_buffer_{}; // level 0 neighbors buffer
std::string upper_neighbors_buffer_{}; // upper layer neighbors buffer
std::string sparse_data_buffer_{}; // aligned spase data buffer
size_t sparse_data_offset_{0}; //
// upper layer offset + level in upper_neighbors_buffer_
std::vector<NeighborIndex> neighbors_index_{};
size_t memory_quota_{0UL};
size_t neighbors_size_{0U}; // level 0 neighbors size
size_t upper_neighbors_size_{0U}; // level 0 neighbors size
size_t padding_size_{}; // padding size for each vector element
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,221 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_chunk.h"
#include <chrono>
#include <random>
#include <zvec/ailego/hash/crc32c.h>
#include <zvec/ailego/utility/time_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_helper.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_streamer.h"
namespace zvec {
namespace core {
int HnswRabitqChunkBroker::init_storage(size_t chunk_size) {
chunk_meta_.clear();
chunk_meta_.chunk_size = chunk_size;
chunk_meta_.create_time = ailego::Realtime::Seconds();
stats_.set_create_time(chunk_meta_.create_time);
chunk_meta_.update_time = ailego::Realtime::Seconds();
stats_.set_update_time(chunk_meta_.update_time);
//! alloc meta chunk
size_t size = sizeof(HnswChunkMeta);
size = (size + page_mask_) & (~page_mask_);
const std::string segment_id =
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
int ret = stg_->append(segment_id, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
return ret;
}
chunk_meta_segment_ = get_chunk(CHUNK_TYPE_META, kDefaultChunkSeqId);
if (ailego_unlikely(!chunk_meta_segment_)) {
LOG_ERROR("Get meta segment failed");
return IndexError_Runtime;
}
//! update meta info and write to storage
chunk_meta_.chunk_cnts[CHUNK_TYPE_META] += 1;
chunk_meta_.total_size += size;
(*stats_.mutable_index_size()) += size;
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage write data failed, wsize=%zu", size);
return IndexError_WriteData;
}
return 0;
}
int HnswRabitqChunkBroker::load_storage(size_t chunk_size) {
IndexStorage::MemoryBlock data_block;
size_t size = chunk_meta_segment_->read(0UL, data_block,
chunk_meta_segment_->data_size());
if (size != sizeof(HnswChunkMeta)) {
LOG_ERROR("Invalid hnsw meta chunk, read size=%zu chunk size=%zu", size,
chunk_meta_segment_->data_size());
return IndexError_InvalidFormat;
}
std::memcpy(&chunk_meta_, data_block.data(), size);
if (chunk_meta_.chunk_size != chunk_size) {
LOG_ERROR(
"Params hnsw chunk size=%zu mismatch from previous %zu "
"in index",
chunk_size, (size_t)chunk_meta_.chunk_size);
return IndexError_Mismatch;
}
*stats_.mutable_check_point() = stg_->check_point();
stats_.set_revision_id(chunk_meta_.revision_id);
stats_.set_update_time(chunk_meta_.update_time);
stats_.set_create_time(chunk_meta_.create_time);
char create_time[32];
char update_time[32];
ailego::Realtime::Gmtime(chunk_meta_.create_time, "%Y-%m-%d %H:%M:%S",
create_time, sizeof(create_time));
ailego::Realtime::Gmtime(chunk_meta_.update_time, "%Y-%m-%d %H:%M:%S",
update_time, sizeof(update_time));
LOG_DEBUG(
"Load index, indexSize=%zu chunkSize=%zu nodeChunks=%zu "
"upperNeighborChunks=%zu revisionId=%zu "
"createTime=%s updateTime=%s",
(size_t)chunk_meta_.total_size, (size_t)chunk_meta_.chunk_size,
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_NODE],
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_UPPER_NEIGHBOR],
(size_t)chunk_meta_.revision_id, create_time, update_time);
return 0;
}
int HnswRabitqChunkBroker::open(IndexStorage::Pointer stg,
size_t max_index_size, size_t chunk_size,
bool check_crc) {
if (ailego_unlikely(stg_)) {
LOG_ERROR("An storage instance is already opened");
return IndexError_Duplicate;
}
stg_ = std::move(stg);
if (stg_->isHugePage()) {
page_mask_ = ailego::MemoryHelper::HugePageSize() - 1;
} else {
page_mask_ = ailego::MemoryHelper::PageSize() - 1;
}
check_crc_ = check_crc;
max_chunks_size_ = max_index_size;
dirty_ = false;
const std::string segment_id =
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
chunk_meta_segment_ = stg_->get(segment_id);
if (!chunk_meta_segment_) {
LOG_DEBUG("Create new index");
return init_storage(chunk_size);
}
return load_storage(chunk_size);
}
int HnswRabitqChunkBroker::close(void) {
flush(0UL);
stg_.reset();
check_crc_ = false;
dirty_ = false;
return 0;
}
int HnswRabitqChunkBroker::flush(uint64_t checkpoint) {
ailego_assert_with(chunk_meta_segment_, "invalid meta segment");
chunk_meta_.update_time = ailego::Realtime::Seconds();
stats_.set_update_time(chunk_meta_.update_time);
size_t size =
chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage write data failed, wsize=%zu", size);
}
stg_->refresh(checkpoint);
int ret = stg_->flush();
if (ret == 0) {
(*stats_.mutable_check_point()) = checkpoint;
} else {
LOG_ERROR("Storage flush failed for %s", IndexError::What(ret));
}
return ret;
}
std::pair<int, Chunk::Pointer> HnswRabitqChunkBroker::alloc_chunk(
int type, uint64_t seq_id, size_t size) {
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
Chunk::Pointer chunk;
if (ailego_unlikely(!stg_)) {
LOG_ERROR("Init storage first");
return std::make_pair(IndexError_Uninitialized, chunk);
}
//! check exist a empty chunk with the same name
chunk = get_chunk(type, seq_id);
if (chunk) {
if (ailego_unlikely(chunk->capacity() == size &&
chunk->data_size() == 0UL)) {
LOG_ERROR("Exist invalid chunk size %zu, expect size %zu",
chunk->capacity(), size);
chunk.reset();
return std::make_pair(IndexError_Runtime, chunk);
}
return std::make_pair(0, chunk);
}
//! align to page size
size = (size + page_mask_) & (~page_mask_);
if (ailego_unlikely(chunk_meta_.total_size + size >= max_chunks_size_)) {
LOG_ERROR("No space to new a chunk, curIndexSize=%zu allocSize=%zu",
(size_t)chunk_meta_.total_size, size);
return std::make_pair(IndexError_IndexFull, chunk);
}
std::string segment_id = make_segment_id(type, seq_id);
int ret = stg_->append(segment_id, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
return std::make_pair(ret, chunk);
}
chunk_meta_.chunk_cnts[type] += 1;
chunk_meta_.total_size += size;
(*stats_.mutable_index_size()) += size;
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage append segment failed, wsize=%zu", size);
}
chunk = get_chunk(type, seq_id);
return std::make_pair(chunk ? 0 : IndexError_NoMemory, chunk);
}
Chunk::Pointer HnswRabitqChunkBroker::get_chunk(int type,
uint64_t seq_id) const {
std::string segment_id = make_segment_id(type, seq_id);
return stg_->get(segment_id);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,140 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <atomic>
#include <cstddef>
#include <mutex>
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/internal/platform.h>
#include <zvec/ailego/utility/string_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_storage.h"
#include "zvec/core/framework/index_streamer.h"
namespace zvec {
namespace core {
using Chunk = IndexStorage::Segment;
class HnswRabitqChunkBroker {
public:
typedef std::shared_ptr<HnswRabitqChunkBroker> Pointer;
enum CHUNK_TYPE {
CHUNK_TYPE_HEADER = 1,
CHUNK_TYPE_META = 2,
CHUNK_TYPE_NODE = 3,
CHUNK_TYPE_UPPER_NEIGHBOR = 4,
CHUNK_TYPE_NEIGHBOR_INDEX = 5,
CHUNK_TYPE_SPARSE_NODE = 6,
CHUNK_TYPE_MAX = 8
};
static constexpr size_t kDefaultChunkSeqId = 0UL;
HnswRabitqChunkBroker(IndexStreamer::Stats &stats) : stats_(stats) {}
//! Open storage
int open(IndexStorage::Pointer stg, size_t max_index_size, size_t chunk_size,
bool check_crc);
int close(void);
int flush(uint64_t checkpoint);
//! alloc a new chunk with size, not thread-safe
std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id,
size_t size);
//! alloc a new chunk with chunk size
inline std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id) {
return alloc_chunk(type, seq_id, chunk_meta_.chunk_size);
}
Chunk::Pointer get_chunk(int type, uint64_t seq_id) const;
inline size_t get_chunk_cnt(int type) const {
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
return chunk_meta_.chunk_cnts[type];
}
inline bool dirty(void) const {
return dirty_;
}
inline void mark_dirty(void) {
if (!dirty_) {
dirty_ = true;
chunk_meta_.revision_id += 1;
stats_.set_revision_id(chunk_meta_.revision_id);
}
}
const IndexStorage::Pointer storage(void) const {
return stg_;
}
private:
HnswRabitqChunkBroker(const HnswRabitqChunkBroker &) = delete;
HnswRabitqChunkBroker &operator=(const HnswRabitqChunkBroker &) = delete;
struct HnswChunkMeta {
HnswChunkMeta(void) {
memset(this, 0, sizeof(HnswChunkMeta));
}
void clear() {
memset(this, 0, sizeof(HnswChunkMeta));
}
uint64_t chunk_cnts[CHUNK_TYPE_MAX];
uint64_t chunk_size; // size of per chunk
uint64_t total_size; // total size of allocated chunk
uint64_t revision_id; // index revision
uint64_t create_time;
uint64_t update_time;
uint64_t reserved[3];
};
static_assert(sizeof(HnswChunkMeta) % 32 == 0,
"HnswChunkMeta must be aligned with 32 bytes");
//! Init the storage after open an empty index
int init_storage(size_t chunk_size);
//! Load index from storage
int load_storage(size_t chunk_size);
static inline const std::string make_segment_id(int type, uint64_t seq_id) {
return "HnswT" + ailego::StringHelper::ToString(type) + "S" +
ailego::StringHelper::ToString(seq_id);
}
private:
IndexStreamer::Stats &stats_;
HnswChunkMeta chunk_meta_{};
size_t page_mask_{0UL};
size_t max_chunks_size_{0UL};
IndexStorage::Pointer stg_{};
IndexStorage::Segment::Pointer chunk_meta_segment_{};
bool check_crc_{false};
bool dirty_{false}; // set as true if index is modified , the flag
// will not be cleared even if flushed
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,297 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_context.h"
#include <chrono>
#include "hnsw_rabitq_params.h"
namespace zvec {
namespace core {
HnswRabitqContext::HnswRabitqContext(size_t dimension,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity)
: IndexContext(metric),
entity_(entity),
add_dc_(entity_.get(), metric, dimension) {}
HnswRabitqContext::HnswRabitqContext(const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity)
: IndexContext(metric), entity_(entity), add_dc_(entity_.get(), metric) {}
HnswRabitqContext::~HnswRabitqContext() {
visit_filter_.destroy();
}
int HnswRabitqContext::init(ContextType type) {
int ret;
uint32_t doc_cnt;
type_ = type;
switch (type) {
case kBuilderContext:
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
max_scan_num_, negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
break;
case kSearcherContext:
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(), max_scan_num_,
negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
break;
case kStreamerContext:
// maxScanNum is unknown if inited from streamer, so the docCnt may
// change. we need to compute maxScanNum by scan ratio, and preserve
// max_doc_cnt space from visit filter
doc_cnt = entity_->doc_cnt();
max_scan_num_ = compute_max_scan_num(doc_cnt);
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
ret = visit_filter_.init(filter_mode_, reserve_max_doc_cnt_,
max_scan_num_, negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
candidates_.limit(max_scan_num_);
check_need_adjuct_ctx();
break;
default:
LOG_ERROR("Init context failed");
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqContext::update(const ailego::Params &params) {
auto update_visit_filter_param = [&]() {
bool need_update = false;
std::string p;
switch (type_) {
case kSearcherContext:
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE;
break;
case kStreamerContext:
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE;
break;
}
if (params.has(p)) {
bool bf_enabled;
params.get(p, &bf_enabled);
if (bf_enabled ^ (filter_mode_ == VisitFilter::BloomFilter)) {
need_update = true;
filter_mode_ =
bf_enabled ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
}
}
float prob = negative_probability_;
p.clear();
switch (type_) {
case kSearcherContext:
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
break;
case kStreamerContext:
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
break;
}
params.get(p, &prob);
if (filter_mode_ == VisitFilter::BloomFilter &&
std::abs(prob - negative_probability_) > 1e-6) {
need_update = true;
}
if (need_update) {
visit_filter_.destroy();
int max_doc_cnt = 0;
if (type_ == kSearcherContext) {
max_doc_cnt = entity_->doc_cnt();
} else {
max_doc_cnt = reserve_max_doc_cnt_;
}
int ret = visit_filter_.init(filter_mode_, max_doc_cnt, max_scan_num_,
negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
}
return 0;
};
switch (type_) {
case kSearcherContext:
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_EF)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
topk_heap_.limit(std::max(topk_, ef_));
}
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
max_scan_num_ =
static_cast<uint32_t>(max_scan_ratio_ * entity_->doc_cnt());
max_scan_num_ = std::max(10000U, max_scan_num_);
}
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
}
return update_visit_filter_param();
case kStreamerContext:
if (params.has(PARAM_HNSW_RABITQ_STREAMER_EF)) {
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
topk_heap_.limit(std::max(topk_, ef_));
}
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_limit_ < min_scan_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
return IndexError_InvalidArgument;
}
if (params.has(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD)) {
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
}
return update_visit_filter_param();
default:
LOG_ERROR("update context failed, type=%zu", static_cast<size_t>(type_));
return IndexError_Runtime;
}
}
int HnswRabitqContext::update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity,
uint32_t magic_num) {
uint32_t doc_cnt;
if (ailego_unlikely(type != type_)) {
LOG_ERROR(
"HnswRabitqContext doesn't support shared by different type, "
"src=%u dst=%u",
type_, type);
return IndexError_Unsupported;
}
magic_ = kInvalidMgic;
// TODO: support change filter mode?
switch (type) {
case kBuilderContext:
LOG_ERROR("BuildContext doesn't support update");
return IndexError_NotImplemented;
case kSearcherContext:
if (!visit_filter_.reset(entity->doc_cnt(), max_scan_num_)) {
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
return IndexError_Runtime;
}
candidates_.limit(max_scan_num_);
topk_heap_.limit(std::max(topk_, ef_));
break;
case kStreamerContext:
doc_cnt = entity->doc_cnt();
max_scan_num_ = compute_max_scan_num(doc_cnt);
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
if (!visit_filter_.reset(reserve_max_doc_cnt_, max_scan_num_)) {
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
return IndexError_Runtime;
}
update_heap_.limit(entity->l0_neighbor_cnt() + 1);
candidates_.limit(max_scan_num_);
topk_heap_.limit(std::max(topk_, ef_));
break;
default:
LOG_ERROR("update context failed");
return IndexError_Runtime;
}
entity_ = entity;
dc().update(entity_.get(), metric, meta.dimension());
magic_ = magic_num;
level_topks_.clear();
return 0;
}
void HnswRabitqContext::fill_random_to_topk_full(void) {
static std::mt19937 mt(
std::chrono::system_clock::now().time_since_epoch().count());
std::uniform_int_distribution<node_id_t> dt(0, entity_->doc_cnt() - 1);
std::function<node_id_t()> gen;
node_id_t seqid;
std::function<bool(node_id_t)> myfilter = [](node_id_t) { return false; };
if (this->filter().is_valid()) {
myfilter = [&](node_id_t id) {
return this->filter()(entity_->get_key(id));
};
}
if (topk_heap_.limit() < entity_->doc_cnt() / 2) {
gen = [&](void) { return dt(mt); };
} else {
// If topk limit is big value, gen sequential id from an random initial
seqid = dt(mt);
gen = [&](void) {
seqid = seqid == (entity_->doc_cnt() - 1) ? 0 : (seqid + 1);
return seqid;
};
}
for (size_t i = 0; !topk_heap_.full() && i < entity_->doc_cnt(); ++i) {
const auto id = gen();
if (!visit_filter_.visited(id) && !myfilter(id)) {
visit_filter_.set_visited(id);
topk_heap_.emplace(id, dc().dist(id));
}
}
return;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,541 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/logger/logger.h>
#include "utility/visit_filter.h"
#include "zvec/core/framework/index_context.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqContext : public IndexContext {
public:
//! Index Context Pointer
typedef std::unique_ptr<HnswRabitqContext> Pointer;
enum ContextType {
kUnknownContext = 0,
kSearcherContext = 1,
kBuilderContext = 2,
kStreamerContext = 3
};
//! Construct
HnswRabitqContext(size_t dimension, const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity);
//! Construct
HnswRabitqContext(const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity);
//! Destructor
virtual ~HnswRabitqContext();
public:
//! Set topk of search result
virtual void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
return magic_;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
return this->debugging();
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(
buf, sizeof(buf),
"scan_cnt=%zu,get_vector_cnt=%u,get_neighbors_cnt=%u,dup_node=%u",
get_scan_num(), stats_get_vector_cnt_, stats_get_neighbors_cnt_,
stats_visit_dup_cnt_);
return std::string(buf, size);
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override;
public:
//! Init context
int init(ContextType type);
//! Update context, the context may be shared by different searcher/streamer
int update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity,
uint32_t magic_num);
inline const HnswRabitqEntity &get_entity() const {
return *entity_;
}
inline void resize_results(size_t size) {
if (group_by_search()) {
group_results_.resize(size);
} else {
results_.resize(size);
}
}
inline void topk_to_result() {
return topk_to_result(0);
}
//! Construct result from topk heap, result will be normalized
inline void topk_to_result(uint32_t idx) {
if (group_by_search()) {
topk_to_group_result(idx);
} else {
topk_to_single_result(idx);
}
}
inline void topk_to_single_result(uint32_t idx) {
if (force_padding_topk_ && !topk_heap_.full() &&
topk_heap_.size() < entity_->doc_cnt()) {
this->fill_random_to_topk_full();
}
if (ailego_unlikely(topk_heap_.size() == 0)) {
return;
}
ailego_assert_with(idx < results_.size(), "invalid idx");
int size = std::min(topk_, static_cast<uint32_t>(topk_heap_.size()));
topk_heap_.sort();
results_[idx].clear();
for (int i = 0; i < size; ++i) {
auto score = topk_heap_[i].second;
if (score.est_dist > this->threshold()) {
break;
}
node_id_t id = topk_heap_[i].first;
if (fetch_vector_) {
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id,
entity_->get_vector(id));
} else {
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id);
}
}
return;
}
//! Construct result from topk heap, result will be normalized
inline void topk_to_group_result(uint32_t idx) {
ailego_assert_with(idx < group_results_.size(), "invalid idx");
group_results_[idx].clear();
std::vector<std::pair<std::string, TopkHeap>> group_topk_list;
std::vector<std::pair<std::string, ResultRecord>> best_score_in_groups;
for (auto itr = group_topk_heaps_.begin(); itr != group_topk_heaps_.end();
itr++) {
const std::string &group_id = (*itr).first;
auto &heap = (*itr).second;
heap.sort();
if (heap.size() > 0) {
ResultRecord best_score = heap[0].second;
best_score_in_groups.push_back(std::make_pair(group_id, best_score));
}
}
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
[](const std::pair<std::string, ResultRecord> &a,
const std::pair<std::string, ResultRecord> &b) -> int {
return a.second < b.second;
});
// truncate to group num
for (uint32_t i = 0; i < group_num() && i < best_score_in_groups.size();
++i) {
const std::string &group_id = best_score_in_groups[i].first;
group_topk_list.emplace_back(
std::make_pair(group_id, group_topk_heaps_[group_id]));
}
group_results_[idx].resize(group_topk_list.size());
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
const std::string &group_id = group_topk_list[i].first;
group_results_[idx][i].set_group_id(group_id);
uint32_t size = std::min(
group_topk_, static_cast<uint32_t>(group_topk_list[i].second.size()));
for (uint32_t j = 0; j < size; ++j) {
auto score = group_topk_list[i].second[j].second;
if (score > this->threshold()) {
break;
}
node_id_t id = group_topk_list[i].second[j].first;
if (fetch_vector_) {
group_results_[idx][i].mutable_docs()->emplace_back(
entity_->get_key(id), score.est_dist, id,
entity_->get_vector(id));
} else {
group_results_[idx][i].mutable_docs()->emplace_back(
entity_->get_key(id), score.est_dist, id);
}
}
}
}
inline void reset_query(const void *query) {
if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
query_preprocess_func != nullptr) {
size_t dim = dc().dimension();
preprocess_buffer_.resize(dim);
memcpy(preprocess_buffer_.data(), query, dim);
query_preprocess_func(preprocess_buffer_.data(), dim);
query = preprocess_buffer_.data();
}
dc().reset_query(query);
dc().clear_compare_cnt();
query_ = query;
}
inline HnswRabitqAddDistCalculator &dist_calculator() {
return dc();
}
inline TopkHeap &topk_heap() {
return topk_heap_;
}
inline TopkHeap &update_heap() {
return update_heap_;
}
inline VisitFilter &visit_filter() {
return visit_filter_;
}
inline CandidateHeap &candidates() {
return candidates_;
}
inline void set_max_scan_num(uint32_t max_scan_num) {
max_scan_num_ = max_scan_num;
}
inline void set_max_scan_limit(uint32_t max_scan_limit) {
max_scan_limit_ = max_scan_limit;
}
inline void set_min_scan_limit(uint32_t min_scan_limit) {
min_scan_limit_ = min_scan_limit;
}
inline void set_ef(uint32_t v) {
ef_ = v;
}
inline void set_filter_mode(uint32_t v) {
filter_mode_ = v;
}
inline void set_filter_negative_probability(float v) {
negative_probability_ = v;
}
inline void set_max_scan_ratio(float v) {
max_scan_ratio_ = v;
}
virtual void set_magic(uint32_t v) {
magic_ = v;
}
virtual void set_force_padding_topk(bool v) {
force_padding_topk_ = v;
}
void set_bruteforce_threshold(uint32_t v) override {
bruteforce_threshold_ = v;
}
inline uint32_t get_bruteforce_threshold() const {
return bruteforce_threshold_;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
bool fetch_vector() const override {
return fetch_vector_;
}
//! Reset context
void reset(void) override {
set_filter(nullptr);
reset_threshold();
set_fetch_vector(false);
set_group_params(0, 0);
reset_group_by();
}
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
return group_topk_heaps_;
}
inline TopkHeap &level_topk(int level) {
if (ailego_unlikely(level_topks_.size() <= static_cast<size_t>(level))) {
int cur_level = level_topks_.size();
level_topks_.resize(level + 1);
for (; cur_level <= level; ++cur_level) {
size_t heap_size = std::max(entity_->neighbor_cnt(cur_level),
entity_->ef_construction());
level_topks_[cur_level].clear();
level_topks_[cur_level].limit(heap_size);
}
}
return level_topks_[level];
}
inline void check_need_adjuct_ctx(void) {
check_need_adjuct_ctx(entity_->doc_cnt());
}
inline size_t compute_reserve_cnt(uint32_t cur_doc) const {
if (cur_doc > kMaxReserveDocCnt) {
return kMaxReserveDocCnt;
} else if (cur_doc < kMinReserveDocCnt) {
return kMinReserveDocCnt;
}
return cur_doc;
}
//! candidates heap and visitfilter need to resize as doc cnt growing up
inline void check_need_adjuct_ctx(uint32_t doc_cnt) {
if (ailego_unlikely(doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_)) {
while (doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_) {
reserve_max_doc_cnt_ =
reserve_max_doc_cnt_ + compute_reserve_cnt(reserve_max_doc_cnt_);
}
uint32_t max_scan_cnt = compute_max_scan_num(reserve_max_doc_cnt_);
max_scan_num_ = max_scan_cnt;
visit_filter_.reset(reserve_max_doc_cnt_, max_scan_cnt);
candidates_.clear();
candidates_.limit(max_scan_num_);
}
}
inline uint32_t compute_max_scan_num(uint32_t max_doc_cnt) const {
uint32_t max_scan = max_doc_cnt * max_scan_ratio_;
if (max_scan < min_scan_limit_) {
max_scan = min_scan_limit_;
} else if (max_scan > max_scan_limit_) {
max_scan = max_scan_limit_;
}
return max_scan;
}
inline size_t get_scan_num() const {
return dc().compare_cnt();
}
inline uint64_t reach_scan_limit() const {
return dc().compare_cnt() >= max_scan_num_;
}
inline bool error() const {
return dc().error();
}
inline void clear() {
add_dc_.clear();
if (ailego_unlikely(this->debugging())) {
stats_get_neighbors_cnt_ = 0u;
stats_get_vector_cnt_ = 0u;
stats_visit_dup_cnt_ = 0u;
}
// do not clear results_ for the next query will need it
for (auto &it : results_) {
it.clear();
}
}
uint32_t *mutable_stats_get_neighbors() {
return &stats_get_neighbors_cnt_;
}
uint32_t *mutable_stats_get_vector() {
return &stats_get_vector_cnt_;
}
uint32_t *mutable_stats_visit_dup_cnt() {
return &stats_visit_dup_cnt_;
}
inline bool debugging(void) const {
return debug_mode_;
}
inline void update_dist_caculator_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
dc().update_distance(distance, batch_distance);
}
//! Get topk
inline uint32_t topk() const override {
return topk_;
}
//! Get group topk
inline uint32_t group_topk() const {
return group_topk_;
}
//! Get group num
inline uint32_t group_num() const {
return group_num_;
}
//! Get if group by search
inline bool group_by_search() {
return group_num_ > 0;
}
//! Set group params
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
group_num_ = group_num;
group_topk_ = group_topk;
topk_ = group_topk_ * group_num_;
topk_heap_.limit(std::max(topk_, ef_));
group_topk_heaps_.clear();
}
void set_provider(IndexProvider::Pointer provider) {
add_dc_.set_provider(std::move(provider));
}
const void *query() const {
return query_;
}
private:
inline HnswRabitqAddDistCalculator &dc() {
return add_dc_;
}
inline const HnswRabitqAddDistCalculator &dc() const {
return add_dc_;
}
private:
// Filling random nodes if topk not full
void fill_random_to_topk_full(void);
constexpr static uint32_t kTriggerReserveCnt = 4096UL;
constexpr static uint32_t kMinReserveDocCnt = 4096UL;
constexpr static uint32_t kMaxReserveDocCnt = 128 * 1024UL;
constexpr static uint32_t kInvalidMgic = -1U;
private:
HnswRabitqEntity::Pointer entity_;
HnswRabitqAddDistCalculator add_dc_;
IndexMetric::Pointer metric_;
bool debug_mode_{false};
bool force_padding_topk_{false};
uint32_t max_scan_num_{0};
uint32_t max_scan_limit_{0};
uint32_t min_scan_limit_{0};
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
uint32_t topk_{0};
uint32_t group_topk_{0};
uint32_t filter_mode_{VisitFilter::ByteMap};
float negative_probability_{HnswRabitqEntity::kDefaultBFNegativeProbability};
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
uint32_t magic_{0U};
std::vector<IndexDocumentList> results_{};
std::vector<IndexGroupDocumentList> group_results_{};
TopkHeap topk_heap_{};
TopkHeap update_heap_{};
std::vector<TopkHeap> level_topks_{};
CandidateHeap candidates_{};
VisitFilter visit_filter_{};
uint32_t bruteforce_threshold_{};
bool fetch_vector_{false};
uint32_t group_num_{0};
std::map<std::string, TopkHeap> group_topk_heaps_{};
uint32_t type_{kUnknownContext};
//! debug stats info
uint32_t stats_get_neighbors_cnt_{0u};
uint32_t stats_get_vector_cnt_{0u};
uint32_t stats_visit_dup_cnt_{0u};
std::string preprocess_buffer_;
const void *query_{nullptr};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,39 @@
// Copyright 2025-present the centaurdb project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#include "core/algorithm/hnsw_rabitq/hnsw_rabitq_dist_calculator.h"
#include "zvec/core/framework/index_error.h"
namespace zvec::core {
int HnswRabitqAddDistCalculator::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
for (uint32_t i = 0; i < count; ++i) {
const node_id_t id = ids[i];
key_t key = entity_->get_key(id);
if (key == kInvalidKey) {
return IndexError_NoExist;
}
IndexStorage::MemoryBlock block;
int ret = provider_->get_vector(key, block);
if (ret != 0) {
return ret;
}
vec_blocks.push_back(std::move(block));
}
return 0;
}
} // namespace zvec::core
@@ -0,0 +1,240 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_metric.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
//! HnswRabitqAddDistCalculator is only used for index construction
class HnswRabitqAddDistCalculator {
public:
typedef std::shared_ptr<HnswRabitqAddDistCalculator> Pointer;
public:
enum DistType {
DIST_NONE = 0,
DIST_DENSE = 1,
DIST_HYBRID = 2,
DIST_SPARSE = 3
};
public:
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(dim),
compare_cnt_(0) {}
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim,
const void *query)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(query),
dim_(dim),
compare_cnt_(0) {}
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(0),
compare_cnt_(0) {}
void update(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
}
void update(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
dim_ = dim;
}
inline void update_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
distance_ = distance;
batch_distance_ = batch_distance;
}
//! Reset query vector data
inline void reset_query(const void *query) {
error_ = false;
query_ = query;
}
//! Returns distance
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
LOG_ERROR("Nullptr of dense vector");
error_ = true;
return 0.0f;
}
float score{0.0f};
distance_(vec_lhs, vec_rhs, dim_, &score);
return score;
}
//! Returns distance between query and vec.
inline dist_t dist(const void *vec) {
compare_cnt_++;
return dist(vec, query_);
}
//! Return distance between query and node id.
inline dist_t dist(node_id_t id) {
compare_cnt_++;
const void *feat = get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
return dist(feat, query_);
}
//! Return dist node lhs between node rhs
inline dist_t dist(node_id_t lhs, node_id_t rhs) {
compare_cnt_++;
const void *feat = get_vector(lhs);
const void *query = get_vector(rhs);
if (ailego_unlikely(feat == nullptr || query == nullptr)) {
LOG_ERROR("Get nullptr vector");
error_ = true;
return 0.0f;
}
return dist(feat, query);
}
dist_t operator()(const void *vec) {
return dist(vec);
}
dist_t operator()(id_t i) {
return dist(i);
}
dist_t operator()(id_t lhs, id_t rhs) {
return dist(lhs, rhs);
}
void batch_dist(const void **vecs, size_t num, dist_t *distances) {
compare_cnt_++;
batch_distance_(vecs, query_, num, dim_, distances);
}
inline dist_t batch_dist(node_id_t id) {
compare_cnt_++;
const void *feat = get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
dist_t score = 0;
batch_distance_(&feat, query_, 1, dim_, &score);
return score;
}
inline void clear() {
compare_cnt_ = 0;
error_ = false;
}
inline void clear_compare_cnt() {
compare_cnt_ = 0;
}
inline bool error() const {
return error_;
}
//! Get distances compute times
inline uint32_t compare_cnt() const {
return compare_cnt_;
}
inline uint32_t dimension() const {
return dim_;
}
void set_provider(IndexProvider::Pointer provider) {
provider_ = std::move(provider);
}
int get_vector(const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const;
const void *get_vector(node_id_t id) const {
key_t key = entity_->get_key(id);
if (key == kInvalidKey) {
return nullptr;
}
return provider_->get_vector(key);
}
private:
HnswRabitqAddDistCalculator(const HnswRabitqAddDistCalculator &) = delete;
HnswRabitqAddDistCalculator &operator=(const HnswRabitqAddDistCalculator &) =
delete;
private:
const HnswRabitqEntity *entity_;
IndexMetric::MatrixDistance distance_;
IndexMetric::MatrixBatchDistance batch_distance_;
const void *query_;
uint32_t dim_;
uint32_t compare_cnt_; // record distance compute times
uint32_t compare_cnt_batch_; // record batch distance compute time
bool error_{false};
// get raw vector
IndexProvider::Pointer provider_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,382 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_entity.h"
#include <rabitqlib/index/query.hpp>
#include "utility/sparse_utility.h"
#include "zvec/core/framework/index_stats.h"
namespace zvec {
namespace core {
const std::string HnswRabitqEntity::kGraphHeaderSegmentId = "graph.header";
const std::string HnswRabitqEntity::kGraphFeaturesSegmentId = "graph.features";
const std::string HnswRabitqEntity::kGraphKeysSegmentId = "graph.keys";
const std::string HnswRabitqEntity::kGraphNeighborsSegmentId =
"graph.neighbors";
const std::string HnswRabitqEntity::kGraphOffsetsSegmentId = "graph.offsets";
const std::string HnswRabitqEntity::kGraphMappingSegmentId = "graph.mapping";
const std::string HnswRabitqEntity::kHnswHeaderSegmentId = "hnsw.header";
const std::string HnswRabitqEntity::kHnswNeighborsSegmentId = "hnsw.neighbors";
const std::string HnswRabitqEntity::kHnswOffsetsSegmentId = "hnsw.offsets";
void HnswRabitqEntity::update_rabitq_params_and_vector_size(
uint32_t dimension) {
uint32_t padded_dim = ((dimension + 63) / 64) * 64;
header_.graph.padded_dim = padded_dim;
// BinDataMap layout: bin_code (padded_dim/8) + f_add + f_rescale + f_error
header_.graph.size_bin_data =
rabitqlib::BinDataMap<float>::data_bytes(padded_dim);
// ExDataMap layout: ex_code (padded_dim*ex_bits/8) + f_add_ex + f_rescale_ex
header_.graph.size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
padded_dim, header_.graph.ex_bits);
// quantized vector format: cluster_id + bin_data + ex_data
header_.graph.vector_size =
sizeof(uint32_t) + size_bin_data() + size_ex_data();
}
int HnswRabitqEntity::CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size,
size_t *padding_size) {
*padding_size = AlignSize(data_size) - data_size;
if (*padding_size == 0) {
return 0;
}
std::string padding(*padding_size, '\0');
if (dumper->write(padding.data(), *padding_size) != *padding_size) {
LOG_ERROR("Append padding failed, size %zu", *padding_size);
return IndexError_WriteData;
}
return 0;
}
int64_t HnswRabitqEntity::dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id,
const void *data, size_t size) const {
size_t len = dumper->write(data, size);
if (len != size) {
LOG_ERROR("Dump segment %s data failed, expect: %zu, actual: %zu",
segment_id.c_str(), size, len);
return IndexError_WriteData;
}
size_t padding_size = AlignSize(size) - size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %zu", padding_size);
return IndexError_WriteData;
}
}
uint32_t crc = ailego::Crc32c::Hash(data, size);
int ret = dumper->append(segment_id, size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
return ret;
}
return len + padding_size;
}
int64_t HnswRabitqEntity::dump_header(const IndexDumper::Pointer &dumper,
const HNSWHeader &hd) const {
//! dump basic graph header. header is aligned and does not need padding
int64_t graph_hd_size =
dump_segment(dumper, kGraphHeaderSegmentId, &hd.graph, hd.graph.size);
if (graph_hd_size < 0) {
return graph_hd_size;
}
//! dump basic graph header. header is aligned and does not need padding
int64_t hnsw_hd_size =
dump_segment(dumper, kHnswHeaderSegmentId, &hd.hnsw, hd.hnsw.size);
if (hnsw_hd_size < 0) {
return hnsw_hd_size;
}
return graph_hd_size + hnsw_hd_size;
}
void HnswRabitqEntity::reshuffle_vectors(
const std::function<level_t(node_id_t)> & /*get_level*/,
std::vector<node_id_t> * /*n2o_mapping*/,
std::vector<node_id_t> * /*o2n_mapping*/, key_t * /*keys*/) const {
// TODO
return;
}
int64_t HnswRabitqEntity::dump_mapping_segment(
const IndexDumper::Pointer &dumper, const key_t *keys) const {
std::vector<node_id_t> mapping(doc_cnt());
std::iota(mapping.begin(), mapping.end(), 0U);
std::sort(mapping.begin(), mapping.end(),
[&](node_id_t i, node_id_t j) { return keys[i] < keys[j]; });
size_t size = mapping.size() * sizeof(node_id_t);
return dump_segment(dumper, kGraphMappingSegmentId, mapping.data(), size);
}
int64_t HnswRabitqEntity::dump_segments(
const IndexDumper::Pointer &dumper, key_t *keys,
const std::function<level_t(node_id_t)> &get_level) const {
HNSWHeader dump_hd(header());
dump_hd.graph.node_size = AlignSize(vector_size());
std::vector<node_id_t> n2o_mapping; // map new id to origin id
std::vector<node_id_t> o2n_mapping; // map origin id to new id
reshuffle_vectors(get_level, &n2o_mapping, &o2n_mapping, keys);
if (!o2n_mapping.empty()) {
dump_hd.hnsw.entry_point = o2n_mapping[entry_point()];
}
//! Dump header
int64_t hd_size = dump_header(dumper, dump_hd);
if (hd_size < 0) {
return hd_size;
}
//! Dump vectors
int64_t vecs_size = dump_vectors(dumper, n2o_mapping);
if (vecs_size < 0) {
return vecs_size;
}
//! Dump neighbors
auto neighbors_size =
dump_neighbors(dumper, get_level, n2o_mapping, o2n_mapping);
if (neighbors_size < 0) {
return neighbors_size;
}
//! free memory
n2o_mapping = std::vector<node_id_t>();
o2n_mapping = std::vector<node_id_t>();
//! Dump keys
size_t key_segment_size = doc_cnt() * sizeof(key_t);
int64_t keys_size =
dump_segment(dumper, kGraphKeysSegmentId, keys, key_segment_size);
if (keys_size < 0) {
return keys_size;
}
//! Dump mapping
int64_t mapping_size = dump_mapping_segment(dumper, keys);
if (mapping_size < 0) {
return mapping_size;
}
return hd_size + keys_size + vecs_size + neighbors_size + mapping_size;
}
int64_t HnswRabitqEntity::dump_vectors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const {
size_t vector_dump_size = vector_size();
size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size;
char padding[padding_size];
memset(padding, 0, sizeof(padding));
const void *data = nullptr;
uint32_t crc = 0U;
size_t vecs_size = 0UL;
//! dump vectors
for (node_id_t id = 0; id < doc_cnt(); ++id) {
data = get_vector(reorder_mapping.empty() ? id : reorder_mapping[id]);
if (ailego_unlikely(!data)) {
return IndexError_ReadData;
}
size_t len = dumper->write(data, vector_size());
if (len != vector_size()) {
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len,
vector_size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(data, vector_size(), crc);
vecs_size += vector_size();
if (padding_size == 0) {
continue;
}
len = dumper->write(padding, padding_size);
if (len != padding_size) {
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len, padding_size);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(padding, padding_size, crc);
vecs_size += padding_size;
}
int ret = dumper->append(kGraphFeaturesSegmentId, vecs_size, 0UL, crc);
if (ret != 0) {
LOG_ERROR("Dump vectors segment meta failed, ret %d", ret);
return ret;
}
return vecs_size;
}
int64_t HnswRabitqEntity::dump_graph_neighbors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
std::vector<GraphNeighborMeta> graph_meta;
graph_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t mapping[l0_neighbor_cnt()];
uint32_t min_neighbor_count = 10000;
uint32_t max_neighbor_count = 0;
size_t sum_neighbor_count = 0;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
const Neighbors neighbors =
get_neighbors(0, reorder_mapping.empty() ? id : reorder_mapping[id]);
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= l0_neighbor_cnt(),
"invalid neighbors");
uint32_t neighbor_count = neighbors.size();
if (neighbor_count < min_neighbor_count) {
min_neighbor_count = neighbor_count;
}
if (neighbor_count > max_neighbor_count) {
max_neighbor_count = neighbor_count;
}
sum_neighbor_count += neighbor_count;
graph_meta.emplace_back(offset, neighbor_count);
size_t size = neighbors.size() * sizeof(node_id_t);
const node_id_t *data = &neighbors[0];
if (!neighbor_mapping.empty()) {
for (node_id_t i = 0; i < neighbors.size(); ++i) {
mapping[i] = neighbor_mapping[neighbors[i]];
}
data = mapping;
}
if (dumper->write(data, size) != size) {
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
static_cast<size_t>(id), size);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(data, size, crc);
offset += size;
}
uint32_t average_neighbor_count = 0;
if (doc_cnt() > 0) {
average_neighbor_count = sum_neighbor_count / doc_cnt();
}
LOG_INFO(
"Dump hnsw graph: min_neighbor_count[%u] max_neighbor_count[%u] "
"average_neighbor_count[%u]",
min_neighbor_count, max_neighbor_count, average_neighbor_count);
size_t padding_size = 0;
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
if (ret != 0) {
return ret;
}
ret = dumper->append(kGraphNeighborsSegmentId, offset, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s failed, ret %d",
kGraphNeighborsSegmentId.c_str(), ret);
return ret;
}
//! dump level 0 neighbors meta
auto len = dump_segment(dumper, kGraphOffsetsSegmentId, graph_meta.data(),
graph_meta.size() * sizeof(GraphNeighborMeta));
if (len < 0) {
return len;
}
return len + offset + padding_size;
}
int64_t HnswRabitqEntity::dump_upper_neighbors(
const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
std::vector<HnswNeighborMeta> hnsw_meta;
hnsw_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t buffer[upper_neighbor_cnt() + 1];
for (node_id_t id = 0; id < doc_cnt(); ++id) {
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
auto level = get_level(new_id);
if (level == 0) {
hnsw_meta.emplace_back(0U, 0U);
continue;
}
hnsw_meta.emplace_back(offset, level);
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
const Neighbors neighbors = get_neighbors(cur_level, new_id);
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
"invalid neighbors");
memset(buffer, 0, sizeof(buffer));
buffer[0] = neighbors.size();
if (neighbor_mapping.empty()) {
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
} else {
for (node_id_t i = 0; i < neighbors.size(); ++i) {
buffer[i + 1] = neighbor_mapping[neighbors[i]];
}
}
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
static_cast<size_t>(id), sizeof(buffer));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
offset += sizeof(buffer);
}
}
size_t padding_size = 0;
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
if (ret != 0) {
return ret;
}
ret = dumper->append(kHnswNeighborsSegmentId, offset, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s failed, ret %d", kHnswNeighborsSegmentId.c_str(),
ret);
return ret;
}
//! dump level 0 neighbors meta
auto len = dump_segment(dumper, kHnswOffsetsSegmentId, hnsw_meta.data(),
hnsw_meta.size() * sizeof(HnswNeighborMeta));
if (len < 0) {
return len;
}
return len + offset + padding_size;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,614 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <execinfo.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/container/heap.h>
#include <zvec/ailego/logger/logger.h>
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_storage.h"
namespace zvec {
namespace core {
using node_id_t = uint32_t;
using key_t = uint64_t;
using level_t = int32_t;
using dist_t = float;
struct EstimateRecord {
float ip_x0_qr;
float est_dist;
float low_dist;
bool operator<(const EstimateRecord &other) const {
return this->est_dist < other.est_dist;
}
};
struct ResultRecord {
float est_dist;
float low_dist;
ResultRecord() : est_dist(0.0f), low_dist(0.0f) {}
ResultRecord(float dist) : est_dist(dist), low_dist(dist) {}
explicit ResultRecord(const EstimateRecord &other)
: est_dist(other.est_dist), low_dist(other.low_dist) {}
ResultRecord(float est_dist, float low_dist)
: est_dist(est_dist), low_dist(low_dist) {}
bool operator<(const ResultRecord &other) const {
return this->est_dist < other.est_dist;
}
bool operator<=(const ResultRecord &other) const {
return this->est_dist <= other.est_dist;
}
bool operator>(const ResultRecord &other) const {
return this->est_dist > other.est_dist;
}
};
using TopkHeap = ailego::KeyValueHeap<node_id_t, ResultRecord>;
using CandidateHeap =
ailego::KeyValueHeap<node_id_t, ResultRecord, std::greater<ResultRecord>>;
constexpr node_id_t kInvalidNodeId = static_cast<node_id_t>(-1);
constexpr key_t kInvalidKey = static_cast<key_t>(-1);
class DistCalculator;
struct GraphHeader {
uint32_t size;
uint32_t version;
uint32_t graph_type;
uint32_t doc_count;
uint32_t vector_size;
uint32_t node_size;
uint32_t l0_neighbor_count;
uint32_t prune_type;
uint32_t prune_neighbor_count;
uint32_t ef_construction;
uint32_t options;
uint32_t min_neighbor_count;
uint32_t padded_dim;
uint32_t size_bin_data;
uint32_t size_ex_data;
uint8_t ex_bits;
uint8_t reserved_[4067];
};
static_assert(sizeof(GraphHeader) % 32 == 0,
"GraphHeader must be aligned with 32 bytes");
//! Hnsw upper neighbor header
struct HnswHeader {
uint32_t size; // header size
uint32_t revision; // current total docs of the graph
uint32_t upper_neighbor_count;
uint32_t ef_construction;
uint32_t scaling_factor;
uint32_t max_level;
uint32_t entry_point;
uint32_t options;
uint8_t reserved_[30];
};
static_assert(sizeof(HnswHeader) % 32 == 0,
"GraphHeader must be aligned with 32 bytes");
//! Hnsw common header and upper neighbor header
struct HNSWHeader {
HNSWHeader() {
clear();
}
HNSWHeader(const HNSWHeader &header) {
memcpy(this, &header, sizeof(header));
}
HNSWHeader &operator=(const HNSWHeader &header) {
memcpy(this, &header, sizeof(header));
return *this;
}
//! Reset state to zero, and the params is untouched
void inline reset() {
graph.doc_count = 0U;
hnsw.entry_point = kInvalidNodeId;
hnsw.max_level = 0;
}
//! Clear all fields to init value
void inline clear() {
memset(this, 0, sizeof(HNSWHeader));
hnsw.entry_point = kInvalidNodeId;
graph.size = sizeof(GraphHeader);
hnsw.size = sizeof(HnswHeader);
}
size_t l0_neighbor_cnt() const {
return graph.l0_neighbor_count;
}
size_t upper_neighbor_cnt() const {
return hnsw.upper_neighbor_count;
}
size_t vector_size() const {
return graph.vector_size;
}
uint8_t ex_bits() const {
return graph.ex_bits;
}
uint32_t padded_dim() const {
return graph.padded_dim;
}
size_t ef_construction() const {
return graph.ef_construction;
}
size_t scaling_factor() const {
return hnsw.scaling_factor;
}
size_t neighbor_prune_cnt() const {
return graph.prune_neighbor_count;
}
node_id_t entry_point() const {
return hnsw.entry_point;
}
node_id_t doc_cnt() const {
return graph.doc_count;
}
GraphHeader graph;
HnswHeader hnsw;
};
struct NeighborsHeader {
uint32_t neighbor_cnt;
node_id_t neighbors[0];
};
struct Neighbors {
Neighbors() : cnt{0}, data{nullptr} {}
Neighbors(uint32_t cnt_in, const node_id_t *data_in)
: cnt{cnt_in}, data{data_in} {}
Neighbors(IndexStorage::MemoryBlock &&mem_block)
: neighbor_block{std::move(mem_block)} {
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
cnt = hd->neighbor_cnt;
data = hd->neighbors;
}
size_t size(void) const {
return cnt;
}
const node_id_t &operator[](size_t idx) const {
return data[idx];
}
uint32_t cnt;
const node_id_t *data;
IndexStorage::MemoryBlock neighbor_block;
};
//! level 0 neighbors offset
struct GraphNeighborMeta {
GraphNeighborMeta(size_t o, size_t cnt) : offset(o), neighbor_cnt(cnt) {}
uint64_t offset : 48;
uint64_t neighbor_cnt : 16;
};
//! hnsw upper neighbors meta
struct HnswNeighborMeta {
HnswNeighborMeta(size_t o, size_t l) : offset(o), level(l) {}
uint64_t offset : 48; // offset = idx * upper neighors size
uint64_t level : 16;
};
class HnswRabitqEntity {
public:
//! Constructor
HnswRabitqEntity() {}
//! Constructor
HnswRabitqEntity(const HNSWHeader &hd) {
header_ = hd;
}
//! Destructor
virtual ~HnswRabitqEntity() {}
//! HnswRabitqEntity Pointerd;
typedef std::shared_ptr<HnswRabitqEntity> Pointer;
//! Get max neighbor size of graph level
inline size_t neighbor_cnt(level_t level) const {
return level == 0 ? header_.graph.l0_neighbor_count
: header_.hnsw.upper_neighbor_count;
}
//! get max neighbor size of graph level 0
inline size_t l0_neighbor_cnt() const {
return header_.graph.l0_neighbor_count;
}
//! get min neighbor size of graph
inline size_t min_neighbor_cnt() const {
return header_.graph.min_neighbor_count;
}
//! get upper neighbor size of graph level other than 0
inline size_t upper_neighbor_cnt() const {
return header_.hnsw.upper_neighbor_count;
}
//! Get current total doc of the hnsw graph
inline node_id_t *mutable_doc_cnt() {
return &header_.graph.doc_count;
}
inline node_id_t doc_cnt() const {
return header_.graph.doc_count;
}
//! Get hnsw graph scaling params
inline size_t scaling_factor() const {
return header_.hnsw.scaling_factor;
}
//! Get prune_size
inline size_t prune_cnt() const {
return header_.graph.prune_neighbor_count;
}
//! Current entity of top level graph
inline node_id_t entry_point() const {
return header_.hnsw.entry_point;
}
//! Current max graph level
inline level_t cur_max_level() const {
return header_.hnsw.max_level;
}
//! Retrieve index vector size
size_t vector_size() const {
return header_.graph.vector_size;
}
//! Retrieve node size
size_t node_size() const {
return header_.graph.node_size;
}
//! Retrieve ef constuction
size_t ef_construction() const {
return header_.graph.ef_construction;
}
uint8_t ex_bits() const {
return header_.graph.ex_bits;
}
uint32_t padded_dim() const {
return header_.graph.padded_dim;
}
uint32_t size_bin_data() const {
return header_.graph.size_bin_data;
}
uint32_t size_ex_data() const {
return header_.graph.size_ex_data;
}
void update_rabitq_params_and_vector_size(uint32_t dimension);
void set_ex_bits(uint8_t ex_bits) {
header_.graph.ex_bits = ex_bits;
}
void set_prune_cnt(size_t v) {
header_.graph.prune_neighbor_count = v;
}
void set_scaling_factor(size_t val) {
header_.hnsw.scaling_factor = val;
}
void set_l0_neighbor_cnt(size_t cnt) {
header_.graph.l0_neighbor_count = cnt;
}
void set_min_neighbor_cnt(size_t cnt) {
header_.graph.min_neighbor_count = cnt;
}
void set_upper_neighbor_cnt(size_t cnt) {
header_.hnsw.upper_neighbor_count = cnt;
}
void set_ef_construction(size_t ef) {
header_.graph.ef_construction = ef;
}
protected:
inline const HNSWHeader &header() const {
return header_;
}
inline HNSWHeader *mutable_header() {
return &header_;
}
inline size_t header_size() const {
return sizeof(header_);
}
void set_node_size(size_t size) {
header_.graph.node_size = size;
}
//! Dump all segment by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_segments(
const IndexDumper::Pointer &dumper, key_t *keys,
const std::function<level_t(node_id_t)> &get_level) const;
private:
//! dump mapping segment, for get_vector_by_key in provider
int64_t dump_mapping_segment(const IndexDumper::Pointer &dumper,
const key_t *keys) const;
//! dump hnsw head by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_header(const IndexDumper::Pointer &dumper,
const HNSWHeader &hd) const;
//! dump vectors by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_vectors(const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const;
//! dump hnsw neighbors by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_neighbors(const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
auto len1 = dump_graph_neighbors(dumper, reorder_mapping, neighbor_mapping);
if (len1 < 0) {
return len1;
}
auto len2 = dump_upper_neighbors(dumper, get_level, reorder_mapping,
neighbor_mapping);
if (len2 < 0) {
return len2;
}
return len1 + len2;
}
//! dump segment by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id, const void *data,
size_t size) const;
//! Dump level 0 neighbors
//! Return dump size if success, errno(<0) in failure
int64_t dump_graph_neighbors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const;
//! Dump upper level neighbors
//! Return dump size if success, errno(<0) in failure
int64_t dump_upper_neighbors(
const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const;
public:
//! Cleanup the entity
virtual int cleanup(void) {
header_.clear();
return 0;
}
//! Make a copy of searcher entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const {
LOG_ERROR("Update neighbors not implemented");
return HnswRabitqEntity::Pointer();
}
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const = 0;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const = 0;
//! Get vectors feature data by keys
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const = 0;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const = 0;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const = 0;
//! Retrieve a vector using a primary key
virtual const void *get_vector_by_key(uint64_t /*key*/) const {
LOG_ERROR("get vector not implemented");
return nullptr;
}
virtual int get_vector_by_key(const key_t /*key*/,
IndexStorage::MemoryBlock & /*block*/) const {
return IndexError_NotImplemented;
}
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level, node_id_t id) const = 0;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t /*level*/, key_t /*key*/, const void * /*vec*/,
node_id_t * /*id*/) {
return IndexError_NotImplemented;
}
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t /*level*/, node_id_t /*id*/,
const void * /*vec*/) {
return IndexError_NotImplemented;
}
virtual int update_neighbors(
level_t /*level*/, node_id_t /*id*/,
const std::vector<std::pair<node_id_t, ResultRecord>> & /*neighbors*/) {
LOG_ERROR("Update neighbors dense not implemented");
return 0;
}
//! Append neighbor_id to node id neighbors on level, size is the current
//! neighbors size. Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t /*level*/, node_id_t /*id*/,
uint32_t /*size*/, node_id_t /*neighbor_id*/) {
LOG_ERROR("Add neighbor not implemented");
}
//! Update entry point and max level
virtual void update_ep_and_level(node_id_t ep, level_t level) {
header_.hnsw.entry_point = ep;
header_.hnsw.max_level = level;
}
virtual int load(const IndexStorage::Pointer & /*container*/,
bool /*check_crc*/) {
LOG_ERROR("Load not implemented");
return IndexError_NotImplemented;
}
virtual int dump(const IndexDumper::Pointer & /*dumper*/) {
LOG_ERROR("Dump not implemented");
return IndexError_NotImplemented;
}
static int CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size, size_t *padding_size);
uint32_t get_cluster_id(const void *vec) const {
return *reinterpret_cast<const uint32_t *>(
reinterpret_cast<const char *>(vec) + cluster_id_offset());
}
const char *get_bin_data(const void *vec) const {
return reinterpret_cast<const char *>(vec) + bin_data_offset();
}
const char *get_ex_data(const void *vec) const {
return reinterpret_cast<const char *>(vec) + ex_data_offset();
}
uint32_t cluster_id_offset() const {
return 0;
}
uint32_t bin_data_offset() const {
return cluster_id_offset() + sizeof(uint32_t);
}
uint32_t ex_data_offset() const {
return bin_data_offset() + size_bin_data();
}
protected:
static inline size_t AlignSize(size_t size) {
return (size + 0x1F) & (~0x1F);
}
static inline size_t AlignPageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::PageSize() - 1;
return (size + page_mask) & (~page_mask);
}
static inline size_t AlignHugePageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::HugePageSize() - 1;
return (size + page_mask) & (~page_mask);
}
//! rearrange vectors to improve cache locality
void reshuffle_vectors(const std::function<level_t(node_id_t)> &get_level,
std::vector<node_id_t> *n2o_mapping,
std::vector<node_id_t> *o2n_mapping,
key_t *keys) const;
public:
const static std::string kGraphHeaderSegmentId;
const static std::string kGraphFeaturesSegmentId;
const static std::string kGraphKeysSegmentId;
const static std::string kGraphNeighborsSegmentId;
const static std::string kGraphOffsetsSegmentId;
const static std::string kGraphMappingSegmentId;
const static std::string kHnswHeaderSegmentId;
const static std::string kHnswNeighborsSegmentId;
const static std::string kHnswOffsetsSegmentId;
constexpr static uint32_t kRevision = 0U;
constexpr static size_t kMaxGraphLayers = 15;
constexpr static uint32_t kDefaultEfConstruction = 500;
constexpr static uint32_t kDefaultEf = 500;
constexpr static uint32_t kDefaultUpperMaxNeighborCnt = 50; // M of HNSW
constexpr static uint32_t kDefaultL0MaxNeighborCnt = 100;
constexpr static uint32_t kMaxNeighborCnt = 65535;
constexpr static float kDefaultScanRatio = 0.1f;
constexpr static uint32_t kDefaultMinScanLimit = 10000;
constexpr static uint32_t kDefaultMaxScanLimit =
std::numeric_limits<uint32_t>::max();
constexpr static float kDefaultBFNegativeProbability = 0.001f;
constexpr static uint32_t kDefaultScalingFactor = 50U;
constexpr static uint32_t kDefaultBruteForceThreshold = 1000U;
constexpr static uint32_t kDefaultDocsHardLimit = 1 << 30U; // 1 billion
constexpr static float kDefaultDocsSoftLimitRatio = 0.9f;
constexpr static size_t kMaxChunkSize = 0xFFFFFFFF;
constexpr static size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
constexpr static size_t kDefaultMaxChunkCnt = 50000UL;
constexpr static float kDefaultNeighborPruneMultiplier =
1.0f; // prune_cnt = upper_max_neighbor_cnt * multiplier
constexpr static float kDefaultL0MaxNeighborCntMultiplier =
2.0f; // l0_max_neighbor_cnt = upper_max_neighbor_cnt * multiplier
protected:
HNSWHeader header_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,231 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "hnsw_rabitq_chunk.h"
namespace zvec {
namespace core {
//! Persistent hashmap implement through open addressing algorithm
template <class Key, class Val, Val EmptyVal = 0U,
typename =
typename std::enable_if<std::is_integral<Key>::value>::type>
class HnswIndexHashMap {
using key_type = Key;
using val_type = Val;
struct Iterator {
key_type first;
val_type second;
};
typedef Iterator *iterator;
typedef Iterator Item;
typedef const Iterator *const_iterator;
class Slot {
public:
Slot(Chunk::Pointer &&chunk, const void *data)
: chunk_(std::move(chunk)),
items_(reinterpret_cast<const Item *>(data)) {}
//! Return a empty loc or the key item loc
Slot(Chunk::Pointer &&chunk, IndexStorage::MemoryBlock &&mem_block)
: chunk_(std::move(chunk)), items_block_(std::move(mem_block)) {
items_ = reinterpret_cast<const Item *>(items_block_.data());
}
const_iterator find(key_type key, uint32_t max_items, uint32_t mask) const {
auto it = &items_[key & mask];
for (auto i = 0U; i < max_items; ++i) {
if (it->first == key || it->second == EmptyVal) {
// LOG_DEBUG("i=%u", i);
return it;
}
++it;
if (it == &items_[max_items]) {
it = &items_[0];
}
}
return nullptr;
}
bool update(const_iterator it) {
uint32_t offset = reinterpret_cast<const uint8_t *>(it) -
reinterpret_cast<const uint8_t *>(&items_[0]);
if (ailego_unlikely(chunk_->write(offset, it, sizeof(Item)) !=
sizeof(Item))) {
LOG_ERROR("Chunk write failed");
return false;
}
return true;
}
private:
Chunk::Pointer chunk_{};
const Item *items_{nullptr}; // point to chunk data
IndexStorage::MemoryBlock items_block_{};
};
public:
//! Init the hash
//! broker the index allocator
//! chunk_size the size of per chunk allocated, actual size may greater
//! factor factor = 1/ratio, ratio is the probability of a squence
//! number inserted to this container
//! max the max number key can be inserted
//! expansion_ratio memory expansion ratio
int init(HnswRabitqChunkBroker::Pointer &broker, uint32_t chunk_size,
uint32_t factor, size_t max, float expansion_ratio) {
ailego_assert_with(expansion_ratio > 1.0f, "ratio must > 1.0f");
broker_ = broker;
size_t items = std::ceil(chunk_size * 1.0f / sizeof(Item));
slot_items_ = 1UL << static_cast<size_t>((std::ceil(std::log2(items))));
size_t range = slot_items_ * factor / expansion_ratio;
mask_bits_ = std::floor(std::log2(range));
range = 1UL << mask_bits_;
size_t max_slots = std::ceil(max * 1.0f / range);
slots_.reserve(max_slots);
slot_loc_mask_ = slot_items_ - 1U;
int ret = load();
if (ret != 0) {
return ret;
}
LOG_DEBUG(
"HnswRabitqIndexHash init, chunkSize=%u factor=%u max=%zu "
"ratio=%f slotItems=%u maxSlots=%zu maskBits=%u "
"range=%zu",
chunk_size, factor, max, expansion_ratio, slot_items_, max_slots,
mask_bits_, range);
return 0;
}
int cleanup(void) {
broker_.reset();
slots_.clear();
slots_.shrink_to_fit();
mask_bits_ = 0U;
slot_items_ = 0U;
slot_loc_mask_ = 0U;
return 0;
}
const_iterator end(void) const {
return nullptr;
}
const_iterator find(const key_type key) const {
auto idx = key >> mask_bits_;
if (idx >= slots_.size()) {
return end();
}
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
return it && it->second != EmptyVal ? it : nullptr;
}
bool insert(key_type key, val_type val) {
auto idx = key >> mask_bits_;
if (idx >= slots_.size()) {
if (ailego_unlikely(idx >= slots_.capacity())) {
LOG_ERROR("no space to insert");
return false;
}
for (auto i = slots_.size(); i <= idx; ++i) {
if (ailego_unlikely(!alloc_slot(i))) {
return false;
}
}
}
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
if (ailego_unlikely(it == nullptr)) {
LOG_ERROR("no space to insert");
return false;
}
//! TODO: write memory is ok?
const_cast<iterator>(it)->first = key;
const_cast<iterator>(it)->second = val;
return slots_[idx].update(it);
}
private:
bool alloc_slot(size_t idx) {
ailego_assert_with(idx == slots_.size(), "invalid idx");
size_t size = slot_items_ * sizeof(Item);
auto p = broker_->alloc_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, idx, size);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return false;
}
Chunk::Pointer chunk = p.second;
if (ailego_unlikely(chunk->resize(size) != size)) {
LOG_ERROR("Chunk resize failed, size=%zu", size);
return false;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
return true;
}
int load(void) {
size_t slots_cnt = broker_->get_chunk_cnt(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX);
for (size_t i = 0UL; i < slots_cnt; ++i) {
auto chunk = broker_->get_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, i);
if (!chunk) {
LOG_ERROR("Get chunk failed, seq=%zu", i);
return IndexError_InvalidFormat;
}
size_t size = sizeof(Item) * slot_items_;
if (chunk->data_size() < size) {
LOG_ERROR(
"Hash params may be mismatch, seq=%zu, data_size=%zu "
"expect=%zu",
i, chunk->data_size(), size);
return IndexError_InvalidFormat;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
}
return 0;
}
private:
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
std::vector<Slot> slots_{};
uint32_t mask_bits_{0U};
uint32_t slot_items_{}; // must be a power of 2
uint32_t slot_loc_mask_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,134 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_searcher.h"
#include "zvec/core/framework/index_streamer.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqIndexProvider : public IndexProvider {
public:
HnswRabitqIndexProvider(const IndexMeta &meta,
const HnswRabitqEntity::Pointer &entity,
const std::string &owner)
: meta_(meta), entity_(entity), owner_class_(owner) {}
HnswRabitqIndexProvider(const HnswRabitqIndexProvider &) = delete;
HnswRabitqIndexProvider &operator=(const HnswRabitqIndexProvider &) = delete;
public: // holder interface
//! Create a new iterator
IndexProvider::Iterator::Pointer create_iterator() override {
return HnswRabitqIndexProvider::Iterator::Pointer(new (std::nothrow)
Iterator(entity_));
}
//! Retrieve count of vectors
size_t count(void) const override {
return entity_->doc_cnt();
}
//! Retrieve dimension of vector
size_t dimension(void) const override {
return meta_.dimension();
}
//! Retrieve type of vector
IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
//! Retrieve vector size in bytes
size_t element_size(void) const override {
return meta_.element_size();
}
public: // provider's unique interface
//! Retrieve a vector using a primary key
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_class_;
}
private:
class Iterator : public IndexProvider::Iterator {
public:
Iterator(const HnswRabitqEntity::Pointer &entity)
: entity_(entity), cur_id_(0U) {}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
// cur_id_ += 1;
cur_id_ = get_next_valid_id(cur_id_ + 1);
}
//! Reset the iterator
void reset(void) {
cur_id_ = get_next_valid_id(0);
}
private:
node_id_t get_next_valid_id(node_id_t start_id) {
for (node_id_t i = start_id; i < entity_->doc_cnt(); i++) {
if (entity_->get_key(i) != kInvalidNodeId) {
cur_id_ = i;
return i;
}
}
return kInvalidNodeId;
}
private:
const HnswRabitqEntity::Pointer entity_;
node_id_t cur_id_;
};
private:
const IndexMeta &meta_;
const HnswRabitqEntity::Pointer entity_;
const std::string owner_class_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,121 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
inline const std::string PARAM_HNSW_RABITQ_GENERAL_DIMENSION(
"proxima.hnsw_rabitq.general.dimension");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT(
"proxima.hnsw_rabitq.builder.thread_count");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA(
"proxima.hnsw_rabitq.builder.memory_quota");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.builder.efconstruction");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR(
"proxima.hnsw_rabitq.builder.scaling_factor");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS(
"proxima.hnsw_rabitq.builder.check_interval_secs");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER(
"proxima.hnsw_rabitq.builder.neighbor_prune_multiplier");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.builder.min_neighbor_count");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.builder.max_neighbor_count");
inline const std::string
PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
"proxima.hnsw_rabitq.builder.l0_max_neighbor_count_multiplier");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_EF(
"proxima.hnsw_rabitq.searcher.ef");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD(
"proxima.hnsw_rabitq.searcher.brute_force_threshold");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE(
"proxima.hnsw_rabitq.searcher.neighbors_in_memory_enable");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO(
"proxima.hnsw_rabitq.searcher.max_scan_ratio");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE(
"proxima.hnsw_rabitq.searcher.check_crc_enable");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE(
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_enable");
inline const std::string
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_negative_prob");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE(
"proxima.hnsw_rabitq.searcher.force_padding_result_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO(
"proxima.hnsw_rabitq.streamer.max_scan_ratio");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT(
"proxima.hnsw_rabitq.streamer.min_scan_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT(
"proxima.hnsw_rabitq.streamer.max_scan_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EF(
"proxima.hnsw_rabitq.streamer.ef");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.streamer.efconstruction");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.streamer.max_neighbor_count");
inline const std::string
PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
"proxima.hnsw_rabitq.streamer.l0_max_neighbor_count_multiplier");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR(
"proxima.hnsw_rabitq.streamer.scaling_factor");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD(
"proxima.hnsw_rabitq.streamer.brute_force_threshold");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT(
"proxima.hnsw_rabitq.streamer.docs_hard_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT(
"proxima.hnsw_rabitq.streamer.docs_soft_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE(
"proxima.hnsw_rabitq.streamer.max_index_size");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE(
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_enable");
inline const std::string
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_negative_prob");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE(
"proxima.hnsw_rabitq.streamer.check_crc_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER(
"proxima.hnsw_rabitq.streamer.neighbor_prune_multiplier");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE(
"proxima.hnsw_rabitq.streamer.chunk_size");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY(
"proxima.hnsw_rabitq.streamer.filter_same_key");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE(
"proxima.hnsw_rabitq.streamer.get_vector_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.streamer.min_neighbor_count");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE(
"proxima.hnsw_rabitq.streamer.force_padding_result_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT(
"proxima.hnsw_rabitq.streamer.estimate_doc_count");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP(
"proxima.hnsw_rabitq.streamer.use_id_map");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_WORKING_PATH(
"proxima.hnsw_rabitq.reducer.working_path");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_NUM_OF_ADD_THREADS(
"proxima.hnsw_rabitq.reducer.num_of_add_threads");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_INDEX_NAME(
"proxima.hnsw_rabitq.reducer.index_name");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.reducer.efconstruction");
} // namespace core
} // namespace zvec
@@ -0,0 +1,354 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_query_algorithm.h"
#include <chrono>
#include <iostream>
#include <ailego/internal/cpu_features.h>
#include <rabitqlib/index/estimator.hpp>
#include "zvec/ailego/internal/platform.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_query_entity.h"
namespace zvec {
namespace core {
HnswRabitqQueryAlgorithm::HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
size_t num_clusters,
RabitqMetricType metric_type)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt),
num_clusters_(num_clusters),
metric_type_(metric_type) {
ex_bits_ = entity_.ex_bits();
padded_dim_ = entity_.padded_dim();
ip_func_ = rabitqlib::select_excode_ipfunc(ex_bits_);
LOG_INFO(
"Create query algorithm. num_clusters=%zu ex_bits=%zu padded_dim=%zu",
num_clusters_, ex_bits_, padded_dim_);
}
int HnswRabitqQueryAlgorithm::cleanup() {
return 0;
}
int HnswRabitqQueryAlgorithm::search(HnswRabitqQueryEntity *entity,
HnswRabitqContext *ctx) const {
spin_lock_.lock();
auto maxLevel = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
spin_lock_.unlock();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
return 0;
}
EstimateRecord curest;
get_bin_est(entity_.get_vector(entry_point), curest, *entity);
for (level_t cur_level = maxLevel; cur_level >= 1; --cur_level) {
select_entry_point(cur_level, &entry_point, &curest, ctx, entity);
}
auto &topk_heap = ctx->topk_heap();
topk_heap.clear();
search_neighbors(0, &entry_point, &curest, topk_heap, ctx, entity);
if (ctx->group_by_search()) {
expand_neighbors_by_group(topk_heap, ctx, entity);
}
return 0;
}
//! select_entry_point on hnsw level, ef = 1
void HnswRabitqQueryAlgorithm::select_entry_point(
level_t level, node_id_t *entry_point, EstimateRecord *curest,
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
auto &entity = ctx->get_entity();
while (true) {
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
ailego_prefetch(neighbors.data);
uint32_t size = neighbors.size();
if (size == 0) {
break;
}
bool find_closer = false;
for (uint32_t i = 0; i < size; ++i) {
EstimateRecord candest;
get_bin_est(entity_.get_vector(neighbors[i]), candest, *query_entity);
if (candest.est_dist < curest->est_dist) {
*curest = candest;
*entry_point = neighbors[i];
find_closer = true;
}
}
if (!find_closer) {
break;
}
}
return;
}
void HnswRabitqQueryAlgorithm::search_neighbors(
level_t level, node_id_t *entry_point, EstimateRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
const auto &entity = ctx->get_entity();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
candidates.clear();
visit.clear();
visit.set_visited(*entry_point);
if (!filter(*entry_point)) {
topk.emplace(*entry_point, ResultRecord(*dist));
}
candidates.emplace(*entry_point, ResultRecord(*dist));
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
auto main_dist = top->second;
if (topk.full() && main_dist.est_dist > topk[0].second.est_dist) {
break;
}
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(level, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
EstimateRecord candest;
auto *cand_vector = entity_.get_vector(node);
ailego_prefetch(cand_vector);
get_bin_est(cand_vector, candest, *query_entity);
if (ex_bits_ > 0) {
// Check preliminary score against current worst full estimate.
bool flag_update_KNNs =
(!topk.full()) || candest.low_dist < topk[0].second.est_dist;
if (flag_update_KNNs) {
// Compute the full estimate if promising.
get_full_est(cand_vector, candest, *query_entity);
} else {
continue;
}
} else {
// ex_bits_ == 0: est_dist is already the best estimate
if (topk.full() && candest.est_dist >= topk[0].second.est_dist) {
continue;
}
}
candidates.emplace(node, ResultRecord(candest));
// update entry_point for next level scan
if (candest < *dist) {
*entry_point = node;
*dist = candest;
}
if (!filter(node)) {
topk.emplace(node, ResultRecord(candest));
}
} // end for
} // while
return;
}
void HnswRabitqQueryAlgorithm::expand_neighbors_by_group(
TopkHeap &topk, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *query_entity) const {
if (!ctx->group_by().is_valid()) {
return;
}
const auto &entity = ctx->get_entity();
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity.get_key(id));
};
// devide into groups
std::map<std::string, TopkHeap> &group_topk_heaps = ctx->group_topk_heaps();
for (uint32_t i = 0; i < topk.size(); ++i) {
node_id_t id = topk[i].first;
auto score = topk[i].second;
std::string group_id = group_by(id);
auto &topk_heap = group_topk_heaps[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, score);
}
// stage 2, expand to reach group num as possible
if (group_topk_heaps.size() < ctx->group_num()) {
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
// refill to get enough groups
candidates.clear();
visit.clear();
for (uint32_t i = 0; i < topk.size(); ++i) {
node_id_t id = topk[i].first;
auto score = topk[i].second;
visit.set_visited(id);
candidates.emplace_back(id, score);
}
// do expand
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(0, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
EstimateRecord candest;
auto *cand_vector = entity_.get_vector(node);
ailego_prefetch(cand_vector);
get_full_est(cand_vector, candest, *query_entity);
if (!filter(node)) {
std::string group_id = group_by(node);
auto &topk_heap = group_topk_heaps[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(node, ResultRecord(candest));
if (group_topk_heaps.size() >= ctx->group_num()) {
break;
}
}
candidates.emplace(node, ResultRecord(candest));
} // end for
} // end while
} // end if
}
void HnswRabitqQueryAlgorithm::get_bin_est(
const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
const auto &q_to_centroids = entity.q_to_centroids;
auto &query_wrapper = *entity.query_wrapper;
uint32_t cluster_id = entity_.get_cluster_id(vector);
const char *bin_data = entity_.get_bin_data(vector);
if (metric_type_ == RabitqMetricType::kIP) {
float norm = q_to_centroids[cluster_id];
float error = q_to_centroids[cluster_id + num_clusters_];
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
res.ip_x0_qr, res.est_dist, res.low_dist,
-norm, error);
} else {
// L2 distance
float norm = q_to_centroids[cluster_id];
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
res.ip_x0_qr, res.est_dist, res.low_dist,
norm * norm, norm);
}
}
void HnswRabitqQueryAlgorithm::get_full_est(
const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
const auto &q_to_centroids = entity.q_to_centroids;
auto &query_wrapper = *entity.query_wrapper;
uint32_t cluster_id = entity_.get_cluster_id(vector);
const char *bin_data = entity_.get_bin_data(vector);
const char *ex_data = entity_.get_ex_data(vector);
if (metric_type_ == RabitqMetricType::kIP) {
float norm = q_to_centroids[cluster_id];
float error = q_to_centroids[cluster_id + num_clusters_];
rabitqlib::split_single_fulldist(bin_data, ex_data, ip_func_, query_wrapper,
padded_dim_, ex_bits_, res.est_dist,
res.low_dist, res.ip_x0_qr, -norm, error);
} else {
// L2 distance
float norm = q_to_centroids[cluster_id];
rabitqlib::split_single_fulldist(
bin_data, ex_data, ip_func_, query_wrapper, padded_dim_, ex_bits_,
res.est_dist, res.low_dist, res.ip_x0_qr, norm * norm, norm);
}
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,136 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <ailego/parallel/lock.h>
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class HnswRabitqQueryEntity;
//! hnsw graph algorithm implement
class HnswRabitqQueryAlgorithm {
public:
typedef std::unique_ptr<HnswRabitqQueryAlgorithm> UPointer;
public:
//! Constructor
explicit HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
size_t num_clusters,
RabitqMetricType metric_type);
//! Destructor
~HnswRabitqQueryAlgorithm() = default;
//! Cleanup HnswRabitqQueryAlgorithm
int cleanup();
//! do knn search in graph
//! return 0 on success, or errCode in failure. results saved in ctx
int search(HnswRabitqQueryEntity *entity, HnswRabitqContext *ctx) const;
//! Initiate HnswRabitqQueryAlgorithm
int init() {
level_probas_.clear();
double level_mult =
1 / std::log(static_cast<double>(entity_.scaling_factor()));
for (int level = 0;; level++) {
// refers faiss get_random_level alg
double proba =
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
if (proba < 1e-9) {
break;
}
level_probas_.push_back(proba);
}
return 0;
}
//! Generate a random level
//! return graph level
uint32_t get_random_level() const {
// gen rand float (0, 1)
double f = mt_() / static_cast<float>(mt_.max());
for (size_t level = 0; level < level_probas_.size(); level++) {
if (f < level_probas_[level]) {
return level;
}
f -= level_probas_[level];
}
return level_probas_.size() - 1;
}
void get_full_est(node_id_t id, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
return get_full_est(entity_.get_vector(id), res, entity);
}
private:
//! Select in upper layer to get entry point for next layer search
void select_entry_point(level_t level, node_id_t *entry_point,
EstimateRecord *dist, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *entity) const;
//! Given a node id and level, search the nearest neighbors in graph
//! Note: the nearest neighbors result keeps in topk, and entry_point and
//! dist will be updated to current level nearest node id and distance
void search_neighbors(level_t level, node_id_t *entry_point,
EstimateRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx,
HnswRabitqQueryEntity *entity) const;
//! expand neighbors until group nums are reached
void expand_neighbors_by_group(TopkHeap &topk, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *query_entity) const;
void get_full_est(const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const;
void get_bin_est(const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const;
private:
HnswRabitqQueryAlgorithm(const HnswRabitqQueryAlgorithm &) = delete;
HnswRabitqQueryAlgorithm &operator=(const HnswRabitqQueryAlgorithm &) =
delete;
private:
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
HnswRabitqEntity &entity_;
mutable std::mt19937 mt_{};
std::vector<double> level_probas_{};
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
std::mutex mutex_{}; // global mutex
// TODO: spin lock?
std::vector<std::mutex> lock_pool_{};
size_t num_clusters_{0};
RabitqMetricType metric_type_{RabitqMetricType::kL2};
size_t padded_dim_{0};
size_t ex_bits_{0};
float (*ip_func_)(const float *, const uint8_t *, size_t);
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,28 @@
// Copyright 2025-present the centaurdb project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/index/query.hpp>
namespace zvec::core {
struct HnswRabitqQueryEntity {
std::vector<float> rotated_query;
std::vector<float> q_to_centroids;
std::unique_ptr<rabitqlib::SplitSingleQuery<float>> query_wrapper;
};
} // namespace zvec::core
@@ -0,0 +1,28 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#include "hnsw_rabitq_builder.h"
#include "hnsw_rabitq_searcher.h"
#include "hnsw_rabitq_streamer.h"
#include "rabitq_converter.h"
#include "rabitq_reformer.h"
namespace zvec::core {
INDEX_FACTORY_REGISTER_STREAMER(HnswRabitqStreamer);
INDEX_FACTORY_REGISTER_REFORMER_ALIAS(RabitqReformer, RabitqReformer);
INDEX_FACTORY_REGISTER_SEARCHER(HnswRabitqSearcher);
INDEX_FACTORY_REGISTER_CONVERTER_ALIAS(RabitqConverter, RabitqConverter);
INDEX_FACTORY_REGISTER_BUILDER(HnswRabitqBuilder);
} // namespace zvec::core
@@ -0,0 +1,516 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_searcher.h"
#include <rabitqlib/utils/rotator.hpp>
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_index_provider.h"
#include "hnsw_rabitq_params.h"
#include "hnsw_rabitq_query_entity.h"
#include "hnsw_rabitq_searcher_entity.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
HnswRabitqSearcher::HnswRabitqSearcher() {}
HnswRabitqSearcher::~HnswRabitqSearcher() {}
int HnswRabitqSearcher::init(const ailego::Params &search_params) {
params_ = search_params;
params_.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE,
&bf_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE, &check_crc_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE,
&neighbors_in_memory_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
&bf_negative_probability_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
if (ef_ == 0) {
ef_ = HnswRabitqEntity::kDefaultEf;
}
if (bf_negative_probability_ <= 0.0f || bf_negative_probability_ >= 1.0f) {
LOG_ERROR(
"[%s] must be in range (0,1)",
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
return IndexError_InvalidArgument;
}
entity_.set_neighbors_in_memory(neighbors_in_memory_enabled_);
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
int ret = reformer_.init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
state_ = STATE_INITED;
LOG_DEBUG(
"Init params: ef=%u maxScanRatio=%f bfEnabled=%u checkCrcEnabled=%u "
"neighborsInMemoryEnabled=%u bfNagtiveProb=%f bruteForceThreshold=%u "
"forcePadding=%u",
ef_, max_scan_ratio_, bf_enabled_, check_crc_enabled_,
neighbors_in_memory_enabled_, bf_negative_probability_,
bruteforce_threshold_, force_padding_topk_enabled_);
return 0;
}
void HnswRabitqSearcher::print_debug_info() {
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
Neighbors neighbours = entity_.get_neighbors(0, id);
std::cout << "node: " << id << "; ";
for (uint32_t i = 0; i < neighbours.size(); ++i) {
std::cout << neighbours[i];
if (i == neighbours.size() - 1) {
std::cout << std::endl;
} else {
std::cout << ", ";
}
}
}
}
int HnswRabitqSearcher::cleanup() {
LOG_INFO("Begin HnswRabitqSearcher:cleanup");
metric_.reset();
meta_.clear();
stats_.clear_attributes();
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
max_scan_num_ = 0U;
ef_ = HnswRabitqEntity::kDefaultEf;
bf_enabled_ = false;
bf_negative_probability_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
check_crc_enabled_ = false;
neighbors_in_memory_enabled_ = false;
entity_.cleanup();
state_ = STATE_INIT;
LOG_INFO("End HnswRabitqSearcher:cleanup");
return 0;
}
int HnswRabitqSearcher::load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) {
if (state_ != STATE_INITED) {
LOG_ERROR("Init the searcher first before load index");
return IndexError_Runtime;
}
LOG_INFO("Begin HnswRabitqSearcher:load");
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from container");
return ret;
}
ret = reformer_.load(container);
if (ret != 0) {
LOG_ERROR("Failed to load reformer from container: %d", ret);
return ret;
}
ret = entity_.load(container, check_crc_enabled_);
if (ret != 0) {
LOG_ERROR("HnswRabitqSearcher load index failed");
return ret;
}
alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
entity_, reformer_.num_clusters(), reformer_.rabitq_metric_type()));
if (metric) {
metric_ = metric;
} else {
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
if (metric_->query_metric()) {
metric_ = metric_->query_metric();
}
}
if (!metric_->is_matched(meta_)) {
LOG_ERROR("IndexMetric not match index meta");
return IndexError_Mismatch;
}
max_scan_num_ = static_cast<uint32_t>(max_scan_ratio_ * entity_.doc_cnt());
max_scan_num_ = std::max(4096U, max_scan_num_);
stats_.set_loaded_count(entity_.doc_cnt());
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = STATE_LOADED;
magic_ = IndexContext::GenerateMagic();
LOG_INFO("End HnswRabitqSearcher::load");
return 0;
}
int HnswRabitqSearcher::unload() {
LOG_INFO("HnswRabitqSearcher unload index");
meta_.clear();
entity_.cleanup();
metric_.reset();
max_scan_num_ = 0;
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
state_ = STATE_INITED;
return 0;
}
int HnswRabitqSearcher::update_context(HnswRabitqContext *ctx) const {
const HnswRabitqEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
ctx->set_max_scan_num(max_scan_num_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
return ctx->update_context(HnswRabitqContext::kSearcherContext, meta_,
metric_, entity, magic_);
}
int HnswRabitqSearcher::search_impl(const void *query,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ret = alg_->search(&entity, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher fast search failed");
return ret;
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqSearcher::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqSearcher::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
IndexSearcher::Context::Pointer HnswRabitqSearcher::create_context() const {
if (ailego_unlikely(state_ != STATE_LOADED)) {
LOG_ERROR("Load the index first before create context");
return Context::Pointer();
}
const HnswRabitqEntity::Pointer search_ctx_entity = entity_.clone();
if (!search_ctx_entity) {
LOG_ERROR("Failed to create search context entity");
return Context::Pointer();
}
HnswRabitqContext *ctx = new (std::nothrow)
HnswRabitqContext(meta_.dimension(), metric_, search_ctx_entity);
if (ailego_unlikely(ctx == nullptr)) {
LOG_ERROR("Failed to new HnswRabitqContext");
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_max_scan_num(max_scan_num_);
uint32_t filter_mode =
bf_enabled_ ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
ctx->set_filter_mode(filter_mode);
ctx->set_filter_negative_probability(bf_negative_probability_);
ctx->set_magic(magic_);
ctx->set_force_padding_topk(force_padding_topk_enabled_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
if (ailego_unlikely(ctx->init(HnswRabitqContext::kSearcherContext)) != 0) {
LOG_ERROR("Init HnswRabitqContext failed");
delete ctx;
return Context::Pointer();
}
return Context::Pointer(ctx);
}
IndexProvider::Pointer HnswRabitqSearcher::create_provider(void) const {
LOG_DEBUG("HnswRabitqSearcher create provider");
auto entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone HnswRabitqEntity failed");
return Provider::Pointer();
}
return Provider::Pointer(new (std::nothrow) HnswRabitqIndexProvider(
meta_, entity, "HnswRabitqSearcher"));
}
const void *HnswRabitqSearcher::get_vector(uint64_t key) const {
return entity_.get_vector_by_key(key);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,142 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_framework.h"
#include "hnsw_rabitq_query_algorithm.h"
#include "hnsw_rabitq_searcher_entity.h"
#include "rabitq_reformer.h"
namespace zvec {
namespace core {
class HnswRabitqSearcher : public IndexSearcher {
public:
using ContextPointer = IndexSearcher::Context::Pointer;
public:
HnswRabitqSearcher(void);
~HnswRabitqSearcher(void);
HnswRabitqSearcher(const HnswRabitqSearcher &) = delete;
HnswRabitqSearcher &operator=(const HnswRabitqSearcher &) = delete;
protected:
//! Initialize Searcher
virtual int init(const ailego::Params &params) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
//! Load Index from storage
virtual int load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) override;
//! Unload index from storage
virtual int unload(void) override;
//! KNN Search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_impl(query, qmeta, 1, context);
}
//! KNN Search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
ContextPointer &context) const override;
//! Linear Search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Linear Search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
ContextPointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override;
//! Create a searcher context
virtual ContextPointer create_context() const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
virtual const ailego::Params &params(void) const override {
return params_;
}
virtual void print_debug_info() override;
private:
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(HnswRabitqContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
HnswRabitqSearcherEntity entity_{};
HnswRabitqQueryAlgorithm::UPointer alg_; // impl graph algorithm
IndexMetric::Pointer metric_{};
IndexMeta meta_{};
ailego::Params params_{};
Stats stats_;
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
uint32_t max_scan_num_{0U};
uint32_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
bool bf_enabled_{false};
bool check_crc_enabled_{false};
bool neighbors_in_memory_enabled_{false};
bool force_padding_topk_enabled_{false};
float bf_negative_probability_{
HnswRabitqEntity::kDefaultBFNegativeProbability};
uint32_t magic_{0U};
RabitqReformer reformer_;
State state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,515 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_searcher_entity.h"
#include <zvec/ailego/hash/crc32c.h>
#include "utility/sparse_utility.h"
namespace zvec {
namespace core {
HnswRabitqSearcherEntity::HnswRabitqSearcherEntity() {}
int HnswRabitqSearcherEntity::cleanup(void) {
storage_.reset();
vectors_.reset();
keys_.reset();
neighbors_.reset();
neighbors_meta_.reset();
neighbors_in_memory_enabled_ = false;
loaded_ = false;
this->HnswRabitqEntity::cleanup();
return 0;
}
key_t HnswRabitqSearcherEntity::get_key(node_id_t id) const {
const void *key;
if (ailego_unlikely(keys_->read(id * sizeof(key_t), &key, sizeof(key_t)) !=
sizeof(key_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidKey;
}
return *(reinterpret_cast<const key_t *>(key));
}
//! Get vector local id by key
node_id_t HnswRabitqSearcherEntity::get_id(key_t key) const {
if (ailego_unlikely(!mapping_)) {
LOG_ERROR("Index missing mapping segment");
return kInvalidNodeId;
}
//! Do binary search
node_id_t start = 0UL;
node_id_t end = doc_cnt();
const void *data;
node_id_t idx = 0u;
while (start < end) {
idx = start + (end - start) / 2;
if (ailego_unlikely(
mapping_->read(idx * sizeof(node_id_t), &data, sizeof(node_id_t)) !=
sizeof(node_id_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidNodeId;
}
const key_t *mkey;
node_id_t local_id = *reinterpret_cast<const node_id_t *>(data);
if (ailego_unlikely(keys_->read(local_id * sizeof(key_t),
(const void **)(&mkey),
sizeof(key_t)) != sizeof(key_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidNodeId;
}
if (*mkey < key) {
start = idx + 1;
} else if (*mkey > key) {
end = idx;
} else {
return local_id;
}
}
return kInvalidNodeId;
}
const void *HnswRabitqSearcherEntity::get_vector_by_key(key_t key) const {
node_id_t local_id = get_id(key);
if (ailego_unlikely(local_id == kInvalidNodeId)) {
return nullptr;
}
return get_vector(local_id);
}
const void *HnswRabitqSearcherEntity::get_vector(node_id_t id) const {
size_t read_size = vector_size();
size_t offset = node_size() * id;
const void *vec;
if (ailego_unlikely(vectors_->read(offset, &vec, read_size) != read_size)) {
LOG_ERROR("Read vector from segment failed");
return nullptr;
}
return vec;
}
int HnswRabitqSearcherEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
const void *vec = get_vector(id);
block.reset((void *)vec);
return 0;
}
const void *HnswRabitqSearcherEntity::get_vectors() const {
const void *vec;
size_t len = node_size() * doc_cnt();
if (vectors_->read(0, &vec, len) != len) {
LOG_ERROR("Read vectors from segment failed");
return nullptr;
}
return vec;
}
int HnswRabitqSearcherEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
ailego_assert_with(count <= segment_datas_.size(), "invalid count");
size_t read_size = vector_size();
for (uint32_t i = 0; i < count; ++i) {
segment_datas_[i].offset = node_size() * ids[i];
segment_datas_[i].length = read_size;
ailego_assert_with(segment_datas_[i].offset < vectors_->data_size(),
"invalid offset");
}
if (ailego_unlikely(!vectors_->read(&segment_datas_[0], count))) {
LOG_ERROR("Read vectors from segment failed");
return IndexError_ReadData;
}
for (uint32_t i = 0; i < count; ++i) {
vecs[i] = segment_datas_[i].data;
}
return 0;
}
int HnswRabitqSearcherEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
const void *vecs[count];
get_vector(ids, count, vecs);
for (uint32_t i = 0; i < count; ++i) {
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
}
return 0;
}
const Neighbors HnswRabitqSearcherEntity::get_neighbors(level_t level,
node_id_t id) const {
if (level == 0) {
if (neighbors_in_memory_enabled_) {
auto hd = reinterpret_cast<const NeighborsHeader *>(
fixed_neighbors_.get() + neighbors_size() * id);
return {hd->neighbor_cnt, hd->neighbors};
}
const GraphNeighborMeta *m;
if (ailego_unlikely(neighbors_meta_->read(id * sizeof(GraphNeighborMeta),
(const void **)(&m),
sizeof(GraphNeighborMeta)) !=
sizeof(GraphNeighborMeta))) {
LOG_ERROR("Read neighbors meta from segment failed");
return {0, nullptr};
}
const void *data;
if (ailego_unlikely(neighbors_->read(m->offset, &data,
m->neighbor_cnt * sizeof(node_id_t)) !=
m->neighbor_cnt * sizeof(node_id_t))) {
LOG_ERROR("Read neighbors from segment failed");
return {0, nullptr};
}
return {static_cast<uint32_t>(m->neighbor_cnt),
reinterpret_cast<const node_id_t *>(data)};
}
//! Read level > 0 neighbors
const HnswNeighborMeta *m;
if (ailego_unlikely(upper_neighbors_meta_->read(id * sizeof(HnswNeighborMeta),
(const void **)(&m),
sizeof(HnswNeighborMeta)) !=
sizeof(HnswNeighborMeta))) {
LOG_ERROR("Read neighbors meta from segment failed");
return {0, nullptr};
}
ailego_assert_with(level <= m->level, "invalid level");
size_t offset = m->offset + (level - 1) * upper_neighbors_size();
ailego_assert_with(offset <= upper_neighbors_->data_size(), "invalid offset");
const void *data;
if (ailego_unlikely(
upper_neighbors_->read(offset, &data, upper_neighbors_size()) !=
upper_neighbors_size())) {
LOG_ERROR("Read neighbors from segment failed");
return {0, nullptr};
}
auto hd = reinterpret_cast<const NeighborsHeader *>(data);
return {hd->neighbor_cnt, hd->neighbors};
}
int HnswRabitqSearcherEntity::load(const IndexStorage::Pointer &container,
bool check_crc) {
storage_ = container;
int ret = load_segments(check_crc);
if (ret != 0) {
return ret;
}
loaded_ = true;
LOG_INFO(
"Index info: docCnt=%u entryPoint=%u maxLevel=%d efConstruct=%zu "
"l0NeighborCnt=%zu upperNeighborCnt=%zu scalingFactor=%zu "
"vectorSize=%zu nodeSize=%zu vectorSegmentSize=%zu keySegmentSize=%zu "
"neighborsSegmentSize=%zu neighborsMetaSegmentSize=%zu ",
doc_cnt(), entry_point(), cur_max_level(), ef_construction(),
l0_neighbor_cnt(), upper_neighbor_cnt(), scaling_factor(), vector_size(),
node_size(), vectors_->data_size(), keys_->data_size(),
neighbors_ == nullptr ? 0 : neighbors_->data_size(),
neighbors_meta_ == nullptr ? 0 : neighbors_meta_->data_size());
return 0;
}
int HnswRabitqSearcherEntity::load_segments(bool check_crc) {
//! load header
const void *data = nullptr;
HNSWHeader hd;
auto graph_hd_segment = storage_->get(kGraphHeaderSegmentId);
if (!graph_hd_segment || graph_hd_segment->data_size() < sizeof(hd.graph)) {
LOG_ERROR("Miss or invalid segment %s", kGraphHeaderSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (graph_hd_segment->read(0, reinterpret_cast<const void **>(&data),
sizeof(hd.graph)) != sizeof(hd.graph)) {
LOG_ERROR("Read segment %s failed", kGraphHeaderSegmentId.c_str());
return IndexError_ReadData;
}
memcpy(&hd.graph, data, sizeof(hd.graph));
auto hnsw_hd_segment = storage_->get(kHnswHeaderSegmentId);
if (!hnsw_hd_segment || hnsw_hd_segment->data_size() < sizeof(hd.hnsw)) {
LOG_ERROR("Miss or invalid segment %s", kHnswHeaderSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (hnsw_hd_segment->read(0, reinterpret_cast<const void **>(&data),
sizeof(hd.hnsw)) != sizeof(hd.hnsw)) {
LOG_ERROR("Read segment %s failed", kHnswHeaderSegmentId.c_str());
return IndexError_ReadData;
}
memcpy(&hd.hnsw, data, sizeof(hd.hnsw));
*mutable_header() = hd;
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()));
vectors_ = storage_->get(kGraphFeaturesSegmentId);
if (!vectors_) {
LOG_ERROR("IndexStorage get segment %s failed",
kGraphFeaturesSegmentId.c_str());
return IndexError_InvalidFormat;
}
keys_ = storage_->get(kGraphKeysSegmentId);
if (!keys_) {
LOG_ERROR("IndexStorage get segment %s failed",
kGraphKeysSegmentId.c_str());
return IndexError_InvalidFormat;
}
neighbors_ = storage_->get(kGraphNeighborsSegmentId);
if (!neighbors_ || (neighbors_->data_size() == 0 && doc_cnt() > 1)) {
LOG_ERROR("IndexStorage get segment %s failed or empty",
kGraphNeighborsSegmentId.c_str());
return IndexError_InvalidArgument;
}
neighbors_meta_ = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta_ ||
neighbors_meta_->data_size() < sizeof(GraphNeighborMeta) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
upper_neighbors_ = storage_->get(kHnswNeighborsSegmentId);
if (!upper_neighbors_ ||
(upper_neighbors_->data_size() == 0 && cur_max_level() > 0)) {
LOG_ERROR("IndexStorage get segment %s failed or empty",
kHnswNeighborsSegmentId.c_str());
return IndexError_InvalidArgument;
}
upper_neighbors_meta_ = storage_->get(kHnswOffsetsSegmentId);
if (!upper_neighbors_meta_ || upper_neighbors_meta_->data_size() <
sizeof(HnswNeighborMeta) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kHnswOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
mapping_ = storage_->get(kGraphMappingSegmentId);
if (!mapping_ || mapping_->data_size() < sizeof(node_id_t) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kGraphMappingSegmentId.c_str());
return IndexError_InvalidArgument;
}
if (check_crc) {
std::vector<SegmentPointer> segments;
segments.emplace_back(graph_hd_segment);
segments.emplace_back(hnsw_hd_segment);
segments.emplace_back(vectors_);
segments.emplace_back(keys_);
segments.emplace_back(neighbors_);
segments.emplace_back(neighbors_meta_);
segments.emplace_back(upper_neighbors_);
segments.emplace_back(upper_neighbors_meta_);
if (!do_crc_check(segments)) {
LOG_ERROR("Check index crc failed, the index may broken");
return IndexError_Runtime;
}
}
if (neighbors_in_memory_enabled_) {
int ret = load_and_flat_neighbors();
if (ret != 0) {
return ret;
}
}
return 0;
}
int HnswRabitqSearcherEntity::load_and_flat_neighbors() {
fixed_neighbors_.reset(
new (std::nothrow) char[neighbors_size() * doc_cnt()]{},
std::default_delete<char[]>());
if (!fixed_neighbors_) {
LOG_ERROR("Malloc memory failed");
return IndexError_NoMemory;
}
//! Get a new segemnt to release the buffer after loading neighbors
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta) {
LOG_ERROR("IndexStorage get segment graph.offsets failed");
return IndexError_InvalidArgument;
}
const GraphNeighborMeta *neighbors_index = nullptr;
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
neighbors_meta->data_size()) !=
neighbors_meta->data_size()) {
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
const char *neighbor_data;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
size_t rd_size = neighbors_index[id].neighbor_cnt * sizeof(node_id_t);
if (ailego_unlikely(
neighbors_->read(neighbors_index[id].offset,
reinterpret_cast<const void **>(&neighbor_data),
rd_size) != rd_size)) {
LOG_ERROR("Read neighbors from segment failed");
return IndexError_ReadData;
}
// copy level 0 neighbors to fixed size neighbors memory
char *dst = fixed_neighbors_.get() + neighbors_size() * id;
*reinterpret_cast<uint32_t *>(dst) = neighbors_index[id].neighbor_cnt;
memcpy(dst + sizeof(uint32_t), neighbor_data, rd_size);
}
return 0;
}
int HnswRabitqSearcherEntity::get_fixed_neighbors(
std::vector<uint32_t> *fixed_neighbors) const {
//! Get a new segemnt to release the buffer after loading neighbors
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta) {
LOG_ERROR("IndexStorage get segment graph.offsets failed");
return IndexError_InvalidArgument;
}
const GraphNeighborMeta *neighbors_index = nullptr;
size_t meta_size = neighbors_meta->data_size();
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
meta_size) != meta_size) {
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
size_t fixed_neighbor_cnt = l0_neighbor_cnt();
fixed_neighbors->resize((fixed_neighbor_cnt + 1) * doc_cnt(), kInvalidNodeId);
size_t neighbors_cnt_offset = fixed_neighbor_cnt * doc_cnt();
size_t total_neighbor_cnt = 0;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
size_t cur_neighbor_cnt = neighbors_index[id].neighbor_cnt;
if (cur_neighbor_cnt == 0) {
(*fixed_neighbors)[neighbors_cnt_offset + id] = 0;
continue;
}
size_t rd_size = cur_neighbor_cnt * sizeof(node_id_t);
const uint32_t *neighbors;
if (neighbors_->read(neighbors_index[id].offset,
reinterpret_cast<const void **>(&neighbors),
rd_size) != rd_size) {
LOG_ERROR("Read neighbors from segment failed");
return IndexError_ReadData;
}
// copy level 0 neighbors to fixed size neighbors memory
auto it = fixed_neighbors->begin() + id * fixed_neighbor_cnt;
std::copy(neighbors, neighbors + cur_neighbor_cnt, it);
(*fixed_neighbors)[neighbors_cnt_offset + id] = cur_neighbor_cnt;
total_neighbor_cnt += cur_neighbor_cnt;
}
LOG_INFO("total neighbor cnt: %zu, average neighbor cnt: %zu",
total_neighbor_cnt, total_neighbor_cnt / doc_cnt());
return 0;
}
bool HnswRabitqSearcherEntity::do_crc_check(
std::vector<SegmentPointer> &segments) const {
constexpr size_t blk_size = 4096;
const void *data;
for (auto &segment : segments) {
size_t offset = 0;
size_t rd_size;
uint32_t crc = 0;
while (offset < segment->data_size()) {
size_t size = std::min(blk_size, segment->data_size() - offset);
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
break;
}
offset += rd_size;
crc = ailego::Crc32c::Hash(data, rd_size, crc);
}
if (crc != segment->data_crc()) {
return false;
}
}
return true;
}
const HnswRabitqEntity::Pointer HnswRabitqSearcherEntity::clone() const {
auto vectors = vectors_->clone();
if (ailego_unlikely(!vectors)) {
LOG_ERROR("clone segment %s failed", kGraphFeaturesSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto keys = keys_->clone();
if (ailego_unlikely(!keys)) {
LOG_ERROR("clone segment %s failed", kGraphKeysSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto mapping = mapping_->clone();
if (ailego_unlikely(!mapping)) {
LOG_ERROR("clone segment %s failed", kGraphMappingSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto neighbors = neighbors_->clone();
if (ailego_unlikely(!neighbors)) {
LOG_ERROR("clone segment %s failed", kGraphNeighborsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto upper_neighbors = upper_neighbors_->clone();
if (ailego_unlikely(!neighbors)) {
LOG_ERROR("clone segment %s failed", kHnswNeighborsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto neighbors_meta = neighbors_meta_->clone();
if (ailego_unlikely(!neighbors_meta)) {
LOG_ERROR("clone segment %s failed", kGraphOffsetsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto upper_neighbors_meta = upper_neighbors_meta_->clone();
if (ailego_unlikely(!upper_neighbors_meta)) {
LOG_ERROR("clone segment %s failed", kHnswOffsetsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
SegmentGroupParam neighbor_group{neighbors, neighbors_meta, upper_neighbors,
upper_neighbors_meta};
HnswRabitqSearcherEntity *entity = new (std::nothrow)
HnswRabitqSearcherEntity(header(), vectors, keys, mapping, neighbor_group,
fixed_neighbors_, neighbors_in_memory_enabled_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswRabitqSearcherEntity new failed");
}
return HnswRabitqEntity::Pointer(entity);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,158 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "hnsw_rabitq_builder_entity.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqSearcherEntity : public HnswRabitqEntity {
public:
using Pointer = std::shared_ptr<HnswRabitqSearcherEntity>;
using SegmentPointer = IndexStorage::Segment::Pointer;
public:
struct SegmentGroupParam {
SegmentGroupParam(SegmentPointer neighbors_in,
SegmentPointer neighbors_meta_in,
SegmentPointer upper_neighbors_in,
SegmentPointer upper_neighbors_meta_in)
: neighbors{neighbors_in},
neighbors_meta{neighbors_meta_in},
upper_neighbors{upper_neighbors_in},
upper_neighbors_meta{upper_neighbors_meta_in} {}
SegmentPointer neighbors{nullptr};
SegmentPointer neighbors_meta{nullptr};
SegmentPointer upper_neighbors{nullptr};
SegmentPointer upper_neighbors_meta{nullptr};
};
//! Constructor
HnswRabitqSearcherEntity();
//! Make a copy of searcher entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector local id by key
node_id_t get_id(key_t key) const;
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override;
//! Get vector feature data by id
virtual const void *get_vector(node_id_t id) const override;
//! Get vector feature data by id
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get all vectors
const void *get_vectors() const;
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
virtual int load(const IndexStorage::Pointer &container,
bool check_crc) override;
int load_segments(bool check_crc);
virtual int cleanup(void) override;
public:
bool is_loaded() const {
return loaded_;
}
void set_neighbors_in_memory(bool enabled) {
neighbors_in_memory_enabled_ = enabled;
}
//! get fixed length neighbors data
int get_fixed_neighbors(std::vector<uint32_t> *fixed_neighbors) const;
private:
//! Constructor
HnswRabitqSearcherEntity(const HNSWHeader &hd, const SegmentPointer &vectors,
const SegmentPointer &keys,
const SegmentPointer &mapping,
const SegmentGroupParam &neighbor_group,
const std::shared_ptr<char> &fixed_neighbors,
bool neighbors_in_memory_enabled)
: HnswRabitqEntity(hd),
vectors_(vectors),
keys_(keys),
mapping_(mapping),
neighbors_(neighbor_group.neighbors),
neighbors_meta_(neighbor_group.neighbors_meta),
upper_neighbors_(neighbor_group.upper_neighbors),
upper_neighbors_meta_(neighbor_group.upper_neighbors_meta),
neighbors_in_memory_enabled_(neighbors_in_memory_enabled) {
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()),
IndexStorage::SegmentData(0U, 0U));
fixed_neighbors_ = fixed_neighbors;
}
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
//! If neighbors_in_memory_enabled, load the level0 neighbors to memory
int load_and_flat_neighbors(void);
public:
HnswRabitqSearcherEntity(const HnswRabitqSearcherEntity &) = delete;
HnswRabitqSearcherEntity &operator=(const HnswRabitqSearcherEntity &) =
delete;
private:
IndexStorage::Pointer storage_{};
SegmentPointer vectors_{};
SegmentPointer keys_{};
SegmentPointer mapping_{};
SegmentPointer neighbors_{};
SegmentPointer neighbors_meta_{};
SegmentPointer upper_neighbors_{};
SegmentPointer upper_neighbors_meta_{};
mutable std::vector<IndexStorage::SegmentData> segment_datas_{};
std::shared_ptr<char> fixed_neighbors_{}; // level 0 fixed size neighbors
bool neighbors_in_memory_enabled_{false};
bool loaded_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,989 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_streamer.h"
#include <iostream>
#include <memory>
#include <ailego/internal/cpu_features.h>
#include <ailego/pattern/defer.h>
#include <ailego/utility/memory_helper.h>
#include <arrow/compute/ordering.h>
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/ailego/container/params.h"
#include "zvec/ailego/logger/logger.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_index_provider.h"
#include "hnsw_rabitq_query_entity.h"
#include "rabitq_params.h"
#include "rabitq_utils.h"
namespace zvec {
namespace core {
HnswRabitqStreamer::HnswRabitqStreamer() : entity_(stats_) {}
HnswRabitqStreamer::HnswRabitqStreamer(IndexProvider::Pointer provider,
RabitqReformer::Pointer reformer)
: entity_(stats_),
reformer_(std::move(reformer)),
provider_(std::move(provider)) {}
HnswRabitqStreamer::~HnswRabitqStreamer() {
if (state_ == STATE_INITED) {
this->cleanup();
}
}
int HnswRabitqStreamer::init(const IndexMeta &imeta,
const ailego::Params &params) {
meta_ = imeta;
meta_.set_streamer("HnswRabitqStreamer", HnswRabitqEntity::kRevision, params);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE, &max_index_size_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT,
&upper_max_neighbor_cnt_);
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
params.get(PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
&multiplier);
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
params.get(PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
scaling_factor_ = upper_max_neighbor_cnt_;
params.get(PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR, &scaling_factor_);
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION, &ef_construction_);
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE, &bf_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
&bf_negative_prob_);
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE, &check_crc_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE, &chunk_size_);
params.get(PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY, &filter_same_key_);
params.get(PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE,
&get_vector_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
params.get(PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP, &use_id_map_);
entity_.set_use_key_info_map(use_id_map_);
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit_);
if (docs_soft_limit_ > 0 && docs_soft_limit_ > docs_hard_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
return IndexError_InvalidArgument;
} else if (docs_soft_limit_ == 0UL) {
docs_soft_limit_ =
docs_hard_limit_ * HnswRabitqEntity::kDefaultDocsSoftLimitRatio;
}
if (ef_ == 0U) {
ef_ = HnswRabitqEntity::kDefaultEf;
}
if (ef_construction_ == 0U) {
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
}
if (upper_max_neighbor_cnt_ == 0U) {
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (upper_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("[%s] must be in range (0,%d)",
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (l0_max_neighbor_cnt_ == 0U) {
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
}
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("MaxL0NeighborCnt must be in range (0,%d)",
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
LOG_ERROR("[%s]-[%zu] must be <= [%s]-[%zu]",
PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT.c_str(),
static_cast<size_t>(min_neighbor_cnt_),
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
static_cast<size_t>(upper_max_neighbor_cnt_));
return IndexError_InvalidArgument;
}
if (bf_negative_prob_ <= 0.0f || bf_negative_prob_ >= 1.0f) {
LOG_ERROR(
"[%s] must be in range (0,1)",
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
return IndexError_InvalidArgument;
}
if (scaling_factor_ == 0U) {
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
}
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
LOG_ERROR("[%s] must be in range [5,1000]",
PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_limit_ < min_scan_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
return IndexError_InvalidArgument;
}
if (prune_cnt == 0UL) {
prune_cnt = upper_max_neighbor_cnt_;
}
if (chunk_size_ == 0UL) {
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
}
if (chunk_size_ > HnswRabitqEntity::kMaxChunkSize) {
LOG_ERROR("[%s] must be < %zu",
PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE.c_str(),
HnswRabitqEntity::kMaxChunkSize);
return IndexError_InvalidArgument;
}
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
uint8_t ex_bits = total_bits - 1;
entity_.set_ex_bits(ex_bits);
uint32_t dimension = 0;
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
if (dimension == 0) {
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
return IndexError_InvalidArgument;
}
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
kMinRabitqDimSize, kMaxRabitqDimSize);
return IndexError_InvalidArgument;
}
entity_.update_rabitq_params_and_vector_size(dimension);
entity_.set_ef_construction(ef_construction_);
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_.set_scaling_factor(scaling_factor_);
entity_.set_prune_cnt(prune_cnt);
entity_.set_chunk_size(chunk_size_);
entity_.set_filter_same_key(filter_same_key_);
entity_.set_get_vector(get_vector_enabled_);
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
int ret = entity_.init(docs_hard_limit_);
if (ret != 0) {
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
return ret;
}
LOG_DEBUG(
"Init params: maxIndexSize=%zu docsHardLimit=%zu docsSoftLimit=%zu "
"efConstruction=%u ef=%u upperMaxNeighborCnt=%u l0MaxNeighborCnt=%u "
"scalingFactor=%u maxScanRatio=%.3f minScanLimit=%zu maxScanLimit=%zu "
"bfEnabled=%d bruteFoceThreshold=%zu bfNegativeProbability=%.5f "
"checkCrcEnabled=%d pruneSize=%zu vectorSize=%u chunkSize=%zu "
"filterSameKey=%u getVectorEnabled=%u minNeighborCount=%u "
"forcePadding=%u ",
max_index_size_, docs_hard_limit_, docs_soft_limit_, ef_construction_,
ef_, upper_max_neighbor_cnt_, l0_max_neighbor_cnt_, scaling_factor_,
max_scan_ratio_, min_scan_limit_, max_scan_limit_, bf_enabled_,
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt,
meta_.element_size(), chunk_size_, filter_same_key_, get_vector_enabled_,
min_neighbor_cnt_, force_padding_topk_enabled_);
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
ret = alg_->init();
if (ret != 0) {
return ret;
}
state_ = STATE_INITED;
return 0;
}
int HnswRabitqStreamer::cleanup(void) {
if (state_ == STATE_OPENED) {
this->close();
}
LOG_INFO("HnswRabitqStreamer cleanup");
meta_.clear();
metric_.reset();
stats_.clear();
entity_.cleanup();
if (alg_) {
alg_->cleanup();
}
max_index_size_ = 0UL;
docs_hard_limit_ = HnswRabitqEntity::kDefaultDocsHardLimit;
docs_soft_limit_ = 0UL;
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
ef_ = HnswRabitqEntity::kDefaultEf;
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
bf_enabled_ = false;
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
max_scan_limit_ = HnswRabitqEntity::kDefaultMaxScanLimit;
min_scan_limit_ = HnswRabitqEntity::kDefaultMinScanLimit;
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
bf_negative_prob_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
state_ = STATE_INIT;
check_crc_enabled_ = false;
filter_same_key_ = false;
get_vector_enabled_ = false;
return 0;
}
int HnswRabitqStreamer::open(IndexStorage::Pointer stg) {
LOG_INFO("HnswRabitqStreamer open");
if (ailego_unlikely(state_ != STATE_INITED)) {
LOG_ERROR("Open storage failed, init streamer first!");
return IndexError_NoReady;
}
// try to load reformer
if (reformer_ == nullptr) {
reformer_ = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
int ret = reformer_->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
ret = reformer_->load(stg);
if (ret != 0) {
LOG_ERROR("Failed to load reformer, ret=%d", ret);
return ret;
}
} else {
if (!stg->has(RABITQ_CONVERTER_SEG_ID)) {
int ret = reformer_->dump(stg);
if (ret != 0) {
LOG_ERROR("Failed to dump reformer, ret=%d", ret);
return ret;
}
LOG_INFO("Dump reformer success.");
}
}
int ret = entity_.open(std::move(stg), max_index_size_, check_crc_enabled_);
if (ret != 0) {
return ret;
}
IndexMeta index_meta;
ret = entity_.get_index_meta(&index_meta);
if (ret == IndexError_NoExist) {
// Set IndexMeta for the new index
ret = entity_.set_index_meta(meta_);
if (ret != 0) {
LOG_ERROR("Failed to set index meta for %s", IndexError::What(ret));
return ret;
}
} else if (ret != 0) {
LOG_ERROR("Failed to get index meta for %s", IndexError::What(ret));
return ret;
} else {
if (index_meta.dimension() != meta_.dimension() ||
index_meta.element_size() != meta_.element_size() ||
index_meta.metric_name() != meta_.metric_name() ||
index_meta.data_type() != meta_.data_type()) {
LOG_ERROR("IndexMeta mismatch from the previous in index");
return IndexError_Mismatch;
}
// The IndexMetric Params may be updated like MipsSquaredEuclidean
auto metric_params = index_meta.metric_params();
metric_params.merge(meta_.metric_params());
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
}
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("Failed to init metric, ret=%d", ret);
return ret;
}
if (!metric_->distance()) {
LOG_ERROR("Invalid metric distance");
return IndexError_InvalidArgument;
}
if (!metric_->batch_distance()) {
LOG_ERROR("Invalid metric batch distance");
return IndexError_InvalidArgument;
}
add_distance_ = metric_->distance();
add_batch_distance_ = metric_->batch_distance();
search_distance_ = add_distance_;
search_batch_distance_ = add_batch_distance_;
if (metric_->query_metric() && metric_->query_metric()->distance() &&
metric_->query_metric()->batch_distance()) {
search_distance_ = metric_->query_metric()->distance();
search_batch_distance_ = metric_->query_metric()->batch_distance();
}
state_ = STATE_OPENED;
magic_ = IndexContext::GenerateMagic();
query_alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
entity_, reformer_->num_clusters(), reformer_->rabitq_metric_type()));
return 0;
}
int HnswRabitqStreamer::close(void) {
LOG_INFO("HnswRabitqStreamer close");
stats_.clear();
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
int ret = entity_.close();
if (ret != 0) {
return ret;
}
state_ = STATE_INITED;
return 0;
}
int HnswRabitqStreamer::flush(uint64_t checkpoint) {
LOG_INFO("HnswRabitqStreamer flush checkpoint=%zu", (size_t)checkpoint);
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
return entity_.flush(checkpoint);
}
int HnswRabitqStreamer::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("HnswRabitqStreamer dump");
shared_mutex_.lock();
AILEGO_DEFER([&]() { shared_mutex_.unlock(); });
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
ailego::Params());
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
ret = reformer_->dump(dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump reformer into dumper.");
return ret;
}
return entity_.dump(dumper);
}
IndexStreamer::Context::Pointer HnswRabitqStreamer::create_context(void) const {
if (ailego_unlikely(state_ != STATE_OPENED)) {
LOG_ERROR("Create context failed, open storage first!");
return Context::Pointer();
}
HnswRabitqEntity::Pointer entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("CreateContext clone init failed");
return Context::Pointer();
}
HnswRabitqContext *ctx =
new (std::nothrow) HnswRabitqContext(meta_.dimension(), metric_, entity);
if (ailego_unlikely(ctx == nullptr)) {
LOG_ERROR("Failed to new HnswRabitqContext");
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_filter_mode(bf_enabled_ ? VisitFilter::BloomFilter
: VisitFilter::ByteMap);
ctx->set_filter_negative_probability(bf_negative_prob_);
ctx->set_magic(magic_);
ctx->set_force_padding_topk(force_padding_topk_enabled_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
if (ailego_unlikely(ctx->init(HnswRabitqContext::kStreamerContext)) != 0) {
LOG_ERROR("Init HnswRabitqContext failed");
delete ctx;
return Context::Pointer();
}
uint32_t estimate_doc_count = 0;
if (meta_.streamer_params().get(PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT,
&estimate_doc_count)) {
LOG_DEBUG("HnswRabitqStreamer doc_count[%zu] estimate[%zu]",
(size_t)entity_.doc_cnt(), (size_t)estimate_doc_count);
}
ctx->check_need_adjuct_ctx(std::max(entity_.doc_cnt(), estimate_doc_count));
return Context::Pointer(ctx);
}
IndexProvider::Pointer HnswRabitqStreamer::create_provider(void) const {
LOG_DEBUG("HnswRabitqStreamer create provider");
auto entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone HnswRabitqEntity failed");
return nullptr;
}
return Provider::Pointer(
new HnswRabitqIndexProvider(meta_, entity, "HnswRabitqStreamer"));
}
int HnswRabitqStreamer::update_context(HnswRabitqContext *ctx) const {
const HnswRabitqEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
return ctx->update_context(HnswRabitqContext::kStreamerContext, meta_,
metric_, entity, magic_);
}
//! Add a vector with id into index
int HnswRabitqStreamer::add_with_id_impl(
uint32_t id, const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) {
if (!provider_) {
LOG_ERROR("Provider is nullptr, cannot add vector");
return IndexError_InvalidArgument;
}
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->set_provider(provider_);
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
ret = entity_.add_vector_with_id(level, id, converted_vector.data());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, level, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw steamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
//! Add a vector into index
int HnswRabitqStreamer::add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) {
if (!provider_) {
LOG_ERROR("Provider is nullptr, cannot add vector");
return IndexError_InvalidArgument;
}
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->set_provider(provider_);
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
node_id_t id;
ret = entity_.add_vector(level, pkey, converted_vector.data(), &id);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, level, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw steamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
int HnswRabitqStreamer::search_impl(
const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) const {
return search_impl(query, qmeta, 1, context);
}
//! Similarity search
int HnswRabitqStreamer::search_impl(
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
IndexStreamer::Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ret = query_alg_->search(&entity, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher fast search failed");
return ret;
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
void HnswRabitqStreamer::print_debug_info() {
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
Neighbors neighbours = entity_.get_neighbors(0, id);
std::cout << "node: " << id << "; ";
if (neighbours.size() == 0) std::cout << std::endl;
for (uint32_t i = 0; i < neighbours.size(); ++i) {
std::cout << neighbours[i];
if (i == neighbours.size() - 1) {
std::cout << std::endl;
} else {
std::cout << ", ";
}
}
}
// entity_.print_key_map();
}
int HnswRabitqStreamer::search_bf_impl(
const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) const {
return search_bf_impl(query, qmeta, 1, context);
}
int HnswRabitqStreamer::search_bf_impl(
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
IndexStreamer::Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqStreamer::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
key_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,249 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <ailego/parallel/lock.h>
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/core/framework/index_framework.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_query_algorithm.h"
#include "hnsw_rabitq_streamer_entity.h"
namespace zvec {
namespace core {
class HnswRabitqStreamer : public IndexStreamer {
public:
using ContextPointer = IndexStreamer::Context::Pointer;
HnswRabitqStreamer();
explicit HnswRabitqStreamer(IndexProvider::Pointer provider,
RabitqReformer::Pointer reformer = nullptr);
virtual ~HnswRabitqStreamer(void);
HnswRabitqStreamer(const HnswRabitqStreamer &streamer) = delete;
HnswRabitqStreamer &operator=(const HnswRabitqStreamer &streamer) = delete;
void set_provider(IndexProvider::Pointer provider) {
provider_ = std::move(provider);
}
void set_reformer(IndexReformer::Pointer reformer) {
reformer_ = std::dynamic_pointer_cast<RabitqReformer>(reformer);
}
protected:
//! Initialize Streamer
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
//! Cleanup Streamer
virtual int cleanup(void) override;
//! Create a context
virtual Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Add a vector into index
virtual int add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
virtual int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override {
return entity_.get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector_by_key(key, block);
}
//! Fetch vector by id
virtual const void *get_vector_by_id(uint32_t id) const override {
return entity_.get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector(id, block);
}
//! Open index from file path
virtual int open(IndexStorage::Pointer stg) override;
//! Close file
virtual int close(void) override;
//! flush file
virtual int flush(uint64_t checkpoint) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
private:
inline int check_params(const void *query,
const IndexQueryMeta &qmeta) const {
if (ailego_unlikely(!query)) {
LOG_ERROR("null query");
return IndexError_InvalidArgument;
}
if (ailego_unlikely(qmeta.dimension() != meta_.dimension() ||
qmeta.data_type() != meta_.data_type() ||
qmeta.element_size() != meta_.element_size())) {
LOG_ERROR("Unsupported query meta");
return IndexError_Mismatch;
}
return 0;
}
inline int check_sparse_count_is_zero(const uint32_t *sparse_count,
uint32_t count) const {
for (uint32_t i = 0; i < count; ++i) {
if (sparse_count[i] != 0)
LOG_ERROR("Sparse cout is not empty. Index: %u, Sparse Count: %u", i,
sparse_count[i]);
return IndexError_InvalidArgument;
}
return 0;
}
private:
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(HnswRabitqContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
class Stats : public IndexStreamer::Stats {
public:
void clear(void) {
set_revision_id(0u);
set_loaded_count(0u);
set_added_count(0u);
set_discarded_count(0u);
set_index_size(0u);
set_dumped_size(0u);
set_check_point(0u);
set_create_time(0u);
set_update_time(0u);
clear_attributes();
}
};
HnswRabitqStreamerEntity entity_;
HnswRabitqAlgorithm::UPointer alg_;
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexMetric::MatrixDistance add_distance_{};
IndexMetric::MatrixDistance search_distance_{};
IndexMetric::MatrixBatchDistance add_batch_distance_{};
IndexMetric::MatrixBatchDistance search_batch_distance_{};
RabitqReformer::Pointer reformer_{}; // RaBitQ reformer
HnswRabitqQueryAlgorithm::UPointer query_alg_; // query algorithm
// provider_ provides raw vector, which is used to build graph
IndexProvider::Pointer provider_{};
Stats stats_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
size_t chunk_size_{HnswRabitqEntity::kDefaultChunkSize};
size_t docs_hard_limit_{HnswRabitqEntity::kDefaultDocsHardLimit};
size_t docs_soft_limit_{0UL};
uint32_t min_neighbor_cnt_{0u};
uint32_t upper_max_neighbor_cnt_{
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
size_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
size_t max_scan_limit_{HnswRabitqEntity::kDefaultMaxScanLimit};
size_t min_scan_limit_{HnswRabitqEntity::kDefaultMinScanLimit};
float bf_negative_prob_{HnswRabitqEntity::kDefaultBFNegativeProbability};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
uint32_t magic_{0U};
State state_{STATE_INIT};
bool bf_enabled_{false};
bool check_crc_enabled_{false};
bool filter_same_key_{false};
bool get_vector_enabled_{false};
bool force_padding_topk_enabled_{false};
bool use_id_map_{true};
//! avoid add vector while dumping index
ailego::SharedMutex shared_mutex_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,709 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_streamer_entity.h"
#include <ailego/utility/memory_helper.h>
// #define DEBUG_PRINT
namespace zvec {
namespace core {
HnswRabitqStreamerEntity::HnswRabitqStreamerEntity(IndexStreamer::Stats &stats)
: stats_(stats) {}
HnswRabitqStreamerEntity::~HnswRabitqStreamerEntity() {}
int HnswRabitqStreamerEntity::init(size_t max_doc_cnt) {
if (std::pow(scaling_factor(), kMaxGraphLayers) < max_doc_cnt) {
LOG_ERROR("scalingFactor=%zu is too small", scaling_factor());
return IndexError_InvalidArgument;
}
std::lock_guard<std::mutex> lock(mutex_);
broker_ = std::make_shared<HnswRabitqChunkBroker>(stats_);
upper_neighbor_index_ = std::make_shared<NIHashMap>();
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
if (!keys_map_ || !upper_neighbor_index_ || !broker_ || !keys_map_lock_) {
LOG_ERROR("HnswRabitqStreamerEntity new object failed");
return IndexError_NoMemory;
}
keys_map_->set_empty_key(kInvalidKey);
neighbor_size_ = neighbors_size();
upper_neighbor_size_ = upper_neighbors_size();
//! vector + key + level 0 neighbors
size_t size = vector_size() + sizeof(key_t) + neighbor_size_;
size = AlignSize(size);
set_node_size(size);
return 0;
}
int HnswRabitqStreamerEntity::cleanup() {
std::lock_guard<std::mutex> lock(mutex_);
mutable_header()->clear();
chunk_size_ = kDefaultChunkSize;
node_index_mask_bits_ = 0U;
node_index_mask_ = 0U;
node_cnt_per_chunk_ = 0U;
neighbor_size_ = 0U;
upper_neighbor_size_ = 0U;
if (upper_neighbor_index_) {
upper_neighbor_index_->cleanup();
}
if (keys_map_) {
keys_map_->clear();
}
node_chunks_.clear();
upper_neighbor_chunks_.clear();
filter_same_key_ = false;
get_vector_enabled_ = false;
broker_.reset();
return 0;
}
int HnswRabitqStreamerEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
char buffer[neighbor_size_];
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
hd->neighbor_cnt = neighbors.size();
size_t i = 0;
for (; i < neighbors.size(); ++i) {
hd->neighbors[i] = neighbors[i].first;
}
auto loc = get_neighbor_chunk_loc(level, id);
size_t size = reinterpret_cast<char *>(&hd->neighbors[i]) - &buffer[0];
size_t ret = loc.first->write(loc.second, hd, size);
if (ailego_unlikely(ret != size)) {
LOG_ERROR("Write neighbor header failed, ret=%zu", ret);
return IndexError_Runtime;
}
return 0;
}
const Neighbors HnswRabitqStreamerEntity::get_neighbors(level_t level,
node_id_t id) const {
Chunk *chunk = nullptr;
size_t offset = 0UL;
size_t neighbor_size = neighbor_size_;
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
chunk = node_chunks_[chunk_idx].get();
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
chunk = upper_neighbor_chunks_[p.first].get();
offset = p.second;
neighbor_size = upper_neighbor_size_;
}
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
IndexStorage::MemoryBlock neighbor_block;
size_t size = chunk->read(offset, neighbor_block, neighbor_size);
if (ailego_unlikely(size != neighbor_size)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", size);
return Neighbors();
}
return Neighbors(std::move(neighbor_block));
}
//! Get vector data by key
const void *HnswRabitqStreamerEntity::get_vector(node_id_t id) const {
auto loc = get_vector_chunk_loc(id);
const void *vec = nullptr;
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, &vec, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
}
return vec;
}
int HnswRabitqStreamerEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, &vecs[i], read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
}
return 0;
}
int HnswRabitqStreamerEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
auto loc = get_vector_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, block, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
return 0;
}
int HnswRabitqStreamerEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret =
node_chunks_[loc.first]->read(loc.second, vec_blocks[i], read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
}
return 0;
}
key_t HnswRabitqStreamerEntity::get_key(node_id_t id) const {
if (use_key_info_map_) {
auto loc = get_key_chunk_loc(id);
IndexStorage::MemoryBlock key_block;
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t ret =
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(key_block.data());
} else {
return id;
}
}
void HnswRabitqStreamerEntity::add_neighbor(level_t level, node_id_t id,
uint32_t size,
node_id_t neighbor_id) {
auto loc = get_neighbor_chunk_loc(level, id);
size_t offset =
loc.second + sizeof(NeighborsHeader) + size * sizeof(node_id_t);
ailego_assert_with(size < neighbor_cnt(level), "invalid neighbor size");
ailego_assert_with(offset < loc.first->data_size(), "invalid chunk offset");
size_t ret = loc.first->write(offset, &neighbor_id, sizeof(node_id_t));
if (ailego_unlikely(ret != sizeof(node_id_t))) {
LOG_ERROR("Write neighbor id failed, ret=%zu", ret);
return;
}
uint32_t neighbors = size + 1;
ret = loc.first->write(loc.second, &neighbors, sizeof(uint32_t));
if (ailego_unlikely(ret != sizeof(uint32_t))) {
LOG_ERROR("Write neighbor cnt failed, ret=%zu", ret);
}
return;
}
int HnswRabitqStreamerEntity::init_chunks(const Chunk::Pointer &header_chunk) {
if (header_chunk->data_size() < header_size()) {
LOG_ERROR("Invalid header chunk size");
return IndexError_InvalidFormat;
}
IndexStorage::MemoryBlock header_block;
size_t size = header_chunk->read(0UL, header_block, header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Read header chunk failed");
return IndexError_ReadData;
}
*mutable_header() =
*reinterpret_cast<const HNSWHeader *>(header_block.data());
int ret = check_hnsw_index(&header());
if (ret != 0) {
broker_->close();
return ret;
}
node_chunks_.resize(
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE));
for (auto seq = 0UL; seq < node_chunks_.size(); ++seq) {
node_chunks_[seq] =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, seq);
if (!node_chunks_[seq]) {
LOG_ERROR("Missing hnsw streamer data chunk %zu th of %zu", seq,
node_chunks_.size());
return IndexError_InvalidFormat;
}
}
upper_neighbor_chunks_.resize(
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR));
for (auto seq = 0UL; seq < upper_neighbor_chunks_.size(); ++seq) {
upper_neighbor_chunks_[seq] = broker_->get_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, seq);
if (!upper_neighbor_chunks_[seq]) {
LOG_ERROR("Missing hnsw streamer index chunk %zu th of %zu", seq,
upper_neighbor_chunks_.size());
return IndexError_InvalidFormat;
}
}
return 0;
}
int HnswRabitqStreamerEntity::open(IndexStorage::Pointer stg,
uint64_t max_index_size, bool check_crc) {
std::lock_guard<std::mutex> lock(mutex_);
bool huge_page = stg->isHugePage();
LOG_DEBUG("huge_page: %d", (int)huge_page);
int ret = init_chunk_params(max_index_size, huge_page);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("init_chunk_params failed for %s", IndexError::What(ret));
return ret;
}
ret = broker_->open(std::move(stg), max_index_size_, chunk_size_, check_crc);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Open index failed for %s", IndexError::What(ret));
return ret;
}
ret = upper_neighbor_index_->init(broker_, upper_neighbor_chunk_size_,
scaling_factor(), estimate_doc_capacity(),
kUpperHashMemoryInflateRatio);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Init neighbor hash map failed");
return ret;
}
//! init header
auto header_chunk =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId);
if (!header_chunk) { // open empty index, create one
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId,
header_size());
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc header chunk failed");
return p.first;
}
size_t size = p.second->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
//! Open an exist hnsw index
ret = init_chunks(header_chunk);
if (ailego_unlikely(ret != 0)) {
return ret;
}
//! total docs including features wrote in index but neighbors may not ready
node_id_t total_vecs = 0;
if (node_chunks_.size() > 0) {
size_t last_idx = node_chunks_.size() - 1;
auto last_chunk = node_chunks_[last_idx];
if (last_chunk->data_size() % node_size()) {
LOG_WARN("The index may broken");
return IndexError_InvalidFormat;
}
total_vecs = last_idx * node_cnt_per_chunk_ +
node_chunks_[last_idx]->data_size() / node_size();
}
LOG_INFO(
"Open index, l0NeighborCnt=%zu upperNeighborCnt=%zu "
"efConstruction=%zu curDocCnt=%u totalVecs=%u maxLevel=%u",
l0_neighbor_cnt(), upper_neighbor_cnt(), ef_construction(), doc_cnt(),
total_vecs, cur_max_level());
//! try to correct the docCnt if index not fully flushed
if (doc_cnt() != total_vecs) {
LOG_WARN("Index closed abnormally, using totalVecs as curDocCnt");
*mutable_doc_cnt() = total_vecs;
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
for (node_id_t id = 0U; id < doc_cnt(); ++id) {
if (get_key(id) == kInvalidKey) {
continue;
}
(*keys_map_)[get_key(id)] = id;
}
}
}
stats_.set_loaded_count(doc_cnt());
return 0;
}
int HnswRabitqStreamerEntity::close() {
LOG_DEBUG("close index");
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
mutable_header()->reset();
upper_neighbor_index_->cleanup();
keys_map_->clear();
header_.clear();
node_chunks_.clear();
upper_neighbor_chunks_.clear();
return broker_->close();
}
int HnswRabitqStreamerEntity::flush(uint64_t checkpoint) {
LOG_INFO("Flush index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
int ret = broker_->flush(checkpoint);
if (ret != 0) {
return ret;
}
return 0;
}
int HnswRabitqStreamerEntity::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("Dump index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
//! sort by keys, to support get_vector by key in searcher
std::vector<key_t> keys(doc_cnt());
for (node_id_t i = 0; i < doc_cnt(); ++i) {
keys[i] = get_key(i);
}
//! dump neighbors
auto get_level = [&](node_id_t id) {
auto it = upper_neighbor_index_->find(id);
if (it == upper_neighbor_index_->end()) {
return 0U;
};
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
return meta->level;
};
auto ret = dump_segments(dumper, keys.data(), get_level);
if (ailego_unlikely(ret < 0)) {
return ret;
}
*stats_.mutable_dumped_size() += ret;
return 0;
}
int HnswRabitqStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const {
if (l0_neighbor_cnt() != hd->l0_neighbor_cnt() ||
upper_neighbor_cnt() != hd->upper_neighbor_cnt()) {
LOG_ERROR("Param neighbor cnt: %zu:%zu mismatch index previous %zu:%zu",
l0_neighbor_cnt(), upper_neighbor_cnt(), hd->l0_neighbor_cnt(),
hd->upper_neighbor_cnt());
return IndexError_Mismatch;
}
if (vector_size() != hd->vector_size()) {
LOG_ERROR("vector size %zu mismatch index previous %zu", vector_size(),
hd->vector_size());
return IndexError_Mismatch;
}
if (ef_construction() != hd->ef_construction()) {
LOG_WARN("Param efConstruction %zu mismatch index previous %zu",
ef_construction(), hd->ef_construction());
}
if (scaling_factor() != hd->scaling_factor()) {
LOG_WARN("Param scalingFactor %zu mismatch index previous %zu",
scaling_factor(), hd->scaling_factor());
return IndexError_Mismatch;
}
if (prune_cnt() != hd->neighbor_prune_cnt()) {
LOG_WARN("Param pruneCnt %zu mismatch index previous %zu", prune_cnt(),
hd->neighbor_prune_cnt());
return IndexError_Mismatch;
}
if ((hd->entry_point() != kInvalidNodeId &&
hd->entry_point() >= hd->doc_cnt()) ||
(hd->entry_point() == kInvalidNodeId && hd->doc_cnt() > 0U)) {
LOG_WARN("Invalid entryPoint %zu, docCnt %zu",
static_cast<size_t>(hd->entry_point()),
static_cast<size_t>(hd->doc_cnt()));
return IndexError_InvalidFormat;
}
if (hd->entry_point() == kInvalidNodeId &&
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE) > 0) {
LOG_WARN("The index is broken, maybe it haven't flush");
return IndexError_InvalidFormat;
}
return 0;
}
int HnswRabitqStreamerEntity::add_vector(level_t level, key_t key,
const void *vec, node_id_t *id) {
Chunk::Pointer node_chunk;
size_t chunk_offset = -1UL;
std::lock_guard<std::mutex> lock(mutex_);
// duplicate check
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
LOG_WARN("Try to add duplicate key, ignore it");
return IndexError_Duplicate;
}
node_id_t local_id = static_cast<node_id_t>(doc_cnt());
uint32_t chunk_index = node_chunks_.size() - 1U;
if (chunk_index == -1U ||
(node_chunks_[chunk_index]->data_size() >=
node_cnt_per_chunk_ * node_size())) { // no space left and need to alloc
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
chunk_index++;
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
chunk_index, chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
chunk_offset = 0UL;
node_chunks_.emplace_back(node_chunk);
} else {
node_chunk = node_chunks_[chunk_index];
chunk_offset = node_chunk->data_size();
}
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
//! level 0 neighbors is inited to zero by default
int ret = add_upper_neighbor(level, local_id);
if (ret != 0) {
return ret;
}
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = local_id;
keys_map_lock_->unlock();
}
}
*mutable_doc_cnt() += 1;
broker_->mark_dirty();
*id = local_id;
return 0;
}
int HnswRabitqStreamerEntity::add_vector_with_id(level_t level, node_id_t id,
const void *vec) {
Chunk::Pointer node_chunk;
size_t chunk_offset = -1UL;
key_t key = id;
std::lock_guard<std::mutex> lock(mutex_);
// duplicate check
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
LOG_WARN("Try to add duplicate key, ignore it");
return IndexError_Duplicate;
}
// set node_chunk & chunk_offset if succeed
auto func_get_node_chunk_and_offset = [&](node_id_t node_id) -> int {
uint32_t chunk_index = node_id >> node_index_mask_bits_;
ailego_assert_with(chunk_index <= node_chunks_.size(), "invalid chunk idx");
// belongs to next chunk
if (chunk_index == node_chunks_.size()) {
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
chunk_index, chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
node_chunks_.emplace_back(node_chunk);
}
node_chunk = node_chunks_[chunk_index];
chunk_offset = (node_id & node_index_mask_) * node_size();
return 0;
};
for (size_t start_id = doc_cnt(); start_id < id; ++start_id) {
if (auto ret = func_get_node_chunk_and_offset(start_id); ret != 0) {
LOG_ERROR("func_get_node_chunk_and_offset failed");
return ret;
}
size_t size = node_chunk->write(chunk_offset + vector_size(), &kInvalidKey,
sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write key failed, ret=%zu", size);
return IndexError_WriteData;
}
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (auto ret = func_get_node_chunk_and_offset(id); ret != 0) {
LOG_ERROR("func_get_node_chunk_and_offset failed");
return ret;
}
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
//! level 0 neighbors is inited to zero by default
int ret = add_upper_neighbor(level, id);
if (ret != 0) {
return ret;
}
if (*mutable_doc_cnt() <= id) {
*mutable_doc_cnt() = id + 1;
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = id;
keys_map_lock_->unlock();
}
}
broker_->mark_dirty();
return 0;
}
void HnswRabitqStreamerEntity::update_ep_and_level(node_id_t ep,
level_t level) {
HnswRabitqEntity::update_ep_and_level(ep, level);
flush_header();
return;
}
const HnswRabitqEntity::Pointer HnswRabitqStreamerEntity::clone() const {
std::vector<Chunk::Pointer> node_chunks;
node_chunks.reserve(node_chunks_.size());
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
node_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!node_chunks[i])) {
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
return HnswRabitqEntity::Pointer();
}
}
std::vector<Chunk::Pointer> upper_neighbor_chunks;
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
return HnswRabitqEntity::Pointer();
}
}
HnswRabitqStreamerEntity *entity =
new (std::nothrow) HnswRabitqStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswRabitqStreamerEntity new failed");
}
return HnswRabitqEntity::Pointer(entity);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,522 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <ailego/parallel/lock.h>
#include <sparsehash/dense_hash_map>
#include <sparsehash/dense_hash_set>
#include <zvec/ailego/container/heap.h>
#include "zvec/core/framework/index_framework.h"
#include "hnsw_rabitq_chunk.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_index_hash.h"
#include "hnsw_rabitq_params.h"
namespace zvec {
namespace core {
//! HnswRabitqStreamerEntity manage vector data, pkey, and node's neighbors
class HnswRabitqStreamerEntity : public HnswRabitqEntity {
public:
//! Cleanup
//! return 0 on success, or errCode in failure
virtual int cleanup() override;
//! Make a copy of streamer entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
//! Get vectors feature data by local ids
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t level, node_id_t id,
const void *vec) override;
virtual int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
override;
//! Append neighbor_id to node id neighbors on level
//! Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
}
public:
//! Constructor
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats);
//! Destructor
~HnswRabitqStreamerEntity();
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);
} else {
return IndexError_InvalidArgument;
}
}
//! Init entity
int init(size_t max_doc_cnt);
//! Flush graph entity to disk
//! return 0 on success, or errCode in failure
int flush(uint64_t checkpoint);
//! Open entity from storage
//! return 0 on success, or errCode in failure
int open(IndexStorage::Pointer stg, uint64_t max_index_size, bool check_crc);
//! Close entity
//! return 0 on success, or errCode in failure
int close();
//! Set meta information from entity
int set_index_meta(const IndexMeta &meta) const {
return IndexHelper::SerializeToStorage(meta, broker_->storage().get());
}
//! Get meta information from entity
int get_index_meta(IndexMeta *meta) const {
return IndexHelper::DeserializeFromStorage(broker_->storage().get(), meta);
}
//! Set params: chunk size
inline void set_chunk_size(size_t val) {
chunk_size_ = val;
}
//! Set params
inline void set_filter_same_key(bool val) {
filter_same_key_ = val;
}
//! Set params
inline void set_get_vector(bool val) {
get_vector_enabled_ = val;
}
//! Get vector local id by key
inline node_id_t get_id(key_t key) const {
if (use_key_info_map_) {
keys_map_lock_->lock_shared();
auto it = keys_map_->find(key);
keys_map_lock_->unlock_shared();
return it == keys_map_->end() ? kInvalidNodeId : it->second;
} else {
return key;
}
}
void print_key_map() const {
std::cout << "key map begins" << std::endl;
auto iter = keys_map_->begin();
while (iter != keys_map_->end()) {
std::cout << "key: " << iter->first << ", id: " << iter->second
<< std::endl;
;
iter++;
}
std::cout << "key map ends" << std::endl;
}
//! Get l0 neighbors size
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
//! Get neighbors size for level > 0
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
private:
union UpperNeighborIndexMeta {
struct {
uint32_t level : 4;
uint32_t index : 28; // index is composite type: chunk idx, and the
// N th neighbors in chunk, they two composite
// the 28 bits location
};
uint32_t data;
};
template <class Key, class T>
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
template <class Key, class T>
using HashMapPointer = std::shared_ptr<HashMap<Key, T>>;
template <class Key>
using HashSet = google::dense_hash_set<Key, std::hash<Key>>;
template <class Key>
using HashSetPointer = std::shared_ptr<HashSet<Key>>;
//! upper neighbor index hashmap
using NIHashMap = HnswIndexHashMap<node_id_t, uint32_t>;
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
//! Private construct, only be called by clone method
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
uint32_t upper_neighbor_mask_bits,
bool filter_same_key, bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map,
bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const HnswRabitqChunkBroker::Pointer &broker)
: stats_(stats),
chunk_size_(chunk_size),
node_index_mask_bits_(node_index_mask_bits),
node_cnt_per_chunk_(1UL << node_index_mask_bits_),
node_index_mask_(node_cnt_per_chunk_ - 1),
upper_neighbor_mask_bits_(upper_neighbor_mask_bits),
upper_neighbor_mask_((1U << upper_neighbor_mask_bits_) - 1),
filter_same_key_(filter_same_key),
get_vector_enabled_(get_vector_enabled),
use_key_info_map_(use_key_info_map),
upper_neighbor_index_(upper_neighbor_index),
keys_map_lock_(keys_map_lock),
keys_map_(keys_map),
node_chunks_(std::move(node_chunks)),
upper_neighbor_chunks_(std::move(upper_neighbor_chunks)),
broker_(broker) {
*mutable_header() = hd;
neighbor_size_ = neighbors_size();
upper_neighbor_size_ = upper_neighbors_size();
}
//! Called only in searching procedure per context, so no need to lock
void sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE type, size_t idx,
std::vector<Chunk::Pointer> *chunks) const {
if (ailego_likely(idx < chunks->size())) {
return;
}
for (size_t i = chunks->size(); i <= idx; ++i) {
auto chunk = broker_->get_chunk(type, i);
// the storage can ensure get chunk will success after the first get
ailego_assert_with(!!chunk, "get chunk failed");
chunks->emplace_back(std::move(chunk));
}
}
//! return pair: chunk index + chunk offset
inline std::pair<uint32_t, uint32_t> get_vector_chunk_loc(
node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
return std::make_pair(chunk_idx, offset);
}
//! return pair: chunk index + chunk offset
inline std::pair<uint32_t, uint32_t> get_key_chunk_loc(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
return std::make_pair(chunk_idx, offset);
}
inline std::pair<uint32_t, uint32_t> get_upper_neighbor_chunk_loc(
level_t level, node_id_t id) const {
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
uint32_t offset = (((meta->index) & upper_neighbor_mask_) + level - 1) *
upper_neighbor_size_;
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, chunk_idx,
&upper_neighbor_chunks_);
ailego_assert_abort(chunk_idx < upper_neighbor_chunks_.size(),
"invalid chunk idx");
ailego_assert_abort(offset < upper_neighbor_chunks_[chunk_idx]->data_size(),
"invalid chunk offset");
return std::make_pair(chunk_idx, offset);
}
//! return pair: chunk + chunk offset
inline std::pair<Chunk *, size_t> get_neighbor_chunk_loc(level_t level,
node_id_t id) const {
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
ailego_assert_abort(chunk_idx < node_chunks_.size(), "invalid chunk idx");
ailego_assert_abort(offset < node_chunks_[chunk_idx]->data_size(),
"invalid chunk offset");
return std::make_pair(node_chunks_[chunk_idx].get(), offset);
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
return std::make_pair(upper_neighbor_chunks_[p.first].get(), p.second);
}
}
//! Chunk hnsw index valid
int check_hnsw_index(const HNSWHeader *hd) const;
size_t get_total_upper_neighbors_size(level_t level) const {
return level * upper_neighbor_size_;
}
//! Add upper neighbor header and reserve space for upper neighbor
int add_upper_neighbor(level_t level, node_id_t id) {
if (level == 0) {
return 0;
}
Chunk::Pointer chunk;
uint64_t chunk_offset = -1UL;
size_t neighbors_size = get_total_upper_neighbors_size(level);
uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL;
if (chunk_index == -1UL ||
(upper_neighbor_chunks_[chunk_index]->padding_size() <
neighbors_size)) { // no space left and need to alloc
chunk_index++;
if (ailego_unlikely(upper_neighbor_chunks_.capacity() ==
upper_neighbor_chunks_.size())) {
LOG_ERROR("add upper neighbor failed for no memory quota");
return IndexError_IndexFull;
}
auto p =
broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR,
chunk_index, upper_neighbor_chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
chunk = p.second;
chunk_offset = 0UL;
upper_neighbor_chunks_.emplace_back(chunk);
} else {
chunk = upper_neighbor_chunks_[chunk_index];
chunk_offset = chunk->data_size();
}
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
ailego_assert_with(chunk_offset % upper_neighbor_size_ == 0,
"invalid offset");
ailego_assert_with((chunk_offset / upper_neighbor_size_) <
(1U << upper_neighbor_mask_bits_),
"invalid offset");
ailego_assert_with(chunk_index < (1U << (28 - upper_neighbor_mask_bits_)),
"invalid chunk index");
UpperNeighborIndexMeta meta;
meta.level = level;
meta.index = (chunk_index << upper_neighbor_mask_bits_) |
(chunk_offset / upper_neighbor_size_);
chunk_offset += upper_neighbor_size_ * level;
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
return IndexError_Runtime;
}
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
return IndexError_Runtime;
}
return 0;
}
size_t estimate_doc_capacity() const {
return node_chunks_.capacity() * node_cnt_per_chunk_;
}
int init_chunk_params(size_t max_index_size, bool huge_page) {
node_cnt_per_chunk_ = std::max<uint32_t>(1, chunk_size_ / node_size());
//! align node cnt per chunk to pow of 2
node_index_mask_bits_ = std::ceil(std::log2(node_cnt_per_chunk_));
node_cnt_per_chunk_ = 1UL << node_index_mask_bits_;
if (huge_page) {
chunk_size_ = AlignHugePageSize(node_cnt_per_chunk_ * node_size());
} else {
chunk_size_ = AlignPageSize(node_cnt_per_chunk_ * node_size());
}
node_index_mask_ = node_cnt_per_chunk_ - 1;
if (max_index_size == 0UL) {
max_index_size_ = chunk_size_ * kDefaultMaxChunkCnt;
} else {
max_index_size_ = max_index_size;
}
//! To get a balanced upper neighbor chunk size.
//! If the upper chunk size is equal to node chunk size, it may waste
//! upper neighbor chunk space; if the upper neighbor chunk size is too
//! small, the will need large upper neighbor chunks index space. So to
//! get a balanced ratio be sqrt of the node/neighbor size ratio
float ratio =
std::sqrt(node_size() * scaling_factor() * 1.0f / upper_neighbor_size_);
if (huge_page) {
upper_neighbor_chunk_size_ = AlignHugePageSize(
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
static_cast<size_t>(chunk_size_ / ratio)));
} else {
upper_neighbor_chunk_size_ = AlignPageSize(
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
static_cast<size_t>(chunk_size_ / ratio)));
}
upper_neighbor_mask_bits_ =
std::ceil(std::log2(upper_neighbor_chunk_size_ / upper_neighbor_size_));
upper_neighbor_mask_ = (1 << upper_neighbor_mask_bits_) - 1;
size_t max_node_chunk_cnt = std::ceil(max_index_size_ / chunk_size_);
size_t max_upper_chunk_cnt = std::ceil(
(max_node_chunk_cnt * node_cnt_per_chunk_ * 1.0f / scaling_factor()) /
(upper_neighbor_chunk_size_ / upper_neighbor_size_));
max_upper_chunk_cnt =
max_upper_chunk_cnt + std::ceil(max_upper_chunk_cnt / scaling_factor());
//! reserve space to avoid memmove in chunks vector emplace chunk, so
//! as to lock-free in reading chunk
node_chunks_.reserve(max_node_chunk_cnt);
upper_neighbor_chunks_.reserve(max_upper_chunk_cnt);
LOG_DEBUG(
"Settings: nodeSize=%zu chunkSize=%u upperNeighborSize=%u "
"upperNeighborChunkSize=%u "
"nodeCntPerChunk=%u maxChunkCnt=%zu maxNeighborChunkCnt=%zu "
"maxIndexSize=%zu ratio=%.3f",
node_size(), chunk_size_, upper_neighbor_size_,
upper_neighbor_chunk_size_, node_cnt_per_chunk_, max_node_chunk_cnt,
max_upper_chunk_cnt, max_index_size_, ratio);
return 0;
}
//! Init node chunk and neighbor chunks
int init_chunks(const Chunk::Pointer &header_chunk);
int flush_header(void) {
if (!broker_->dirty()) {
// do not need to flush
return 0;
}
auto header_chunk =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId);
if (ailego_unlikely(!header_chunk)) {
LOG_ERROR("get header chunk failed");
return IndexError_Runtime;
}
size_t size = header_chunk->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
private:
HnswRabitqStreamerEntity(const HnswRabitqStreamerEntity &) = delete;
HnswRabitqStreamerEntity &operator=(const HnswRabitqStreamerEntity &) =
delete;
static constexpr uint64_t kUpperHashMemoryInflateRatio = 2.0f;
private:
IndexStreamer::Stats &stats_;
HNSWHeader header_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
uint32_t chunk_size_{kDefaultChunkSize};
uint32_t upper_neighbor_chunk_size_{kDefaultChunkSize};
uint32_t node_index_mask_bits_{0U};
uint32_t node_cnt_per_chunk_{0U};
uint32_t node_index_mask_{0U};
uint32_t neighbor_size_{0U};
uint32_t upper_neighbor_size_{0U};
//! UpperNeighborIndex.index composite chunkIdx and offset in chunk by the
//! following mask
uint32_t upper_neighbor_mask_bits_{0U};
uint32_t upper_neighbor_mask_{0U};
bool filter_same_key_{false};
bool get_vector_enabled_{false};
bool use_key_info_map_{true};
NIHashMapPointer upper_neighbor_index_{};
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_{};
HashMapPointer<key_t, node_id_t> keys_map_{};
//! the chunks will be changed in searcher, so need mutable
//! data chunk include: vector, key, level 0 neighbors
mutable std::vector<Chunk::Pointer> node_chunks_{};
//! upper neighbor chunk inlude: UpperNeighborHeader + (1~level) neighbors
mutable std::vector<Chunk::Pointer> upper_neighbor_chunks_{};
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,315 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_converter.h"
#include <cstring>
#include <memory>
#include <rabitqlib/utils/rotator.hpp>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/ailego/utility/string_helper.h>
#include "ailego/pattern/defer.h"
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/core/framework/index_cluster.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_features.h"
#include "zvec/core/framework/index_holder.h"
#include "zvec/core/framework/index_memory.h"
#include "zvec/core/framework/index_meta.h"
#include "rabitq_params.h"
#include "rabitq_utils.h"
#ifdef _MSC_VER
#define strncasecmp _strnicmp
#endif
namespace zvec {
namespace core {
RabitqConverter::~RabitqConverter() {
this->cleanup();
}
int RabitqConverter::init(const IndexMeta &meta, const ailego::Params &params) {
// Copy meta and ensure it has metric information
meta_ = meta;
dimension_ = meta.dimension();
if (meta_.metric_name().empty()) {
LOG_ERROR("Meta metric is empty");
return IndexError_InvalidArgument;
}
// Round up dimension to multiple of 64
padded_dim_ = ((dimension_ + 63) / 64) * 64;
// Get RaBitQ parameters with defaults
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
ex_bits_ = total_bits - 1;
params.get(PARAM_RABITQ_NUM_CLUSTERS, &num_clusters_);
if (num_clusters_ == 0) {
num_clusters_ = kDefaultNumClusters;
}
if (ex_bits_ > 8) {
LOG_ERROR("Invalid ex_bits: %zu, must be <= 8", ex_bits_);
return IndexError_InvalidArgument;
}
if (meta.data_type() != IndexMeta::DataType::DT_FP32) {
LOG_ERROR("RaBitQ only supports FP32 data type");
return IndexError_Unsupported;
}
params.get(PARAM_RABITQ_SAMPLE_COUNT, &sample_count_);
std::string rotator_type_str;
params.get(PARAM_RABITQ_ROTATOR_TYPE, &rotator_type_str);
if (rotator_type_str.empty()) {
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
} else if (strncasecmp(rotator_type_str.c_str(), "fht", 3) == 0) {
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
} else if (strncasecmp(rotator_type_str.c_str(), "matrix", 6) == 0) {
rotator_type_ = rabitqlib::RotatorType::MatrixRotator;
} else {
LOG_ERROR("Invalid rotator_type: %s", rotator_type_str.c_str());
return IndexError_InvalidArgument;
}
// Create rotator
rotator_.reset(
rabitqlib::choose_rotator<float>(dimension_, rotator_type_, padded_dim_));
LOG_INFO(
"RabitqConverter initialized: dim=%zu, padded_dim=%zu, "
"num_clusters=%zu, ex_bits=%zu, rotator_type=%d[%s] sample_count[%zu]",
dimension_, padded_dim_, num_clusters_, ex_bits_, (int)rotator_type_,
rotator_type_str.c_str(), sample_count_);
return 0;
}
int RabitqConverter::cleanup() {
centroids_.clear();
rotated_centroids_.clear();
result_holder_.reset();
rotator_.reset();
return 0;
}
int RabitqConverter::train(IndexHolder::Pointer holder) {
if (!holder) {
LOG_ERROR("Null holder for training");
return IndexError_InvalidArgument;
}
ailego::ElapsedTime timer;
size_t vector_count = holder->count();
if (vector_count == 0) {
LOG_ERROR("No vectors for training");
return IndexError_InvalidArgument;
}
// do sampling from all data
size_t sample_count = vector_count;
if (sample_count_ > 0) {
sample_count = std::min(sample_count_, vector_count);
}
LOG_INFO("Training with %zu vectors from %zu of holder", sample_count,
vector_count);
auto sampler = std::make_shared<SampleIndexFeatures<CompactIndexFeatures>>(
meta_, sample_count);
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator error");
return IndexError_Runtime;
}
for (; iter->is_valid(); iter->next()) {
sampler->emplace(iter->data());
}
// Holder is not needed, cleanup it.
holder.reset();
if (sampler->count() == 0) {
LOG_ERROR("Load training data error");
return IndexError_InvalidLength;
}
// Create KmeansCluster for training centroids
auto cluster = IndexFactory::CreateCluster("OptKmeansCluster");
if (!cluster) {
LOG_ERROR("Failed to create OptKmeansCluster");
return IndexError_NoExist;
}
// Initialize cluster
LOG_INFO(
"Initializing KmeansCluster with meta: dim=%u, data_type=%d, metric=%s",
meta_.dimension(), (int)meta_.data_type(), meta_.metric_name().c_str());
ailego::Params cluster_params;
int ret = cluster->init(meta_, cluster_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize KmeansCluster: %d", ret);
return ret;
}
ret = cluster->mount(sampler);
if (ret != 0) {
LOG_ERROR("Failed to mount training data: %d", ret);
return ret;
}
cluster->suggest(num_clusters_);
// Perform clustering
IndexCluster::CentroidList cents;
// TODO: support specify threads with argument
auto threads = std::make_shared<SingleQueueIndexThreads>(0, false);
ret = cluster->cluster(threads, cents);
if (ret != 0) {
LOG_ERROR("Failed to perform clustering: %d", ret);
return ret;
}
if (cents.size() != num_clusters_) {
LOG_WARN("Expected %zu clusters, got %zu", num_clusters_, cents.size());
num_clusters_ = cents.size();
}
// Extract original centroids (for LinearSeeker query)
centroids_.resize(num_clusters_ * dimension_);
// Extract rotated centroids (for quantization)
rotated_centroids_.resize(num_clusters_ * padded_dim_);
for (uint32_t i = 0; i < num_clusters_; ++i) {
const float *cent_data = static_cast<const float *>(cents[i].feature());
// Save original centroids
std::memcpy(&centroids_[i * dimension_], cent_data,
dimension_ * sizeof(float));
// Save rotated centroids
this->rotator_->rotate(cent_data, &rotated_centroids_[i * padded_dim_]);
}
stats_.set_trained_count(sampler->count());
stats_.set_trained_costtime(timer.milli_seconds());
LOG_INFO("Training completed: %zu centroids, cost %zu ms", num_clusters_,
static_cast<size_t>(timer.milli_seconds()));
return 0;
}
int RabitqConverter::transform(IndexHolder::Pointer holder) {
if (!holder) {
LOG_ERROR("Null holder for transformation");
return IndexError_InvalidArgument;
}
if (rotated_centroids_.empty()) {
LOG_ERROR("Centroids not trained yet");
return IndexError_NoReady;
}
LOG_ERROR("Not implemented");
return IndexError_NotImplemented;
}
int RabitqConverter::dump(const IndexDumper::Pointer &dumper) {
if (!dumper) {
LOG_ERROR("Null dumper");
return IndexError_InvalidArgument;
}
if (rotated_centroids_.empty() || centroids_.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
ailego::ElapsedTime timer;
size_t dumped_size = 0;
int ret = dump_rabitq_centroids(
dumper, dimension_, padded_dim_, ex_bits_, num_clusters_, rotator_type_,
rotated_centroids_, centroids_, rotator_, &dumped_size);
if (ret != 0) {
return ret;
}
stats_.set_dumped_size(dumped_size);
stats_.set_dumped_costtime(timer.milli_seconds());
LOG_INFO("Dump completed: %zu bytes, cost %zu ms", stats_.dumped_size(),
static_cast<size_t>(timer.milli_seconds()));
return 0;
}
int RabitqConverter::to_reformer(IndexReformer::Pointer *reformer) {
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
memory_dumper->init(ailego::Params());
std::string file_id = ailego::StringHelper::Concat(
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
int ret = memory_dumper->create(file_id);
if (ret != 0) {
LOG_ERROR("Failed to create memory dumper: %d", ret);
return ret;
}
// Release memory
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
ret = this->dump(memory_dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
ret = memory_dumper->close();
if (ret != 0) {
LOG_ERROR("Failed to close memory dumper: %d", ret);
return ret;
}
auto res = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
ret = res->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
ret = memory_storage->open(file_id, false);
if (ret != 0) {
LOG_ERROR("Failed to open memory storage: %d", ret);
return ret;
}
ret = res->load(memory_storage);
if (ret != 0) {
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
return ret;
}
*reformer = std::move(res);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,101 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/utils/rotator.hpp>
#include "zvec/core/framework/index_cluster.h"
#include "zvec/core/framework/index_converter.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_threads.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class RabitqReformer;
/*! RaBitQ Converter
* Trains KMeans centroids and quantizes vectors using RaBitQ
*/
class RabitqConverter : public IndexConverter {
public:
//! Constructor
RabitqConverter() = default;
//! Destructor
~RabitqConverter() override;
//! Initialize Converter
int init(const IndexMeta &meta, const ailego::Params &params) override;
//! Cleanup Converter
int cleanup(void) override;
//! Train the data - perform KMeans clustering
int train(IndexHolder::Pointer holder) override;
//! Transform the data - quantize vectors using RaBitQ
int transform(IndexHolder::Pointer holder) override;
//! Dump centroids and config into storage
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve a holder as result
IndexHolder::Pointer result(void) const override {
return result_holder_;
}
//! Retrieve Index Meta
const IndexMeta &meta(void) const override {
return meta_;
}
int to_reformer(IndexReformer::Pointer *reformer) override;
private:
static inline size_t AlignSize(size_t size) {
return (size + 0x1F) & (~0x1F);
}
private:
IndexMeta meta_;
IndexHolder::Pointer result_holder_;
Stats stats_;
size_t sample_count_{0};
// RaBitQ parameters
size_t num_clusters_{0};
size_t ex_bits_{0};
size_t dimension_{0};
size_t padded_dim_{0};
// Original centroids: num_clusters * dimension (for LinearSeeker query)
std::vector<float> centroids_;
// Rotated centroids: num_clusters * padded_dim (for quantization)
std::vector<float> rotated_centroids_;
// Rotator for vector transformation
rabitqlib::RotatorType rotator_type_{rabitqlib::RotatorType::FhtKacRotator};
std::unique_ptr<rabitqlib::Rotator<float>> rotator_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,49 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
// Local metric type enum that mirrors rabitqlib::MetricType,
// without exposing rabitqlib headers to consumers of this file.
enum class RabitqMetricType {
kL2 = 0,
kIP = 1,
};
// RaBitQ Converter parameters
static const std::string PARAM_RABITQ_NUM_CLUSTERS(
"proxima.rabitq.num_clusters");
static const std::string PARAM_RABITQ_TOTAL_BITS("proxima.rabitq.total_bits");
static const std::string PARAM_RABITQ_METRIC_NAME("proxima.rabitq.metric_name");
static const std::string PARAM_RABITQ_ROTATOR_TYPE(
"proxima.rabitq.rotator.type");
static const std::string PARAM_RABITQ_SAMPLE_COUNT(
"proxima.rabitq.sample_count");
// Default values
constexpr size_t kDefaultNumClusters = 16;
// 4-bit, 5-bit, and 7-bit quantization typically achieve 90%, 95%, and 99%
// recall, respectively—without accessing raw vectors for reranking
constexpr size_t kDefaultRabitqTotalBits = 7;
constexpr int kMinRabitqDimSize = 64;
constexpr int kMaxRabitqDimSize = 4095;
} // namespace core
} // namespace zvec
@@ -0,0 +1,467 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_reformer.h"
#include <string>
#include <vector>
#include <rabitqlib/defines.hpp>
#include <rabitqlib/index/query.hpp>
#include <rabitqlib/quantization/rabitq.hpp>
#include <rabitqlib/utils/rotator.hpp>
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/utility/string_helper.h>
#include "core/algorithm/cluster/linear_seeker.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_features.h"
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_storage.h"
#include "hnsw_rabitq_query_entity.h"
#include "rabitq_converter.h"
#include "rabitq_utils.h"
namespace zvec {
namespace core {
// All rabitqlib types are confined to this translation unit via pimpl.
struct RabitqReformer::Impl {
// RaBitQ parameters
size_t num_clusters{0};
size_t ex_bits{0};
size_t dimension{0};
size_t padded_dim{0};
size_t size_bin_data{0};
size_t size_ex_data{0};
bool loaded{false};
// Original centroids: num_clusters * dimension (for LinearSeeker query)
std::vector<float> centroids;
// Rotated centroids: num_clusters * padded_dim (for quantization)
std::vector<float> rotated_centroids;
rabitqlib::RotatorType rotator_type{rabitqlib::RotatorType::FhtKacRotator};
std::unique_ptr<rabitqlib::Rotator<float>> rotator;
rabitqlib::quant::RabitqConfig query_config;
rabitqlib::quant::RabitqConfig config;
rabitqlib::MetricType metric_type{rabitqlib::METRIC_L2};
LinearSeeker::Pointer centroid_seeker;
CoherentIndexFeatures::Pointer centroid_features;
// Translate local enum to rabitqlib enum (used only inside this .cc).
static rabitqlib::MetricType to_rabitq(RabitqMetricType m) {
return m == RabitqMetricType::kIP ? rabitqlib::METRIC_IP
: rabitqlib::METRIC_L2;
}
// Translate rabitqlib enum to local enum.
static RabitqMetricType from_rabitq(rabitqlib::MetricType m) {
return m == rabitqlib::METRIC_IP ? RabitqMetricType::kIP
: RabitqMetricType::kL2;
}
int quantize_vector(const float *raw_vector, uint32_t cluster_id,
std::string *quantized_data) const;
};
RabitqReformer::RabitqReformer() : impl_(std::make_unique<Impl>()) {}
RabitqReformer::~RabitqReformer() {
this->cleanup();
}
size_t RabitqReformer::num_clusters() const {
return impl_->num_clusters;
}
RabitqMetricType RabitqReformer::rabitq_metric_type() const {
return Impl::from_rabitq(impl_->metric_type);
}
int RabitqReformer::init(const ailego::Params &params) {
std::string metric_name = params.get_as_string(PARAM_RABITQ_METRIC_NAME);
if (metric_name == "SquaredEuclidean") {
impl_->metric_type = rabitqlib::METRIC_L2;
} else if (metric_name == "InnerProduct") {
impl_->metric_type = rabitqlib::METRIC_IP;
} else if (metric_name == "Cosine") {
impl_->metric_type = rabitqlib::METRIC_IP;
} else {
LOG_ERROR("Unsupported metric name: %s", metric_name.c_str());
return IndexError_InvalidArgument;
}
LOG_DEBUG("Rabitq reformer init done. metric_name=%s metric_type=%d",
metric_name.c_str(), static_cast<int>(impl_->metric_type));
return 0;
}
int RabitqReformer::cleanup() {
impl_->centroids.clear();
impl_->rotated_centroids.clear();
impl_->centroid_seeker.reset();
impl_->centroid_features.reset();
impl_->loaded = false;
impl_->rotator.reset();
return 0;
}
int RabitqReformer::unload() {
return this->cleanup();
}
int RabitqReformer::load(IndexStorage::Pointer storage) {
if (!storage) {
LOG_ERROR("Invalid storage for load");
return IndexError_InvalidArgument;
}
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
if (!segment) {
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
size_t offset = 0;
RabitqConverterHeader header;
IndexStorage::MemoryBlock block;
size_t size = segment->read(offset, block, sizeof(header));
if (size != sizeof(header)) {
LOG_ERROR("Failed to read header");
return IndexError_InvalidFormat;
}
memcpy(&header, block.data(), sizeof(header));
impl_->dimension = header.dim;
impl_->padded_dim = header.padded_dim;
impl_->ex_bits = header.ex_bits;
impl_->num_clusters = header.num_clusters;
impl_->rotator_type =
static_cast<rabitqlib::RotatorType>(header.rotator_type);
offset += sizeof(header);
// Read rotated centroids
size_t rotated_centroids_size =
sizeof(float) * header.num_clusters * header.padded_dim;
size = segment->read(offset, block, rotated_centroids_size);
if (size != rotated_centroids_size) {
LOG_ERROR("Failed to read rotated centroids");
return IndexError_InvalidFormat;
}
impl_->rotated_centroids.resize(header.num_clusters * header.padded_dim);
memcpy(impl_->rotated_centroids.data(), block.data(), rotated_centroids_size);
offset += size;
// Read original centroids (for LinearSeeker query)
size_t centroids_size = sizeof(float) * header.num_clusters * header.dim;
size = segment->read(offset, block, centroids_size);
if (size != centroids_size) {
LOG_ERROR("Failed to read centroids");
return IndexError_InvalidFormat;
}
impl_->centroids.resize(header.num_clusters * header.dim);
memcpy(impl_->centroids.data(), block.data(), centroids_size);
offset += size;
// Read rotator
size_t rotator_size = header.rotator_size;
size = segment->read(offset, block, rotator_size);
if (size != rotator_size) {
LOG_ERROR("Failed to read rotator");
return IndexError_InvalidFormat;
}
impl_->rotator.reset(rabitqlib::choose_rotator<float>(
impl_->dimension, impl_->rotator_type, impl_->padded_dim));
impl_->rotator->load(reinterpret_cast<const char *>(block.data()));
offset += size;
impl_->query_config = rabitqlib::quant::faster_config(
impl_->padded_dim, rabitqlib::SplitSingleQuery<float>::kNumBits);
impl_->config =
rabitqlib::quant::faster_config(impl_->padded_dim, impl_->ex_bits + 1);
impl_->size_bin_data =
rabitqlib::BinDataMap<float>::data_bytes(impl_->padded_dim);
impl_->size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
impl_->padded_dim, impl_->ex_bits);
// Initialize LinearSeeker for centroid search
IndexMeta centroid_meta;
centroid_meta.set_data_type(IndexMeta::DataType::DT_FP32);
centroid_meta.set_dimension(static_cast<uint32_t>(impl_->dimension));
// Note:
// 1. spherical kmeans is used for InnerProduct and Cosine, so centroids are
// normalized.
// 2. for Cosine metric, `transform_to_entity` input is normalized, need to
// use InnerProduct metric as Cosine metric requires extra dimension which is
// unsuitable for centroids.
centroid_meta.set_metric(impl_->metric_type == rabitqlib::METRIC_L2
? "SquaredEuclidean"
: "InnerProduct",
0, ailego::Params());
impl_->centroid_features = std::make_shared<CoherentIndexFeatures>();
impl_->centroid_features->mount(centroid_meta, impl_->centroids.data(),
impl_->centroids.size() * sizeof(float));
impl_->centroid_seeker = std::make_shared<LinearSeeker>();
int ret = impl_->centroid_seeker->init(centroid_meta);
if (ret != 0) {
LOG_ERROR("Failed to init centroid seeker. ret[%d]", ret);
return ret;
}
ret = impl_->centroid_seeker->mount(impl_->centroid_features);
if (ret != 0) {
LOG_ERROR("Failed to mount centroid features. ret[%d]", ret);
return ret;
}
LOG_INFO(
"Rabitq reformer load done. dimension=%zu, padded_dim=%zu, "
"ex_bits=%zu, num_clusters=%zu, size_bin_data=%zu, size_ex_data=%zu "
"rotator_type=%d",
impl_->dimension, impl_->padded_dim, impl_->ex_bits, impl_->num_clusters,
impl_->size_bin_data, impl_->size_ex_data, (int)impl_->rotator_type);
impl_->loaded = true;
return 0;
}
int RabitqReformer::convert(const void *record, const IndexQueryMeta &rmeta,
std::string *out, IndexQueryMeta *ometa) const {
if (!impl_->loaded) {
LOG_ERROR("Centroids not loaded yet");
return IndexError_NoReady;
}
if (!record || !out) {
LOG_ERROR("Invalid arguments for convert");
return IndexError_InvalidArgument;
}
// input may be transformed, require rmeta.dimension >= dimension
if (rmeta.dimension() < impl_->dimension ||
rmeta.data_type() != IndexMeta::DataType::DT_FP32) {
LOG_ERROR("Invalid record meta: dimension=%zu, data_type=%d",
static_cast<size_t>(rmeta.dimension()), (int)rmeta.data_type());
return IndexError_InvalidArgument;
}
// Find nearest centroid using LinearSeeker
Seeker::Document doc;
int ret = impl_->centroid_seeker->seek(
record, impl_->dimension * sizeof(float), &doc);
if (ret != 0) {
LOG_ERROR("Failed to seek centroid. ret[%d]", ret);
return ret;
}
uint32_t cluster_id = doc.index;
const float *vector = static_cast<const float *>(record);
ret = impl_->quantize_vector(vector, cluster_id, out);
if (ret != 0) {
LOG_ERROR("Failed to quantize vector");
return ret;
}
ometa->set_meta(IndexMeta::DataType::DT_INT8, (uint32_t)out->size());
return 0;
}
int RabitqReformer::transform(const void *, const IndexQueryMeta &,
std::string *, IndexQueryMeta *) const {
return IndexError_NotImplemented;
}
int RabitqReformer::transform_to_entity(const void *query,
HnswRabitqQueryEntity *entity) const {
if (!impl_->loaded) {
LOG_ERROR("Centroids not loaded yet");
return IndexError_NoReady;
}
if (!query) {
LOG_ERROR("Invalid arguments for transform");
return IndexError_InvalidArgument;
}
const float *query_vector = static_cast<const float *>(query);
// Apply rotator
entity->rotated_query.resize(impl_->padded_dim);
impl_->rotator->rotate(query_vector, entity->rotated_query.data());
// Quantize query to 4-bit representation
entity->query_wrapper = std::make_unique<rabitqlib::SplitSingleQuery<float>>(
entity->rotated_query.data(), impl_->padded_dim, impl_->ex_bits,
impl_->query_config, impl_->metric_type);
// Preprocess - get the distance from query to all centroids
entity->q_to_centroids.resize(impl_->num_clusters);
if (impl_->metric_type == rabitqlib::METRIC_L2) {
for (size_t i = 0; i < impl_->num_clusters; i++) {
entity->q_to_centroids[i] = std::sqrt(rabitqlib::euclidean_sqr(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim));
}
} else if (impl_->metric_type == rabitqlib::METRIC_IP) {
entity->q_to_centroids.resize(impl_->num_clusters * 2);
// first half as g_add, second half as g_error
for (size_t i = 0; i < impl_->num_clusters; i++) {
entity->q_to_centroids[i] = rabitqlib::dot_product(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim);
entity->q_to_centroids[i + impl_->num_clusters] =
std::sqrt(rabitqlib::euclidean_sqr(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim));
}
}
return 0;
}
int RabitqReformer::Impl::quantize_vector(const float *raw_vector,
uint32_t cluster_id,
std::string *quantized_data) const {
std::vector<float> rotated_data(padded_dim);
rotator->rotate(raw_vector, rotated_data.data());
// quantized format: cluster_id + bin_data + ex_data
quantized_data->resize(sizeof(cluster_id) + size_bin_data + size_ex_data);
memcpy(&(*quantized_data)[0], &cluster_id, sizeof(cluster_id));
int bin_data_offset = sizeof(cluster_id);
int ex_data_offset = bin_data_offset + size_bin_data;
rabitqlib::quant::quantize_split_single(
rotated_data.data(), rotated_centroids.data() + (cluster_id * padded_dim),
padded_dim, ex_bits, &(*quantized_data)[bin_data_offset],
&(*quantized_data)[ex_data_offset], metric_type, config);
return 0;
}
int RabitqReformer::dump(const IndexDumper::Pointer &dumper) {
if (!dumper) {
LOG_ERROR("Null dumper");
return IndexError_InvalidArgument;
}
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
impl_->centroids.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
size_t dumped_size = 0;
int ret = dump_rabitq_centroids(
dumper, impl_->dimension, impl_->padded_dim, impl_->ex_bits,
impl_->num_clusters, impl_->rotator_type, impl_->rotated_centroids,
impl_->centroids, impl_->rotator, &dumped_size);
if (ret != 0) {
return ret;
}
LOG_INFO("RabitqReformer dump completed: %zu bytes", dumped_size);
return 0;
}
int RabitqReformer::dump(const IndexStorage::Pointer &storage) {
if (!storage) {
LOG_ERROR("Null storage");
return IndexError_InvalidArgument;
}
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
impl_->centroids.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
auto align_size = [](size_t size) -> size_t {
return (size + 0x1F) & (~0x1F);
};
size_t header_size = sizeof(RabitqConverterHeader);
size_t rotated_centroids_size =
impl_->rotated_centroids.size() * sizeof(float);
size_t centroids_size = impl_->centroids.size() * sizeof(float);
size_t rotator_size = impl_->rotator->dump_bytes();
size_t data_size =
header_size + rotated_centroids_size + centroids_size + rotator_size;
size_t total_size = align_size(data_size);
int ret = storage->append(RABITQ_CONVERTER_SEG_ID, total_size);
if (ret != 0) {
LOG_ERROR("Failed to append segment %s, ret=%d",
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
return ret;
}
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
if (!segment) {
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
return IndexError_ReadData;
}
size_t offset = 0;
RabitqConverterHeader header;
header.dim = static_cast<uint32_t>(impl_->dimension);
header.padded_dim = static_cast<uint32_t>(impl_->padded_dim);
header.num_clusters = static_cast<uint32_t>(impl_->num_clusters);
header.ex_bits = static_cast<uint8_t>(impl_->ex_bits);
header.rotator_type = static_cast<uint8_t>(impl_->rotator_type);
header.rotator_size = static_cast<uint32_t>(rotator_size);
size_t written = segment->write(offset, &header, header_size);
if (written != header_size) {
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", written,
header_size);
return IndexError_WriteData;
}
offset += header_size;
written = segment->write(offset, impl_->rotated_centroids.data(),
rotated_centroids_size);
if (written != rotated_centroids_size) {
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
written, rotated_centroids_size);
return IndexError_WriteData;
}
offset += rotated_centroids_size;
written = segment->write(offset, impl_->centroids.data(), centroids_size);
if (written != centroids_size) {
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", written,
centroids_size);
return IndexError_WriteData;
}
offset += centroids_size;
std::vector<char> buffer(rotator_size);
impl_->rotator->save(buffer.data());
written = segment->write(offset, buffer.data(), rotator_size);
if (written != rotator_size) {
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu",
written, rotator_size);
return IndexError_WriteData;
}
LOG_INFO("RabitqReformer dump to storage completed: %zu bytes", data_size);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,72 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#pragma once
#include <memory>
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_storage.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class HnswRabitqQueryEntity;
/*! RaBitQ Reformer
* Loads centroids and performs query transformation and vector quantization.
*
* All rabitqlib types are hidden behind a pimpl to avoid leaking rabitqlib
* headers to consumers of this class.
*/
class RabitqReformer : public IndexReformer {
public:
typedef std::shared_ptr<RabitqReformer> Pointer;
RabitqReformer();
~RabitqReformer() override;
// Non-copyable
RabitqReformer(const RabitqReformer &) = delete;
RabitqReformer &operator=(const RabitqReformer &) = delete;
int init(const ailego::Params &params) override;
int cleanup(void) override;
int load(IndexStorage::Pointer storage) override;
int unload(void) override;
// transform() is not implemented for RabitqReformer; use transform_to_entity.
int transform(const void *query, const IndexQueryMeta &qmeta,
std::string *out, IndexQueryMeta *ometa) const override;
int convert(const void *record, const IndexQueryMeta &rmeta, std::string *out,
IndexQueryMeta *ometa) const override;
int dump(const IndexDumper::Pointer &dumper);
int dump(const IndexStorage::Pointer &storage);
int transform_to_entity(const void *query,
HnswRabitqQueryEntity *entity) const;
size_t num_clusters() const;
RabitqMetricType rabitq_metric_type() const;
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,115 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_utils.h"
#include <string>
#include <zvec/ailego/hash/crc32c.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_logger.h"
namespace zvec {
namespace core {
int dump_rabitq_centroids(
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
const std::vector<float> &rotated_centroids,
const std::vector<float> &centroids,
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
size_t *out_dumped_size) {
auto align_size = [](size_t size) -> size_t {
return (size + 0x1F) & (~0x1F);
};
uint32_t crc = 0;
size_t dumped_size = 0;
// Write header
RabitqConverterHeader header;
header.dim = static_cast<uint32_t>(dimension);
header.padded_dim = static_cast<uint32_t>(padded_dim);
header.num_clusters = static_cast<uint32_t>(num_clusters);
header.ex_bits = static_cast<uint8_t>(ex_bits);
header.rotator_type = static_cast<uint8_t>(rotator_type);
header.rotator_size = static_cast<uint32_t>(rotator->dump_bytes());
size_t size = dumper->write(&header, sizeof(header));
if (size != sizeof(header)) {
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", size,
sizeof(header));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(&header, sizeof(header), crc);
dumped_size += size;
// Write rotated centroids
size = dumper->write(rotated_centroids.data(),
rotated_centroids.size() * sizeof(float));
if (size != rotated_centroids.size() * sizeof(float)) {
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
size, rotated_centroids.size() * sizeof(float));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(rotated_centroids.data(),
rotated_centroids.size() * sizeof(float), crc);
dumped_size += size;
// Write original centroids
size = dumper->write(centroids.data(), centroids.size() * sizeof(float));
if (size != centroids.size() * sizeof(float)) {
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", size,
centroids.size() * sizeof(float));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(centroids.data(), centroids.size() * sizeof(float),
crc);
dumped_size += size;
// Write rotator data
std::vector<char> buffer(rotator->dump_bytes());
rotator->save(buffer.data());
size = dumper->write(buffer.data(), buffer.size());
if (size != buffer.size()) {
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu", size,
buffer.size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer.data(), buffer.size(), crc);
dumped_size += size;
// Write padding
size_t padding_size = align_size(dumped_size) - dumped_size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
int ret =
dumper->append(RABITQ_CONVERTER_SEG_ID, dumped_size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d",
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
return ret;
}
if (out_dumped_size) {
*out_dumped_size = dumped_size;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,53 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/utils/rotator.hpp>
#include "zvec/core/framework/index_dumper.h"
namespace zvec {
namespace core {
inline const std::string RABITQ_CONVERTER_SEG_ID{"rabitq.converter"};
struct RabitqConverterHeader {
uint32_t num_clusters;
uint32_t dim;
uint32_t padded_dim;
uint32_t rotator_size;
uint8_t ex_bits;
uint8_t rotator_type;
uint8_t padding[2];
uint32_t reserve[3];
RabitqConverterHeader() {
memset(this, 0, sizeof(RabitqConverterHeader));
}
};
static_assert(sizeof(RabitqConverterHeader) % 32 == 0,
"RabitqConverterHeader must be aligned with 32 bytes");
// Common dump implementation for RabitqConverter and RabitqReformer
int dump_rabitq_centroids(
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
const std::vector<float> &rotated_centroids,
const std::vector<float> &centroids,
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
size_t *out_dumped_size = nullptr);
} // namespace core
} // namespace zvec
+1 -1
View File
@@ -5,6 +5,6 @@ cc_library(
NAME core_interface STATIC STRICT ALWAYS_LINK
SRCS *.cc indexes/*.cc
INCS . ${PROJECT_ROOT_DIR}/src/ ${PROJECT_ROOT_DIR}/src/core
LIBS zvec_ailego core_framework sparsehash magic_enum
LIBS zvec_ailego core_framework sparsehash magic_enum rabitqlib
VERSION "${PROXIMA_ZVEC_VERSION}"
)
+37
View File
@@ -134,6 +134,14 @@ int Index::CreateAndInitConverterReformer(const QuantizerParam &param,
return core::IndexError_Unsupported;
}
break;
case QuantizerType::kRabitq:
if (index_param.data_type == DataType::DT_FP32) {
converter_name = "CosineNormalizeConverter";
} else {
LOG_ERROR("Unsupported data type: ");
return core::IndexError_Unsupported;
}
break;
case QuantizerType::kFP16:
converter_name = "CosineFp16Converter";
break;
@@ -160,6 +168,9 @@ int Index::CreateAndInitConverterReformer(const QuantizerParam &param,
case QuantizerType::kInt4:
converter_name = "Int4StreamingConverter";
break;
case QuantizerType::kRabitq:
// no converter here
return 0;
default:
LOG_ERROR("Unsupported quantizer type: ");
return core::IndexError_Unsupported;
@@ -814,4 +825,30 @@ int Index::_get_coarse_search_topk(
return floor(search_param->topk * scale_factor);
}
std::string Index::get_metric_name(MetricType metric_type, bool is_sparse) {
if (is_sparse) {
switch (metric_type) {
case MetricType::kInnerProduct:
return "InnerProductSparse";
case MetricType::kMIPSL2sq:
return "MipsSquaredEuclideanSparse";
default:
return "";
}
} else {
switch (metric_type) {
case MetricType::kL2sq:
return "SquaredEuclidean";
case MetricType::kInnerProduct:
return "InnerProduct";
case MetricType::kCosine:
return "Cosine";
case MetricType::kMIPSL2sq:
return "MipsSquaredEuclidean";
default:
return "";
}
}
}
} // namespace zvec::core_interface
+33
View File
@@ -45,6 +45,8 @@ Index::Pointer IndexFactory::CreateAndInitIndex(const BaseIndexParam &param) {
ptr = std::make_shared<HNSWIndex>();
} else if (param.index_type == IndexType::kIVF) {
ptr = std::make_shared<IVFIndex>();
} else if (param.index_type == IndexType::kHNSWRabitq) {
ptr = std::make_shared<HNSWRabitqIndex>();
} else {
LOG_ERROR("Unsupported index type: ");
return nullptr;
@@ -104,6 +106,15 @@ BaseIndexParam::Pointer IndexFactory::DeserializeIndexParamFromJson(
}
return param;
}
case IndexType::kHNSWRabitq: {
HNSWRabitqIndexParam::Pointer param =
std::make_shared<HNSWRabitqIndexParam>();
if (!param->DeserializeFromJson(json_str)) {
LOG_ERROR("Failed to deserialize hnsqrabitq index param");
return nullptr;
}
return param;
}
default:
LOG_ERROR("Unsupported index type: %s",
magic_enum::enum_name(index_type).data());
@@ -151,6 +162,11 @@ std::string IndexFactory::QueryParamSerializeToJson(const QueryParamType &param,
// ailego::JsonValue(QueryParamSerializeToJson(param.l1QueryParam)));
// json_obj.set("l2QueryParam",
// ailego::JsonValue(QueryParamSerializeToJson(param.l2QueryParam)));
} else if constexpr (std::is_same_v<QueryParamType, HNSWRabitqQueryParam>) {
if (!omit_empty_value || param.ef_search != 0) {
json_obj.set("ef_search", ailego::JsonValue(param.ef_search));
}
index_type = IndexType::kHNSWRabitq;
}
json_obj.set("index_type",
@@ -245,6 +261,17 @@ typename QueryParamType::Pointer IndexFactory::QueryParamDeserializeFromJson(
return nullptr;
}
return param;
} else if (index_type == IndexType::kHNSWRabitq) {
auto param = std::make_shared<HNSWRabitqQueryParam>();
if (!parse_common_fields(param)) {
return nullptr;
}
if (!extract_value_from_json(json_obj, "ef_search", param->ef_search,
tmp_json_value)) {
LOG_ERROR("Failed to deserialize ef_search");
return nullptr;
}
return param;
} else {
LOG_ERROR("Unsupported index type: %s",
magic_enum::enum_name(index_type).data());
@@ -268,6 +295,12 @@ typename QueryParamType::Pointer IndexFactory::QueryParamDeserializeFromJson(
LOG_ERROR("Failed to deserialize nprobe");
return nullptr;
}
} else if constexpr (std::is_same_v<QueryParamType, HNSWRabitqQueryParam>) {
if (!extract_value_from_json(json_obj, "ef_search", param->ef_search,
tmp_json_value)) {
LOG_ERROR("Failed to deserialize ef_search");
return nullptr;
}
} else {
LOG_ERROR("Unsupported index type: %s",
magic_enum::enum_name(index_type).data());
+33
View File
@@ -141,6 +141,39 @@ bool HNSWIndexParam::DeserializeFromJsonObject(
return true;
}
bool HNSWRabitqIndexParam::DeserializeFromJsonObject(
const ailego::JsonObject &json_obj) {
if (!BaseIndexParam::DeserializeFromJsonObject(json_obj)) {
return false;
}
if (index_type != IndexType::kHNSWRabitq) {
LOG_ERROR("index_type is not kHNSWRabitq");
return false;
}
DESERIALIZE_VALUE_FIELD(json_obj, m);
DESERIALIZE_VALUE_FIELD(json_obj, ef_construction);
DESERIALIZE_VALUE_FIELD(json_obj, total_bits);
DESERIALIZE_VALUE_FIELD(json_obj, num_clusters);
DESERIALIZE_VALUE_FIELD(json_obj, sample_count);
return true;
}
ailego::JsonObject HNSWRabitqIndexParam::SerializeToJsonObject(
bool omit_empty_value) const {
auto json_obj = BaseIndexParam::SerializeToJsonObject(omit_empty_value);
json_obj.set("m", ailego::JsonValue(m));
json_obj.set("ef_construction", ailego::JsonValue(ef_construction));
json_obj.set("total_bits", ailego::JsonValue(total_bits));
json_obj.set("num_clusters", ailego::JsonValue(num_clusters));
if (!omit_empty_value || sample_count != 0) {
json_obj.set("sample_count", ailego::JsonValue(sample_count));
}
return json_obj;
}
ailego::JsonObject QuantizerParam::SerializeToJsonObject(
bool omit_empty_value) const {
ailego::JsonObject json_obj;
@@ -0,0 +1,140 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <zvec/core/interface/index.h>
#include "zvec/core/framework/index_error.h"
#if RABITQ_SUPPORTED
#include "algorithm/hnsw_rabitq/hnsw_rabitq_params.h"
#include "algorithm/hnsw_rabitq/hnsw_rabitq_streamer.h"
#include "algorithm/hnsw_rabitq/rabitq_params.h"
#endif
namespace zvec::core_interface {
int HNSWRabitqIndex::CreateAndInitStreamer(const BaseIndexParam &param) {
#if !RABITQ_SUPPORTED
LOG_ERROR("RaBitQ is not supported on this platform (Linux x86_64 only)");
return core::IndexError_Unsupported;
#else
param_ = dynamic_cast<const HNSWRabitqIndexParam &>(param);
if (is_sparse_) {
LOG_ERROR("Sparse index is not supported");
return core::IndexError_Runtime;
}
if (param.dimension < core::kMinRabitqDimSize ||
param.dimension > core::kMaxRabitqDimSize) {
LOG_ERROR("Unsupported dimension: %d", param.dimension);
return core::IndexError_Unsupported;
}
// validate parameters
param_.ef_construction = std::max(1, std::min(2048, param_.ef_construction));
param_.m = std::max(5, std::min(1024, param_.m));
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION,
param_.ef_construction);
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT,
param_.m);
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE,
true);
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_STREAMER_EF,
kDefaultHnswEfSearch);
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP,
param_.use_id_map);
proxima_index_params_.set(core::PARAM_HNSW_RABITQ_GENERAL_DIMENSION,
input_vector_meta_.dimension());
proxima_index_params_.set(core::PARAM_RABITQ_TOTAL_BITS, param_.total_bits);
// num_clusters, sample_count are parameters for rabitq converter
// proxima_index_params_.set(core::PARAM_RABITQ_NUM_CLUSTERS,
// param_.num_clusters);
auto streamer = std::make_shared<core::HnswRabitqStreamer>();
streamer->set_provider(param_.provider);
streamer->set_reformer(param_.reformer);
streamer_ = streamer;
if (ailego_unlikely(!streamer_)) {
LOG_ERROR("Failed to create HnswRabitqStreamer");
return core::IndexError_Runtime;
}
if (ailego_unlikely(
streamer_->init(proxima_index_meta_, proxima_index_params_) != 0)) {
LOG_ERROR("Failed to init HnswRabitqStreamer");
return core::IndexError_Runtime;
}
return 0;
#endif // RABITQ_SUPPORTED
}
int HNSWRabitqIndex::_prepare_for_search(
const VectorData & /*vector_data*/,
const BaseIndexQueryParam::Pointer &search_param,
core::IndexContext::Pointer &context) {
#if !RABITQ_SUPPORTED
LOG_ERROR("RaBitQ is not supported on this platform (Linux x86_64 only)");
return core::IndexError_Unsupported;
#else
const auto &hnsw_search_param =
std::dynamic_pointer_cast<HNSWRabitqQueryParam>(search_param);
if (ailego_unlikely(!hnsw_search_param)) {
LOG_ERROR("Invalid search param type, expected HNSWRabitqQueryParam");
return core::IndexError_Runtime;
}
if (0 >= hnsw_search_param->ef_search ||
hnsw_search_param->ef_search > 2048) {
LOG_ERROR(
"ef_search must be greater than 0 and less than or equal to 2048.");
return core::IndexError_Runtime;
}
context->set_topk(hnsw_search_param->topk);
context->set_fetch_vector(hnsw_search_param->fetch_vector);
if (hnsw_search_param->filter) {
context->set_filter(std::move(*hnsw_search_param->filter));
}
if (hnsw_search_param->radius > 0.0f) {
context->set_threshold(hnsw_search_param->radius);
}
ailego::Params params;
const int real_search_ef =
std::max(1u, std::min(2048u, hnsw_search_param->ef_search));
params.set(core::PARAM_HNSW_RABITQ_STREAMER_EF, real_search_ef);
context->update(params);
return 0;
#endif // RABITQ_SUPPORTED
}
int HNSWRabitqIndex::_get_coarse_search_topk(
const BaseIndexQueryParam::Pointer &search_param) {
#if !RABITQ_SUPPORTED
LOG_ERROR("RaBitQ is not supported on this platform (Linux x86_64 only)");
return -1;
#else
const auto &hnsw_search_param =
std::dynamic_pointer_cast<HNSWRabitqQueryParam>(search_param);
auto ret = std::max(search_param->topk, hnsw_search_param->ef_search);
return ret;
#endif // RABITQ_SUPPORTED
}
} // namespace zvec::core_interface
+3
View File
@@ -61,6 +61,9 @@ constexpr uint32_t kMaxOutputFieldSize = 1024;
constexpr uint32_t kMaxWriteBatchSize = 1024;
constexpr uint32_t kMinRabitqDimSize = 64;
constexpr uint32_t kMaxRabitqDimSize = 4095;
// Inverted index
const std::string INVERT_SUFFIX_TERMS{"$TERMS"};
@@ -20,6 +20,8 @@
#include <zvec/db/doc.h>
#include <zvec/db/query_params.h>
#include <zvec/db/status.h>
#include "zvec/db/index_params.h"
#include "zvec/db/type.h"
#include "vector_column_params.h"
@@ -162,6 +164,25 @@ class ProximaEngineHelper {
return std::move(hnsw_query_param);
}
case IndexType::HNSW_RABITQ: {
auto hnsw_query_param_result =
_build_common_query_param<core_interface::HNSWRabitqQueryParam>(
query_params);
if (!hnsw_query_param_result.has_value()) {
return tl::make_unexpected(Status::InvalidArgument(
"failed to build query param: " +
hnsw_query_param_result.error().message()));
}
auto &hnsw_query_param = hnsw_query_param_result.value();
if (query_params.query_params) {
auto db_hnsw_rabitq_query_params =
dynamic_cast<const HnswRabitqQueryParams *>(
query_params.query_params.get());
hnsw_query_param->ef_search = db_hnsw_rabitq_query_params->ef();
}
return std::move(hnsw_query_param);
}
case IndexType::IVF: {
auto ivf_query_param_result =
_build_common_query_param<core_interface::IVFQueryParam>(
@@ -212,6 +233,8 @@ class ProximaEngineHelper {
return core_interface::QuantizerType::kInt8;
case QuantizeType::INT4:
return core_interface::QuantizerType::kInt4;
case QuantizeType::RABITQ:
return core_interface::QuantizerType::kRabitq;
default:
return tl::make_unexpected(
Status::InvalidArgument("unsupported quantize type"));
@@ -244,6 +267,9 @@ class ProximaEngineHelper {
_build_common_index_param(const FieldSchema &field_schema) {
auto db_index_params = dynamic_cast<const DBIndexParamType *>(
field_schema.index_params().get());
if (db_index_params == nullptr) {
return tl::make_unexpected(Status::InvalidArgument("bad_cast"));
}
auto index_param_builder = std::make_shared<IndexParamBuilderType>();
// db will ensure the id is consecutive
@@ -322,6 +348,32 @@ class ProximaEngineHelper {
return index_param_builder->Build();
}
case IndexType::HNSW_RABITQ: {
auto index_param_builder_result = _build_common_index_param<
HnswRabitqIndexParams, core_interface::HNSWRabitqIndexParamBuilder>(
field_schema);
if (!index_param_builder_result.has_value()) {
return tl::make_unexpected(Status::InvalidArgument(
"failed to build index param: " +
index_param_builder_result.error().message()));
}
auto index_param_builder = index_param_builder_result.value();
auto db_index_params = dynamic_cast<const HnswRabitqIndexParams *>(
field_schema.index_params().get());
index_param_builder->WithM(db_index_params->m());
index_param_builder->WithEFConstruction(
db_index_params->ef_construction());
index_param_builder->WithTotalBits(db_index_params->total_bits());
index_param_builder->WithNumClusters(db_index_params->num_clusters());
index_param_builder->WithSampleCount(db_index_params->sample_count());
index_param_builder->WithProvider(
db_index_params->raw_vector_provider());
index_param_builder->WithReformer(db_index_params->rabitq_reformer());
return index_param_builder->Build();
}
case IndexType::IVF: {
auto index_param_builder_result = _build_common_index_param<
IVFIndexParams, core_interface::IVFIndexParamBuilder>(field_schema);
@@ -26,6 +26,7 @@
#include "db/common/typedef.h"
#include "db/index/column/common/index_results.h"
#include "db/index/common/meta.h"
#include "zvec/core/framework/index_provider.h"
#include "vector_column_params.h"
#include "vector_index_results.h"
@@ -88,6 +89,9 @@ class VectorColumnIndexer {
// Result<VectorDataset> BatchFetch(const std::vector<uint32_t> &doc_ids)
// const;
core::IndexProvider::Pointer create_index_provider() const {
return index->create_index_provider();
}
public:
std::string index_file_path() const {
+36
View File
@@ -37,6 +37,32 @@ proto::HnswIndexParams ProtoConverter::ToPb(const HnswIndexParams *params) {
return params_pb;
}
// HnswRabitqIndexParams
HnswRabitqIndexParams::OPtr ProtoConverter::FromPb(
const proto::HnswRabitqIndexParams &params_pb) {
auto params = std::make_shared<HnswRabitqIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.total_bits(), params_pb.num_clusters(), params_pb.m(),
params_pb.ef_construction(), params_pb.sample_count());
return params;
}
proto::HnswRabitqIndexParams ProtoConverter::ToPb(
const HnswRabitqIndexParams *params) {
proto::HnswRabitqIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.set_m(params->m());
params_pb.set_ef_construction(params->ef_construction());
params_pb.set_total_bits(params->total_bits());
params_pb.set_num_clusters(params->num_clusters());
params_pb.set_sample_count(params->sample_count());
return params_pb;
}
// FlatIndexParams
FlatIndexParams::OPtr ProtoConverter::FromPb(
const proto::FlatIndexParams &params_pb) {
@@ -157,6 +183,8 @@ IndexParams::Ptr ProtoConverter::FromPb(const proto::IndexParams &params_pb) {
return ProtoConverter::FromPb(params_pb.ivf());
} else if (params_pb.has_flat()) {
return ProtoConverter::FromPb(params_pb.flat());
} else if (params_pb.has_hnsw_rabitq()) {
return ProtoConverter::FromPb(params_pb.hnsw_rabitq());
}
return nullptr;
@@ -211,6 +239,14 @@ proto::IndexParams ProtoConverter::ToPb(const IndexParams *params) {
}
break;
}
case IndexType::HNSW_RABITQ: {
auto hnsw_rabitq_params =
dynamic_cast<const HnswRabitqIndexParams *>(params);
if (hnsw_rabitq_params) {
params_pb.mutable_hnsw_rabitq()->CopyFrom(
ProtoConverter::ToPb(hnsw_rabitq_params));
}
}
default:
break;
}
+5
View File
@@ -25,6 +25,11 @@ struct ProtoConverter {
static proto::HnswIndexParams ToPb(const HnswIndexParams *params);
// HnswRabitqIndexParams
static HnswRabitqIndexParams::OPtr FromPb(
const proto::HnswRabitqIndexParams &params_pb);
static proto::HnswRabitqIndexParams ToPb(const HnswRabitqIndexParams *params);
// FlatIndexParams
static FlatIndexParams::OPtr FromPb(const proto::FlatIndexParams &params_pb);
static proto::FlatIndexParams ToPb(const FlatIndexParams *params);
+46 -2
View File
@@ -20,6 +20,7 @@
#include <zvec/db/schema.h>
#include <zvec/db/status.h>
#include <zvec/db/type.h>
#include "ailego/internal/cpu_features.h"
#include "db/common/constants.h"
#include "db/common/typedef.h"
#include "db/common/utils.h"
@@ -27,9 +28,16 @@
namespace zvec {
#if defined(RABITQ_COMPILED_AVX512)
constexpr const int kRabitqCompiledAvx512 = RABITQ_COMPILED_AVX512;
#else
constexpr const int kRabitqCompiledAvx512 = 0;
#endif
std::unordered_map<DataType, std::set<QuantizeType>> quantize_type_map = {
{DataType::VECTOR_FP32,
{QuantizeType::FP16, QuantizeType::INT4, QuantizeType::INT8}},
{QuantizeType::FP16, QuantizeType::INT4, QuantizeType::INT8,
QuantizeType::RABITQ}},
// {DataType::VECTOR_FP64, {QuantizeType::FP16}},
{DataType::SPARSE_VECTOR_FP32, {QuantizeType::FP16}},
};
@@ -46,7 +54,7 @@ std::unordered_set<DataType> support_sparse_vector_type = {
};
std::unordered_set<IndexType> support_dense_vector_index = {
IndexType::FLAT, IndexType::HNSW, IndexType::IVF};
IndexType::FLAT, IndexType::HNSW, IndexType::HNSW_RABITQ, IndexType::IVF};
std::unordered_set<IndexType> support_sparse_vector_index = {IndexType::FLAT,
IndexType::HNSW};
@@ -126,6 +134,42 @@ Status FieldSchema::validate() const {
}
}
if (index_params_->type() == IndexType::HNSW_RABITQ) {
if (dimension_ < kMinRabitqDimSize || dimension_ > kMaxRabitqDimSize) {
return Status::InvalidArgument(
"schema validate failed: HNSW_RABITQ index only support "
"dimension in [",
kMinRabitqDimSize, ", ", kMaxRabitqDimSize, "]");
}
if (data_type_ != DataType::VECTOR_FP32) {
return Status::InvalidArgument(
"schema validate failed: HNSW_RABITQ index only support FP32 "
"data types");
}
auto metric_type = vector_index_params->metric_type();
if (metric_type != MetricType::L2 && metric_type != MetricType::IP &&
metric_type != MetricType::COSINE) {
return Status::InvalidArgument(
"schema validate failed: HNSW_RABITQ index only support "
"L2/IP/COSINE metric");
}
#if !RABITQ_SUPPORTED
return Status::NotSupported(
"RabitQ is not supported on this platform (Linux x86_64 only)");
#endif
auto &flags = zvec::ailego::internal::CpuFeatures::static_flags_;
if (!flags.AVX2 && !flags.AVX512F) {
return Status::NotSupported(
"RabitQ requires AVX2/AVX512F to be supported");
}
if (kRabitqCompiledAvx512 && !flags.AVX512F) {
return Status::NotSupported(
"RabitQ compiled with AVX512F while runtime does not support");
}
}
if (vector_index_params->quantize_type() != QuantizeType::UNDEFINED) {
auto iter = quantize_type_map.find(data_type_);
if (iter == quantize_type_map.end()) {
+16 -2
View File
@@ -27,6 +27,8 @@ struct IndexTypeCodeBook {
switch (type) {
case proto::IT_HNSW:
return IndexType::HNSW;
case proto::IT_HNSW_RABITQ:
return IndexType::HNSW_RABITQ;
case proto::IT_FLAT:
return IndexType::FLAT;
case proto::IT_IVF:
@@ -44,6 +46,8 @@ struct IndexTypeCodeBook {
switch (type) {
case IndexType::HNSW:
return proto::IT_HNSW;
case IndexType::HNSW_RABITQ:
return proto::IT_HNSW_RABITQ;
case IndexType::FLAT:
return proto::IT_FLAT;
case IndexType::IVF:
@@ -61,8 +65,12 @@ struct IndexTypeCodeBook {
switch (type) {
case IndexType::HNSW:
return "HNSW";
// case IndexType::SPARSE_HNSW:
// return "SPARSE_HNSW";
case IndexType::HNSW_RABITQ:
return "HNSW_RABITQ";
case IndexType::FLAT:
return "FLAT";
case IndexType::IVF:
return "IVF";
case IndexType::INVERT:
return "INVERT";
default:
@@ -414,6 +422,8 @@ struct QuantizeTypeCodeBook {
return QuantizeType::INT4;
case proto::QuantizeType::QT_INT8:
return QuantizeType::INT8;
case proto::QuantizeType::QT_RABITQ:
return QuantizeType::RABITQ;
default:
return QuantizeType::UNDEFINED;
}
@@ -427,6 +437,8 @@ struct QuantizeTypeCodeBook {
return proto::QuantizeType::QT_INT4;
case QuantizeType::INT8:
return proto::QuantizeType::QT_INT8;
case QuantizeType::RABITQ:
return proto::QuantizeType::QT_RABITQ;
default:
return proto::QuantizeType::QT_UNDEFINED;
}
@@ -440,6 +452,8 @@ struct QuantizeTypeCodeBook {
return "INT4";
case QuantizeType::INT8:
return "INT8";
case QuantizeType::RABITQ:
return "RABITQ";
default:
return "UNDEFINED";
}
+113 -21
View File
@@ -37,11 +37,15 @@
#include <zvec/db/schema.h>
#include <zvec/db/status.h>
#include <zvec/db/type.h>
#if RABITQ_SUPPORTED
#include "core/algorithm/hnsw_rabitq/rabitq_params.h"
#endif
#include "db/common/constants.h"
#include "db/common/file_helper.h"
#include "db/common/global_resource.h"
#include "db/common/typedef.h"
#include "db/index/column/inverted_column/inverted_indexer.h"
#include "db/index/column/vector_column/engine_helper.hpp"
#include "db/index/column/vector_column/vector_column_indexer.h"
#include "db/index/column/vector_column/vector_column_params.h"
#include "db/index/common/index_filter.h"
@@ -53,6 +57,11 @@
#include "db/index/storage/mmap_forward_store.h"
#include "db/index/storage/store_helper.h"
#include "db/index/storage/wal/wal_file.h"
#include "zvec/ailego/container/params.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "column_merging_reader.h"
#include "sql_expr_parser.h"
@@ -1646,6 +1655,8 @@ Status SegmentImpl::create_vector_index(
auto original_index_params =
std::dynamic_pointer_cast<VectorIndexParams>(field->index_params());
core::IndexProvider::Pointer raw_vector_provider;
if (!(vector_index_params->metric_type() ==
original_index_params->metric_type() &&
vector_indexers_[column].size() == 1)) {
@@ -1674,32 +1685,113 @@ Status SegmentImpl::create_vector_index(
block.set_max_doc_id(meta()->max_doc_id());
block.set_doc_count(meta()->doc_count());
new_segment_meta->add_persisted_block(block);
if (vector_index_params->quantize_type() == QuantizeType::RABITQ) {
raw_vector_provider = vector_indexer.value()->create_index_provider();
}
} else {
raw_vector_provider =
vector_indexers_[column][0]->create_index_provider();
}
auto quant_block_id = allocate_block_id();
auto field_with_new_index_params = std::make_shared<FieldSchema>(*field);
field_with_new_index_params->set_index_params(index_params);
if (vector_index_params->quantize_type() != QuantizeType::RABITQ) {
auto quant_block_id = allocate_block_id();
auto field_with_new_index_params = std::make_shared<FieldSchema>(*field);
field_with_new_index_params->set_index_params(index_params);
std::string index_file_path = FileHelper::MakeQuantizeVectorIndexPath(
path_, column, segment_meta_->id(), quant_block_id);
auto vector_indexer = merge_vector_indexer(
index_file_path, column, *field_with_new_index_params, concurrency);
if (!vector_indexer.has_value()) {
return vector_indexer.error();
std::string index_file_path = FileHelper::MakeQuantizeVectorIndexPath(
path_, column, segment_meta_->id(), quant_block_id);
auto vector_indexer = merge_vector_indexer(
index_file_path, column, *field_with_new_index_params, concurrency);
if (!vector_indexer.has_value()) {
return vector_indexer.error();
}
quant_vector_indexers->insert({column, vector_indexer.value()});
new_segment_meta->remove_vector_persisted_block(column, true);
BlockMeta block;
block.set_id(quant_block_id);
block.set_type(BlockType::VECTOR_INDEX_QUANTIZE);
block.set_columns({column});
block.set_min_doc_id(meta()->min_doc_id());
block.set_max_doc_id(meta()->max_doc_id());
block.set_doc_count(meta()->doc_count());
new_segment_meta->add_persisted_block(block);
} else {
#if !RABITQ_SUPPORTED
return Status::NotSupported(
"RabitQ is not supported on this platform (Linux x86_64 only)");
#else
// rabitq
auto rabitq_params = std::dynamic_pointer_cast<HnswRabitqIndexParams>(
vector_index_params->clone());
if (!rabitq_params) {
return Status::InternalError("Expect HnswRabitqIndexParams");
}
// train rabitq converter
auto converter = core::IndexFactory::CreateConverter("RabitqConverter");
if (!converter) {
return Status::NotSupported("RabitqConverter not found");
}
core::IndexMeta index_meta;
index_meta.set_meta(
ProximaEngineHelper::convert_to_engine_data_type(field->data_type())
.value(),
// use field dimension
field->dimension());
index_meta.set_metric(
core_interface::Index::get_metric_name(
ProximaEngineHelper::convert_to_engine_metric_type(
vector_index_params->metric_type())
.value(),
false),
0, ailego::Params{});
ailego::Params converter_params;
converter_params.set(core::PARAM_RABITQ_TOTAL_BITS,
rabitq_params->total_bits());
converter_params.set(core::PARAM_RABITQ_NUM_CLUSTERS,
rabitq_params->num_clusters());
converter_params.set(core::PARAM_RABITQ_SAMPLE_COUNT,
rabitq_params->sample_count());
if (int ret = converter->init(index_meta, converter_params); ret != 0) {
return Status::InternalError("Failed to init rabitq converter:", ret);
}
if (int ret = converter->train(raw_vector_provider); ret != 0) {
return Status::InternalError("Failed to train rabitq converter:", ret);
}
core::IndexReformer::Pointer reformer;
if (int ret = converter->to_reformer(&reformer); ret != 0) {
return Status::InternalError("Failed to to get rabitq reformer:", ret);
}
rabitq_params->set_rabitq_reformer(reformer);
rabitq_params->set_raw_vector_provider(raw_vector_provider);
auto quant_block_id = allocate_block_id();
auto field_with_new_index_params = std::make_shared<FieldSchema>(*field);
field_with_new_index_params->set_index_params(rabitq_params);
std::string index_file_path = FileHelper::MakeQuantizeVectorIndexPath(
path_, column, segment_meta_->id(), quant_block_id);
auto vector_indexer = merge_vector_indexer(
index_file_path, column, *field_with_new_index_params, concurrency);
if (!vector_indexer.has_value()) {
return vector_indexer.error();
}
quant_vector_indexers->insert({column, vector_indexer.value()});
new_segment_meta->remove_vector_persisted_block(column, true);
BlockMeta block;
block.set_id(quant_block_id);
block.set_type(BlockType::VECTOR_INDEX_QUANTIZE);
block.set_columns({column});
block.set_min_doc_id(meta()->min_doc_id());
block.set_max_doc_id(meta()->max_doc_id());
block.set_doc_count(meta()->doc_count());
new_segment_meta->add_persisted_block(block);
#endif
}
quant_vector_indexers->insert({column, vector_indexer.value()});
new_segment_meta->remove_vector_persisted_block(column, true);
BlockMeta block;
block.set_id(quant_block_id);
block.set_type(BlockType::VECTOR_INDEX_QUANTIZE);
block.set_columns({column});
block.set_min_doc_id(meta()->min_doc_id());
block.set_max_doc_id(meta()->max_doc_id());
block.set_doc_count(meta()->doc_count());
new_segment_meta->add_persisted_block(block);
*segment_meta = new_segment_meta;
}
+20 -5
View File
@@ -56,6 +56,8 @@ enum IndexType {
IT_IVF = 2;
// Proxima FLAT Index
IT_FLAT = 3;
// Proxima HNSW RABITQ Index
IT_HNSW_RABITQ = 4;
// Invert Index
IT_INVERT = 10;
};
@@ -65,6 +67,7 @@ enum QuantizeType {
QT_FP16 = 1;
QT_INT8 = 2;
QT_INT4 = 3;
QT_RABITQ = 4;
};
enum MetricType {
@@ -74,9 +77,11 @@ enum MetricType {
MT_COSINE = 3;
};
message InvertIndexParams { bool enable_range_optimization = 1; };
message InvertIndexParams {
bool enable_range_optimization = 1;
};
message BaseIndexParams {
message BaseIndexParams {
MetricType metric_type = 1;
QuantizeType quantize_type = 2;
};
@@ -85,7 +90,16 @@ message HnswIndexParams {
BaseIndexParams base = 1;
int32 m = 2;
int32 ef_construction = 3;
};
}
message HnswRabitqIndexParams {
BaseIndexParams base = 1;
int32 m = 2;
int32 ef_construction = 3;
int32 total_bits = 4;
int32 num_clusters = 5;
int32 sample_count = 6;
}
message FlatIndexParams {
BaseIndexParams base = 1;
@@ -104,6 +118,7 @@ message IndexParams {
HnswIndexParams hnsw = 2;
FlatIndexParams flat = 3;
IVFIndexParams ivf = 4;
HnswRabitqIndexParams hnsw_rabitq = 5;
};
};
@@ -131,11 +146,11 @@ enum BlockType {
message BlockMeta {
uint32 block_id = 1;
BlockType block_type = 2; // for getting filename prefix
BlockType block_type = 2; // for getting filename prefix
uint64 min_doc_id = 3;
uint64 max_doc_id = 4;
uint64 doc_count = 5;
repeated string columns = 6; // columns contained in this block
repeated string columns = 6; // columns contained in this block
};
// message AlterColumnMeta {
@@ -18,6 +18,7 @@
#include <zvec/core/framework/index_holder.h>
#include <zvec/core/framework/index_meta.h>
#include <zvec/core/framework/index_stats.h>
#include "zvec/core/framework/index_reformer.h"
namespace zvec {
namespace core {
@@ -219,6 +220,11 @@ class IndexConverter : public IndexModule {
static int TrainTransformAndDump(const IndexConverter::Pointer &converter,
IndexHolder::Pointer holder,
const IndexDumper::Pointer &dumper);
//! Convert to reformer
virtual int to_reformer(IndexReformer::Pointer *) {
return IndexError_NotImplemented;
}
};
} // namespace core
+24 -6
View File
@@ -398,13 +398,22 @@ class MultiPassNumericalIndexHolder : public IndexHolder {
features_.reserve(size);
}
private:
//! Disable them
MultiPassNumericalIndexHolder(void) = delete;
//! Get vector data pointer by index
const void *get_vector_by_index(size_t index) const {
if (index >= features_.size()) {
return nullptr;
}
return features_[index].second.data();
}
protected:
//! Members
size_t dimension_{0};
std::vector<std::pair<uint64_t, ailego::NumericalVector<T>>> features_;
private:
//! Disable them
MultiPassNumericalIndexHolder(void) = delete;
};
/*! One-Pass Binary Index Holder
@@ -617,13 +626,22 @@ class MultiPassBinaryIndexHolder : public IndexHolder {
features_.reserve(size);
}
private:
//! Disable them
MultiPassBinaryIndexHolder(void) = delete;
//! Get vector data pointer by index
const void *get_vector_by_index(size_t index) const {
if (index >= features_.size()) {
return nullptr;
}
return features_[index].second.data();
}
protected:
//! Members
size_t dimension_{0};
std::vector<std::pair<uint64_t, ailego::BinaryVector<T>>> features_;
private:
//! Disable them
MultiPassBinaryIndexHolder(void) = delete;
};
/*! One-Pass Index Hybrid Holder
@@ -14,6 +14,7 @@
#pragma once
#include <unordered_map>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/framework/index_holder.h>
#include <zvec/core/framework/index_storage.h>
@@ -71,5 +72,399 @@ struct IndexSparseProvider : IndexSparseHolder {
virtual const std::string &owner_class(void) const = 0;
};
/*! Multi-Pass Numerical Index Provider
*/
template <typename T>
class MultiPassNumericalIndexProvider : public IndexProvider {
public:
//! Constructor
explicit MultiPassNumericalIndexProvider(size_t dim)
: holder_(dim), owner_class_("MultiPassNumericalIndexProvider") {}
//! Destructor
virtual ~MultiPassNumericalIndexProvider(void) {}
//! Retrieve count of elements in holder
size_t count(void) const override {
return holder_.count();
}
//! Retrieve dimension
size_t dimension(void) const override {
return holder_.dimension();
}
//! Retrieve element size in bytes
size_t element_size(void) const override {
return holder_.element_size();
}
//! Create a new iterator
IndexHolder::Iterator::Pointer create_iterator(void) override {
return holder_.create_iterator();
}
//! Retrieve a vector using a primary key
const void *get_vector(const uint64_t key) const override {
auto it = indice_map_.find(key);
if (it == indice_map_.end()) {
return nullptr;
}
return holder_.get_vector_by_index(it->second);
}
//! Retrieve a vector using a primary key
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
const void *data = get_vector(key);
if (data == nullptr) {
return IndexError_NoExist;
}
block.reset(const_cast<void *>(data));
return 0;
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_class_;
}
//! Append an element into holder
bool emplace(uint64_t key, const ailego::NumericalVector<T> &vec) {
if (!holder_.emplace(key, vec)) {
return false;
}
indice_map_[key] = static_cast<int>(holder_.count() - 1);
return true;
}
//! Append an element into holder
bool emplace(uint64_t key, ailego::NumericalVector<T> &&vec) {
if (!holder_.emplace(key, std::move(vec))) {
return false;
}
indice_map_[key] = static_cast<int>(holder_.count() - 1);
return true;
}
private:
//! Members
MultiPassNumericalIndexHolder<T> holder_;
std::unordered_map<uint64_t, int> indice_map_;
std::string owner_class_;
};
/*! Multi-Pass Binary Index Provider
*/
template <typename T>
class MultiPassBinaryIndexProvider : public IndexProvider {
public:
//! Constructor
explicit MultiPassBinaryIndexProvider(size_t dim)
: holder_(dim), owner_class_("MultiPassBinaryIndexProvider") {}
//! Destructor
virtual ~MultiPassBinaryIndexProvider(void) {}
//! Retrieve count of elements in holder
size_t count(void) const override {
return holder_.count();
}
//! Retrieve dimension
size_t dimension(void) const override {
return holder_.dimension();
}
//! Retrieve element size in bytes
size_t element_size(void) const override {
return holder_.element_size();
}
//! Create a new iterator
IndexHolder::Iterator::Pointer create_iterator(void) override {
return holder_.create_iterator();
}
//! Retrieve a vector using a primary key
const void *get_vector(const uint64_t key) const override {
auto it = indice_map_.find(key);
if (it == indice_map_.end()) {
return nullptr;
}
return holder_.get_vector_by_index(it->second);
}
//! Retrieve a vector using a primary key
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
const void *data = get_vector(key);
if (data == nullptr) {
return IndexError_NoExist;
}
block.reset(const_cast<void *>(data));
return 0;
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_class_;
}
//! Append an element into holder
bool emplace(uint64_t key, const ailego::BinaryVector<T> &vec) {
if (!holder_.emplace(key, vec)) {
return false;
}
indice_map_[key] = static_cast<int>(holder_.count() - 1);
return true;
}
//! Append an element into holder
bool emplace(uint64_t key, ailego::BinaryVector<T> &&vec) {
if (!holder_.emplace(key, std::move(vec))) {
return false;
}
indice_map_[key] = static_cast<int>(holder_.count() - 1);
return true;
}
private:
//! Members
MultiPassBinaryIndexHolder<T> holder_;
std::unordered_map<uint64_t, int> indice_map_;
std::string owner_class_;
};
/*! Multi-Pass Index Provider
*/
template <IndexMeta::DataType FT>
struct MultiPassIndexProvider;
/*! Multi-Pass Index Provider (BINARY32)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_BINARY32>
: public MultiPassBinaryIndexProvider<uint32_t> {
//! Constructor
using MultiPassBinaryIndexProvider::MultiPassBinaryIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_BINARY32;
}
};
/*! Multi-Pass Index Provider (BINARY64)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_BINARY64>
: public MultiPassBinaryIndexProvider<uint64_t> {
//! Constructor
using MultiPassBinaryIndexProvider::MultiPassBinaryIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_BINARY64;
}
};
/*! Multi-Pass Index Provider (FP16)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_FP16>
: public MultiPassNumericalIndexProvider<ailego::Float16> {
//! Constructor
using MultiPassNumericalIndexProvider::MultiPassNumericalIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_FP16;
}
};
/*! Multi-Pass Index Provider (FP32)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>
: public MultiPassNumericalIndexProvider<float> {
//! Constructor
using MultiPassNumericalIndexProvider::MultiPassNumericalIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_FP32;
}
};
/*! Multi-Pass Index Provider (FP64)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_FP64>
: public MultiPassNumericalIndexProvider<double> {
//! Constructor
using MultiPassNumericalIndexProvider::MultiPassNumericalIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_FP64;
}
};
/*! Multi-Pass Index Provider (INT8)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_INT8>
: public MultiPassNumericalIndexProvider<int8_t> {
//! Constructor
using MultiPassNumericalIndexProvider::MultiPassNumericalIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_INT8;
}
};
/*! Multi-Pass Index Provider (INT16)
*/
template <>
struct MultiPassIndexProvider<IndexMeta::DataType::DT_INT16>
: public MultiPassNumericalIndexProvider<int16_t> {
//! Constructor
using MultiPassNumericalIndexProvider::MultiPassNumericalIndexProvider;
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return IndexMeta::DataType::DT_INT16;
}
};
/*! Convert IndexHolder to IndexProvider
* @param holder The IndexHolder to convert
* @return IndexProvider::Pointer
*/
inline IndexProvider::Pointer convert_holder_to_provider(
const IndexHolder::Pointer &holder) {
if (!holder) {
return nullptr;
}
IndexMeta::DataType data_type = holder->data_type();
size_t dimension = holder->dimension();
switch (data_type) {
case IndexMeta::DataType::DT_FP16: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_FP16>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const ailego::Float16 *data =
static_cast<const ailego::Float16 *>(iter->data());
ailego::NumericalVector<ailego::Float16> vec(dimension);
std::memcpy(vec.data(), data, dimension * sizeof(ailego::Float16));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_FP32: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const float *data = static_cast<const float *>(iter->data());
ailego::NumericalVector<float> vec(dimension);
std::memcpy(vec.data(), data, dimension * sizeof(float));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_FP64: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_FP64>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const double *data = static_cast<const double *>(iter->data());
ailego::NumericalVector<double> vec(dimension);
std::memcpy(vec.data(), data, dimension * sizeof(double));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_INT8: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_INT8>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const int8_t *data = static_cast<const int8_t *>(iter->data());
ailego::NumericalVector<int8_t> vec(dimension);
std::memcpy(vec.data(), data, dimension * sizeof(int8_t));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_INT16: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_INT16>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const int16_t *data = static_cast<const int16_t *>(iter->data());
ailego::NumericalVector<int16_t> vec(dimension);
std::memcpy(vec.data(), data, dimension * sizeof(int16_t));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_BINARY32: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_BINARY32>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const uint32_t *data = static_cast<const uint32_t *>(iter->data());
size_t binary_size = (dimension + 31) / 32;
ailego::BinaryVector<uint32_t> vec(dimension);
std::memcpy(vec.data(), data, binary_size * sizeof(uint32_t));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
case IndexMeta::DataType::DT_BINARY64: {
auto provider = std::make_shared<
MultiPassIndexProvider<IndexMeta::DataType::DT_BINARY64>>(dimension);
auto iter = holder->create_iterator();
while (iter->is_valid()) {
uint64_t key = iter->key();
const uint64_t *data = static_cast<const uint64_t *>(iter->data());
size_t binary_size = (dimension + 63) / 64;
ailego::BinaryVector<uint64_t> vec(dimension);
std::memcpy(vec.data(), data, binary_size * sizeof(uint64_t));
provider->emplace(key, std::move(vec));
iter->next();
}
return provider;
}
default:
return nullptr;
}
}
} // namespace core
} // namespace zvec
@@ -23,5 +23,8 @@ constexpr static uint32_t kDefaultHnswNeighborCnt = 50;
constexpr static uint32_t kDefaultHnswEfSearch = 300;
constexpr const uint32_t kDefaultRabitqTotalBits = 7;
constexpr const uint32_t kDefaultRabitqNumClusters = 16;
} // namespace zvec::core_interface
+24
View File
@@ -31,6 +31,7 @@
#include <zvec/core/framework/index_searcher.h>
#include <zvec/core/framework/index_storage.h>
#include <zvec/core/interface/index_param.h>
#include "zvec/core/framework/index_provider.h"
namespace zvec::core_interface {
@@ -154,6 +155,12 @@ class Index {
return streamer_;
}
core::IndexProvider::Pointer create_index_provider() const {
return streamer_->create_provider();
}
static std::string get_metric_name(MetricType metric_type, bool is_sparse);
protected:
int _sparse_fetch(const uint32_t doc_id,
VectorDataBuffer *vector_data_buffer);
@@ -292,5 +299,22 @@ class HNSWIndex : public Index {
HNSWIndexParam param_{};
};
class HNSWRabitqIndex : public Index {
public:
HNSWRabitqIndex() = default;
protected:
virtual int CreateAndInitStreamer(const BaseIndexParam &param) override;
virtual int _prepare_for_search(
const VectorData &query, const BaseIndexQueryParam::Pointer &search_param,
core::IndexContext::Pointer &context) override;
int _get_coarse_search_topk(
const BaseIndexQueryParam::Pointer &search_param) override;
private:
HNSWRabitqIndexParam param_{};
};
} // namespace zvec::core_interface
+47 -1
View File
@@ -23,6 +23,7 @@
#include <zvec/core/framework/index_filter.h>
#include <zvec/core/framework/index_meta.h>
#include <zvec/core/interface/constants.h>
#include "zvec/core/framework/index_framework.h"
namespace zvec::core_interface {
#define MAX_DIMENSION 65536
@@ -61,6 +62,7 @@ enum class IndexType {
kFlat,
kIVF, // it's actual a two-layer index
kHNSW,
kHNSWRabitq,
};
enum class IVFSearchMethod { kBF, kHNSW };
@@ -80,7 +82,8 @@ enum class QuantizerType {
kAQ,
kFP16,
kInt8,
kInt4
kInt4,
kRabitq,
};
struct SerializableBase {
@@ -186,6 +189,16 @@ struct HNSWQueryParam : public BaseIndexQueryParam {
}
};
struct HNSWRabitqQueryParam : public BaseIndexQueryParam {
using Pointer = std::shared_ptr<HNSWRabitqQueryParam>;
uint32_t ef_search = kDefaultHnswEfSearch;
BaseIndexQueryParam::Pointer Clone() const override {
return std::make_shared<HNSWRabitqQueryParam>(*this);
}
};
struct IVFQueryParam : public BaseIndexQueryParam {
int nprobe = 10;
std::shared_ptr<BaseIndexQueryParam> l1QueryParam = nullptr;
@@ -316,4 +329,37 @@ struct HNSWIndexParam : public BaseIndexParam {
bool omit_empty_value = false) const override;
};
struct HNSWRabitqIndexParam : public BaseIndexParam {
using Pointer = std::shared_ptr<HNSWRabitqIndexParam>;
// HNSW parameters
int m = kDefaultHnswNeighborCnt;
int ef_construction = kDefaultHnswEfConstruction;
// Rabitq parameters
int total_bits = kDefaultRabitqTotalBits;
int num_clusters = kDefaultRabitqNumClusters;
int sample_count = 0;
core::IndexProvider::Pointer provider = nullptr;
core::IndexReformer::Pointer reformer = nullptr;
// Constructors with delegation
HNSWRabitqIndexParam() : BaseIndexParam(IndexType::kHNSWRabitq) {}
HNSWRabitqIndexParam(int m, int ef_construction)
: BaseIndexParam(IndexType::kHNSWRabitq),
m(m),
ef_construction(ef_construction) {}
HNSWRabitqIndexParam(MetricType metric, int dim, int m, int ef_construction)
: BaseIndexParam(IndexType::kHNSWRabitq, metric, dim),
m(m),
ef_construction(ef_construction) {}
protected:
bool DeserializeFromJsonObject(const ailego::JsonObject &json_obj) override;
ailego::JsonObject SerializeToJsonObject(
bool omit_empty_value = false) const override;
};
} // namespace zvec::core_interface
@@ -16,6 +16,9 @@
#include <memory>
#include <zvec/core/interface/index_param.h>
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/interface/index.h"
namespace zvec::core_interface {
@@ -148,6 +151,46 @@ class HNSWIndexParamBuilder
}
};
class HNSWRabitqIndexParamBuilder
: public BaseIndexParamBuilder<HNSWRabitqIndexParamBuilder,
HNSWRabitqIndexParam> {
public:
HNSWRabitqIndexParamBuilder() = default;
HNSWRabitqIndexParamBuilder &WithM(int m) {
param->m = m;
return *this;
}
HNSWRabitqIndexParamBuilder &WithEFConstruction(int ef_construction) {
param->ef_construction = ef_construction;
return *this;
}
HNSWRabitqIndexParamBuilder &WithTotalBits(int total_bits) {
param->total_bits = total_bits;
return *this;
}
HNSWRabitqIndexParamBuilder &WithNumClusters(int num_clusters) {
param->num_clusters = num_clusters;
return *this;
}
HNSWRabitqIndexParamBuilder &WithSampleCount(int sample_count) {
param->sample_count = sample_count;
return *this;
}
HNSWRabitqIndexParamBuilder &WithReformer(
core::IndexReformer::Pointer reformer) {
param->reformer = std::move(reformer);
return *this;
}
HNSWRabitqIndexParamBuilder &WithProvider(
core::IndexProvider::Pointer provider) {
param->provider = std::move(provider);
return *this;
}
std::shared_ptr<HNSWRabitqIndexParam> Build() override {
return param;
}
};
// class CompositeIndexParamBuilder : public
// BaseIndexParamBuilder<CompositeIndexParamBuilder, CompositeIndexParam>
// { public:
@@ -291,6 +334,21 @@ class IVFQueryParamBuilder
}
};
// HNSW-Rabitq builder (adds ef_search field)
class HNSWRabitqQueryParamBuilder
: public BaseIndexQueryParamBuilder<HNSWRabitqQueryParam,
HNSWRabitqQueryParamBuilder> {
public:
HNSWRabitqQueryParamBuilder &with_ef_search(int ef_search) {
m_param.ef_search = ef_search;
return *this;
}
HNSWRabitqQueryParam::Pointer build() {
return std::make_shared<HNSWRabitqQueryParam>(std::move(m_param));
}
};
// Example Usage:
// // First, build the required nested params
// auto nested_hnsw = HNSWQueryParamBuilder().with_ef_search(64).build();
+116 -2
View File
@@ -18,6 +18,8 @@
#include <string>
#include <zvec/core/interface/constants.h>
#include <zvec/db/type.h>
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
namespace zvec {
@@ -44,7 +46,7 @@ class IndexParams {
bool is_vector_index_type() const {
return type_ == IndexType::FLAT || type_ == IndexType::HNSW ||
type_ == IndexType::IVF;
type_ == IndexType::HNSW_RABITQ || type_ == IndexType::IVF;
}
IndexType type() const {
@@ -200,11 +202,123 @@ class HnswIndexParams : public VectorIndexParams {
return ef_construction_;
}
private:
protected:
int m_;
int ef_construction_;
};
class HnswRabitqIndexParams : public VectorIndexParams {
public:
HnswRabitqIndexParams(
MetricType metric_type,
int total_bits = core_interface::kDefaultRabitqTotalBits,
int num_clusters = core_interface::kDefaultRabitqNumClusters,
int m = core_interface::kDefaultHnswNeighborCnt,
int ef_construction = core_interface::kDefaultHnswEfConstruction,
int sample_count = 0)
: VectorIndexParams(IndexType::HNSW_RABITQ, metric_type,
QuantizeType::RABITQ),
total_bits_(total_bits),
num_clusters_(num_clusters),
sample_count_(sample_count),
m_(m),
ef_construction_(ef_construction) {}
using OPtr = std::shared_ptr<HnswRabitqIndexParams>;
Ptr clone() const override {
auto obj = std::make_shared<HnswRabitqIndexParams>(
metric_type_, total_bits_, num_clusters_, m_, ef_construction_,
sample_count_);
obj->set_rabitq_reformer(rabitq_reformer_);
obj->set_raw_vector_provider(raw_vector_provider_);
return obj;
}
std::string to_string() const override {
auto base_str = vector_index_params_to_string("HnswRabitqIndexParams",
metric_type_, quantize_type_);
std::ostringstream oss;
oss << base_str << ",total_bits:" << total_bits_
<< ",num_clusters:" << num_clusters_
<< ",sample_count:" << sample_count_ << ",m:" << m_
<< ",ef_construction:" << ef_construction_ << "}";
return oss.str();
}
bool operator==(const IndexParams &other) const override {
if (type() != other.type()) {
return false;
}
auto &other_rabitq = dynamic_cast<const HnswRabitqIndexParams &>(other);
return metric_type() == other_rabitq.metric_type() &&
quantize_type_ == other_rabitq.quantize_type_ &&
total_bits_ == other_rabitq.total_bits_ &&
num_clusters_ == other_rabitq.num_clusters_ &&
sample_count_ == other_rabitq.sample_count_ &&
m_ == other_rabitq.m_ &&
ef_construction_ == other_rabitq.ef_construction_;
}
void set_m(int m) {
m_ = m;
}
int m() const {
return m_;
}
void set_ef_construction(int ef_construction) {
ef_construction_ = ef_construction;
}
int ef_construction() const {
return ef_construction_;
}
void set_raw_vector_provider(
core::IndexProvider::Pointer raw_vector_provider) {
raw_vector_provider_ = std::move(raw_vector_provider);
}
void set_rabitq_reformer(core::IndexReformer::Pointer rabitq_reformer) {
rabitq_reformer_ = std::move(rabitq_reformer);
}
core::IndexReformer::Pointer rabitq_reformer() const {
return rabitq_reformer_;
}
core::IndexProvider::Pointer raw_vector_provider() const {
return raw_vector_provider_;
}
void set_total_bits(int total_bits) {
total_bits_ = total_bits;
}
int total_bits() const {
return total_bits_;
}
void set_num_clusters(int num_clusters) {
num_clusters_ = num_clusters;
}
int num_clusters() const {
return num_clusters_;
}
void set_sample_count(int sample_count) {
sample_count_ = sample_count;
}
int sample_count() const {
return sample_count_;
}
private:
int total_bits_;
int num_clusters_;
int sample_count_;
int m_;
int ef_construction_;
core::IndexProvider::Pointer raw_vector_provider_;
core::IndexReformer::Pointer rabitq_reformer_;
};
class FlatIndexParams : public VectorIndexParams {
public:
FlatIndexParams(MetricType metric_type,
+25
View File
@@ -125,6 +125,31 @@ class IVFQueryParams : public QueryParams {
float scale_factor_{10};
};
class HnswRabitqQueryParams : public QueryParams {
public:
HnswRabitqQueryParams(int ef = core_interface::kDefaultHnswEfSearch,
float radius = 0.0f, bool is_linear = false,
bool is_using_refiner = false)
: QueryParams(IndexType::HNSW_RABITQ), ef_(ef) {
set_radius(radius);
set_is_linear(is_linear);
set_is_using_refiner(is_using_refiner);
}
virtual ~HnswRabitqQueryParams() = default;
int ef() const {
return ef_;
}
void set_ef(int ef) {
ef_ = ef;
}
private:
int ef_;
};
class FlatQueryParams : public QueryParams {
public:
FlatQueryParams(bool is_using_refiner = false, float scale_factor = 10)
+4 -2
View File
@@ -23,8 +23,9 @@ namespace zvec {
enum class IndexType : uint32_t {
UNDEFINED = 0,
HNSW = 1,
IVF = 3,
FLAT = 4,
IVF = 2,
FLAT = 3,
HNSW_RABITQ = 4,
INVERT = 10,
};
@@ -72,6 +73,7 @@ enum class QuantizeType : uint32_t {
FP16 = 1,
INT8 = 2,
INT4 = 3,
RABITQ = 4,
};
enum class MetricType : uint32_t {
+3
View File
@@ -7,3 +7,6 @@ cc_directories(flat_sparse)
cc_directories(ivf)
cc_directories(hnsw)
cc_directories(hnsw_sparse)
if(RABITQ_SUPPORTED)
cc_directories(hnsw_rabitq)
endif()
@@ -0,0 +1,34 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/option.cmake)
if(APPLE)
set(APPLE_FRAMEWORK_LIBS
-framework CoreFoundation
-framework CoreGraphics
-framework CoreData
-framework CoreText
-framework Security
-framework Foundation
-Wl,-U,_MallocExtension_ReleaseFreeMemory
-Wl,-U,_ProfilerStart
-Wl,-U,_ProfilerStop
-Wl,-U,_RegisterThriftProtocol
)
endif()
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw_rabitq core_knn_flat core_knn_cluster
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm/hnsw_rabitq
LDFLAGS ${APPLE_FRAMEWORK_LIBS}
)
cc_test_suite(hnsw_rabitq ${CC_TARGET})
endforeach()
@@ -0,0 +1,428 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_builder.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <future>
#include <gtest/gtest.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/container/vector.h>
#include "zvec/core/framework/index_framework.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_provider.h"
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
using namespace std;
using namespace zvec::ailego;
namespace zvec {
namespace core {
constexpr size_t static dim = 128;
class HnswRabitqBuilderTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string _dir;
static shared_ptr<IndexMeta> _index_meta_ptr;
};
std::string HnswRabitqBuilderTest::_dir("hnswRabitqBuilderTest");
shared_ptr<IndexMeta> HnswRabitqBuilderTest::_index_meta_ptr;
void HnswRabitqBuilderTest::SetUp(void) {
IndexLoggerBroker::SetLevel(0);
_index_meta_ptr.reset(new (nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
_index_meta_ptr->set_metric("SquaredEuclidean", 0, ailego::Params());
}
void HnswRabitqBuilderTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
// system(cmdBuf);
}
TEST_F(HnswRabitqBuilderTest, TestGeneral) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestGeneral";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswRabitqBuilderTest, TestLoad) {
// Load index with searcher and verify search
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
string path = _dir + "/TestGeneral";
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
// Perform search verification
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(j) / 1000.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto context = searcher->create_context();
ASSERT_NE(context, nullptr);
context->set_topk(10);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &result = context->result(0);
ASSERT_GT(result.size(), 0UL);
ASSERT_LE(result.size(), 10UL);
}
TEST_F(HnswRabitqBuilderTest, TestDimensions) {
std::vector<size_t> dimensions = {1, 2, 4, 8, 16, 32, 33,
63, 64, 128, 256, 512, 1024, 2047,
2048, 2049, 4095, 4096, 4097, 8192, 16384};
size_t doc_cnt = 100;
for (size_t test_dim : dimensions) {
std::cout << "Testing dimension: " << test_dim << std::endl;
IndexMeta index_meta(IndexMeta::DataType::DT_FP32, test_dim);
index_meta.set_metric("SquaredEuclidean", 0, ailego::Params());
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr) << "dim=" << test_dim;
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", test_dim);
int ret = builder->init(index_meta, params);
// dimension <= 63 or >= 4096: init() should return -31
if (test_dim <= 63 || test_dim >= 4096) {
ASSERT_EQ(-31, ret) << "expected init to fail with -31, dim=" << test_dim;
std::cout << "Dimension " << test_dim
<< " correctly rejected with ret=" << ret << std::endl;
continue;
}
// Valid dimensions: verify full build succeeds
ASSERT_EQ(0, ret) << "init failed, dim=" << test_dim;
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(
test_dim);
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(test_dim);
for (size_t j = 0; j < test_dim; ++j) {
vec[j] = static_cast<float>(i * test_dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec))) << "dim=" << test_dim;
}
ret = builder->train(holder);
ASSERT_EQ(0, ret) << "train failed, dim=" << test_dim;
ret = builder->build(holder);
ASSERT_EQ(0, ret) << "build failed, dim=" << test_dim;
auto &stats = builder->stats();
ASSERT_EQ(doc_cnt, stats.built_count()) << "dim=" << test_dim;
std::cout << "Dimension " << test_dim << " passed, built "
<< stats.built_count() << " docs" << std::endl;
}
}
TEST_F(HnswRabitqBuilderTest, TestMemquota) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
params.set("proxima.hnsw_rabitq.builder.memory_quota", 100000UL);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(IndexError_NoMemory, builder->build(holder));
}
TEST_F(HnswRabitqBuilderTest, TestIndexThreads) {
IndexBuilder::Pointer builder1 =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder1, nullptr);
IndexBuilder::Pointer builder2 =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder2, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
std::srand(ailego::Realtime::MilliSeconds());
auto threads =
std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
auto build_index1 = [&]() {
ASSERT_EQ(0, builder1->train(threads, holder));
ASSERT_EQ(0, builder1->build(threads, holder));
};
auto build_index2 = [&]() {
ASSERT_EQ(0, builder2->train(threads, holder));
ASSERT_EQ(0, builder2->build(threads, holder));
};
auto t1 = std::async(std::launch::async, build_index1);
auto t2 = std::async(std::launch::async, build_index2);
t1.wait();
t2.wait();
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndexThreads";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder1->dump(dumper));
ASSERT_EQ(0, dumper->close());
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder2->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats1 = builder1->stats();
ASSERT_EQ(doc_cnt, stats1.built_count());
auto &stats2 = builder2->stats();
ASSERT_EQ(doc_cnt, stats2.built_count());
}
TEST_F(HnswRabitqBuilderTest, TestCosine) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineFp32Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
converted_holder = convert_holder_to_provider(converted_holder);
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCosine";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswRabitqBuilderTest, TestCleanupAndRebuild) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCleanupAndRebuild";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// Cleanup and rebuild with more documents
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder2->emplace(i, std::move(vec)));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -0,0 +1,573 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_searcher.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <cstdio>
#include <random>
#include <gtest/gtest.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/container/vector.h>
#include "zvec/core/framework/index_framework.h"
#include "zvec/core/framework/index_logger.h"
#include "hnsw_rabitq_builder.h"
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
using namespace std;
using namespace zvec::ailego;
namespace zvec {
namespace core {
constexpr size_t static dim = 128;
class HnswRabitqSearcherTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string _dir;
static shared_ptr<IndexMeta> _index_meta_ptr;
};
std::string HnswRabitqSearcherTest::_dir("HnswRabitqSearcherTest");
shared_ptr<IndexMeta> HnswRabitqSearcherTest::_index_meta_ptr;
void HnswRabitqSearcherTest::SetUp(void) {
IndexLoggerBroker::SetLevel(0);
_index_meta_ptr.reset(new (nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
_index_meta_ptr->set_metric("SquaredEuclidean", 0, ailego::Params());
}
void HnswRabitqSearcherTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
// system(cmdBuf);
}
TEST_F(HnswRabitqSearcherTest, TestBasicSearch) {
// Build index first
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestBasicSearch";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
// Perform search
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(j) / 1000.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto context = searcher->create_context();
ASSERT_TRUE(!!context);
context->set_topk(10);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &result = context->result(0);
ASSERT_GT(result.size(), 0UL);
ASSERT_LE(result.size(), 10UL);
// Verify results are sorted by distance
for (size_t i = 1; i < result.size(); ++i) {
ASSERT_LE(result[i - 1].score(), result[i].score());
}
}
TEST_F(HnswRabitqSearcherTest, DISABLED_TestRnnSearch) {
// Build index first
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestRnnSearch";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher with radius search
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = 0.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto context = searcher->create_context();
ASSERT_NE(context, nullptr);
size_t topk = 50;
context->set_topk(topk);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &results = context->result(0);
ASSERT_EQ(topk, results.size());
// Test with radius threshold
float radius = results[topk / 2].score();
context->set_threshold(radius);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
ASSERT_GT(topk, results.size());
for (size_t k = 0; k < results.size(); ++k) {
ASSERT_GE(radius, results[k].score());
}
// Test reset threshold
context->reset_threshold();
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
ASSERT_EQ(topk, results.size());
ASSERT_LT(radius, results[topk - 1].score());
}
TEST_F(HnswRabitqSearcherTest, DISABLED_TestSearchInnerProduct) {
// Build index with InnerProduct metric
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
IndexMeta index_meta(IndexMeta::DataType::DT_FP32, dim);
index_meta.set_metric("InnerProduct", 0, ailego::Params());
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestSearchInnerProduct";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = 1.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto context = searcher->create_context();
ASSERT_TRUE(!!context);
size_t topk = 50;
context->set_topk(topk);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &results = context->result(0);
ASSERT_EQ(topk, results.size());
// Test with radius threshold (note: InnerProduct uses negative scores)
float radius = -results[topk / 2].score();
context->set_threshold(radius);
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
ASSERT_GT(topk, results.size());
for (size_t k = 0; k < results.size(); ++k) {
LOG_ERROR("radius: %f, score: %f", radius, results[k].score());
EXPECT_GE(radius, results[k].score());
}
// Test reset threshold
context->reset_threshold();
ASSERT_EQ(0, searcher->search_impl(query_vec.data(), query_meta, 1, context));
ASSERT_EQ(topk, results.size());
ASSERT_LT(-radius, results[topk - 1].score());
}
TEST_F(HnswRabitqSearcherTest, TestSearchCosine) {
// Build index with Cosine metric
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-1.0, 1.0);
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = dist(gen);
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineFp32Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
converted_holder = convert_holder_to_provider(converted_holder);
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestSearchCosine";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = 1.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
ASSERT_TRUE(reformer != nullptr);
ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
std::string new_query;
IndexQueryMeta new_meta;
ASSERT_EQ(0, reformer->transform(query_vec.data(), query_meta, &new_query,
&new_meta));
auto context = searcher->create_context();
ASSERT_TRUE(!!context);
size_t topk = 50;
context->set_topk(topk);
ASSERT_EQ(0, searcher->search_impl(new_query.data(), new_meta, 1, context));
const auto &results = context->result(0);
ASSERT_EQ(topk, results.size());
// Test with radius threshold
float radius = 0.5f;
context->set_threshold(radius);
ASSERT_EQ(0, searcher->search_impl(new_query.data(), new_meta, 1, context));
ASSERT_GT(topk, results.size());
for (size_t k = 0; k < results.size(); ++k) {
ASSERT_GE(radius, results[k].score());
}
// Test reset threshold
context->reset_threshold();
ASSERT_EQ(0, searcher->search_impl(new_query.data(), new_meta, 1, context));
ASSERT_EQ(topk, results.size());
ASSERT_LT(radius, results[topk - 1].score());
}
TEST_F(HnswRabitqSearcherTest, TestMultipleQueries) {
// Build index first
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestMultipleQueries";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher with multiple queries
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
// Test with different query vectors
for (size_t query_id = 0; query_id < 5; ++query_id) {
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(query_id * dim + j) / 1000.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
auto context = searcher->create_context();
ASSERT_TRUE(!!context);
context->set_topk(20);
ASSERT_EQ(0,
searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &result = context->result(0);
ASSERT_GT(result.size(), 0UL);
ASSERT_LE(result.size(), 20UL);
// Verify results are sorted
for (size_t i = 1; i < result.size(); ++i) {
ASSERT_LE(result[i - 1].score(), result[i].score());
}
}
}
TEST_F(HnswRabitqSearcherTest, TestDifferentTopK) {
// Build index first
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswRabitqBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 10000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec)));
}
ailego::Params params;
params.set("proxima.rabitq.num_clusters", 16UL);
params.set("proxima.rabitq.total_bits", 2UL);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestDifferentTopK";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
// Test searcher with different topk values
auto searcher = IndexFactory::CreateSearcher("HnswRabitqSearcher");
ASSERT_NE(searcher, nullptr);
ailego::Params search_params;
search_params.set("proxima.hnsw_rabitq.searcher.ef", 100UL);
ASSERT_EQ(0, searcher->init(search_params));
auto loader = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_NE(loader, nullptr);
ASSERT_EQ(0, loader->init(ailego::Params()));
ASSERT_EQ(0, loader->open(path, false));
ASSERT_EQ(0, searcher->load(loader, nullptr));
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(j) / 1000.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
// Test with different topk values
std::vector<size_t> topk_values = {1, 5, 10, 20, 50, 100};
for (size_t topk : topk_values) {
auto context = searcher->create_context();
ASSERT_TRUE(!!context);
context->set_topk(topk);
ASSERT_EQ(0,
searcher->search_impl(query_vec.data(), query_meta, 1, context));
const auto &result = context->result(0);
ASSERT_GT(result.size(), 0UL);
ASSERT_LE(result.size(), topk);
// Verify results are sorted
for (size_t i = 1; i < result.size(); ++i) {
ASSERT_LE(result[i - 1].score(), result[i].score());
}
}
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -0,0 +1,618 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_streamer.h"
#include <memory>
#include <gtest/gtest.h>
#include "zvec/ailego/container/params.h"
#include "zvec/core/framework/index_holder.h"
#include "zvec/core/framework/index_streamer.h"
#include "hnsw_rabitq_streamer.h"
#include "rabitq_converter.h"
#include "rabitq_reformer.h"
using namespace std;
using namespace zvec::ailego;
namespace zvec {
namespace core {
constexpr size_t static dim = 128;
class HnswRabitqStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string dir_;
static shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string HnswRabitqStreamerTest::dir_("hnswRabitqStreamerTest");
shared_ptr<IndexMeta> HnswRabitqStreamerTest::index_meta_ptr_;
void HnswRabitqStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params());
}
void HnswRabitqStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(HnswRabitqStreamerTest, TestBuildAndSearch) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
ASSERT_EQ(0,
streamer->add_impl(it->key(), it->data(), query_meta, context));
}
streamer->flush(0UL);
// Perform search verification
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(j) / 1000.0f;
}
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
context->set_topk(10);
ASSERT_EQ(0, streamer->search_impl(query_vec.data(), query_meta, 1, context));
const auto &result = context->result(0);
ASSERT_GT(result.size(), 0UL);
ASSERT_LE(result.size(), 10UL);
// reopen and load reformer from storage
ASSERT_EQ(0, streamer->close());
IndexStreamer::Pointer new_streamer =
std::make_shared<HnswRabitqStreamer>(holder);
ASSERT_EQ(0, new_streamer->init(*index_meta_ptr_, params));
ASSERT_EQ(0, new_streamer->open(storage));
}
TEST_F(HnswRabitqStreamerTest, TestLinearSearch) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch", true));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
ASSERT_EQ(0,
streamer->add_impl(it->key(), it->data(), query_meta, context));
}
// Test linear search with exact match
size_t topk = 3;
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
NumericalVector<float> query_vec(dim);
for (size_t i = 0; i < doc_cnt; i += 100) {
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(i);
}
context->set_topk(1U);
ASSERT_EQ(0,
streamer->search_bf_impl(query_vec.data(), query_meta, context));
auto &result1 = context->result();
ASSERT_EQ(1UL, result1.size());
ASSERT_EQ(i, result1[0].key());
// Test with slight offset
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(i) + 0.1f;
}
context->set_topk(topk);
ASSERT_EQ(0,
streamer->search_bf_impl(query_vec.data(), query_meta, context));
auto &result2 = context->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
}
}
TEST_F(HnswRabitqStreamerTest, TestKnnSearch) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 2000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 10U);
params.set("proxima.hnsw_rabitq.streamer.efconstruction", 100U);
params.set("proxima.hnsw_rabitq.streamer.ef", 50U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearch", true));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
ASSERT_EQ(0,
streamer->add_impl(it->key(), it->data(), query_meta, context));
}
// Compare KNN search with brute force search
auto linear_ctx = streamer->create_context();
auto knn_ctx = streamer->create_context();
size_t topk = 50;
linear_ctx->set_topk(topk);
knn_ctx->set_topk(topk);
int total_hits = 0;
int total_cnts = 0;
int topk1_hits = 0;
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
NumericalVector<float> query_vec(dim);
for (size_t i = 0; i < doc_cnt; i += 100) {
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(i) + 0.1f;
}
ASSERT_EQ(0,
streamer->search_impl(query_vec.data(), query_meta, 1, knn_ctx));
ASSERT_EQ(
0, streamer->search_bf_impl(query_vec.data(), query_meta, linear_ctx));
auto &knn_result = knn_ctx->result(0);
ASSERT_EQ(topk, knn_result.size());
topk1_hits += (i == knn_result[0].key());
auto &linear_result = linear_ctx->result();
ASSERT_EQ(topk, linear_result.size());
ASSERT_EQ(i, linear_result[0].key());
for (size_t k = 0; k < topk; ++k) {
total_cnts++;
for (size_t j = 0; j < topk; ++j) {
if (linear_result[j].key() == knn_result[k].key()) {
total_hits++;
break;
}
}
}
}
float recall = total_hits * 1.0f / total_cnts;
float topk1_recall = topk1_hits * 100.0f / static_cast<float>(doc_cnt);
EXPECT_GT(recall, 0.60f);
// actual: no guarantee
// TODO(jiliang.ljl): check if ok?
EXPECT_GT(topk1_recall, 0.00f);
}
TEST_F(HnswRabitqStreamerTest, TestRandomData) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1500UL;
// Add random vectors
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 32U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 20U);
params.set("proxima.hnsw_rabitq.streamer.efconstruction", 200U);
params.set("proxima.hnsw_rabitq.streamer.ef", 100U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestRandomData", true));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
ASSERT_EQ(0,
streamer->add_impl(it->key(), it->data(), query_meta, context));
}
// Test with random queries
auto linear_ctx = streamer->create_context();
auto knn_ctx = streamer->create_context();
size_t topk = 50;
linear_ctx->set_topk(topk);
knn_ctx->set_topk(topk);
int total_hits = 0;
int total_cnts = 0;
int topk1_hits = 0;
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
NumericalVector<float> query_vec(dim);
size_t query_cnt = 200;
for (size_t i = 0; i < query_cnt; i++) {
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
}
ASSERT_EQ(
0, streamer->search_bf_impl(query_vec.data(), query_meta, linear_ctx));
ASSERT_EQ(0,
streamer->search_impl(query_vec.data(), query_meta, 1, knn_ctx));
auto &knn_result = knn_ctx->result(0);
ASSERT_EQ(topk, knn_result.size());
auto &linear_result = linear_ctx->result();
ASSERT_EQ(topk, linear_result.size());
topk1_hits += (linear_result[0].key() == knn_result[0].key());
for (size_t k = 0; k < topk; ++k) {
total_cnts++;
for (size_t j = 0; j < topk; ++j) {
if (linear_result[j].key() == knn_result[k].key()) {
total_hits++;
break;
}
}
}
}
float recall = total_hits * 1.0f / total_cnts;
float topk1_recall = topk1_hits * 1.0f / query_cnt;
EXPECT_GT(recall, 0.50f);
EXPECT_GT(topk1_recall, 0.70f);
}
TEST_F(HnswRabitqStreamerTest, TestOpenClose) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 500UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestOpenClose", true));
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
// Add first half of vectors
for (size_t i = 0; i < doc_cnt / 2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_EQ(0, streamer->add_impl(i, vec.data(), query_meta, context));
}
ASSERT_EQ(0, streamer->flush(0UL));
ASSERT_EQ(0, streamer->close());
// Reopen and add second half
IndexStreamer::Pointer streamer2 =
std::make_shared<HnswRabitqStreamer>(holder);
ASSERT_EQ(0, streamer2->init(*index_meta_ptr_, params));
ASSERT_EQ(0, streamer2->open(storage));
auto context2 = streamer2->create_context();
for (size_t i = doc_cnt / 2; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_EQ(0, streamer2->add_impl(i, vec.data(), query_meta, context2));
}
ASSERT_EQ(0, streamer2->flush(0UL));
// Verify search works after reopen
NumericalVector<float> query_vec(dim);
for (size_t j = 0; j < dim; ++j) {
query_vec[j] = 10.0f;
}
context2->set_topk(5);
ASSERT_EQ(0,
streamer2->search_impl(query_vec.data(), query_meta, 1, context2));
const auto &result = context2->result(0);
ASSERT_EQ(5UL, result.size());
ASSERT_EQ(10UL, result[0].key());
}
TEST_F(HnswRabitqStreamerTest, TestCreateIterator) {
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 300UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
RabitqConverter converter;
converter.init(*index_meta_ptr_, ailego::Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder, reformer);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
params.set("proxima.hnsw_rabitq.general.dimension", dim);
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true));
ASSERT_EQ(0, streamer->open(storage));
auto context = streamer->create_context();
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_EQ(0, streamer->add_impl(i, vec.data(), query_meta, context));
}
streamer->flush(0UL);
// Test iterator
auto provider = streamer->create_provider();
auto iter = provider->create_iterator();
ASSERT_TRUE(!!iter);
size_t count = 0;
while (iter->is_valid()) {
ASSERT_EQ(count, iter->key());
// const float *data = (const float *)iter->data();
// for (size_t j = 0; j < dim; ++j) {
// ASSERT_EQ(static_cast<float>(count), data[j]);
// }
iter->next();
count++;
}
ASSERT_EQ(doc_cnt, count);
// Test get_vector
// for (size_t i = 0; i < doc_cnt; i++) {
// const float *data = (const float *)provider->get_vector(i);
// ASSERT_NE(data, nullptr);
// for (size_t j = 0; j < dim; ++j) {
// ASSERT_EQ(static_cast<float>(i), data[j]);
// }
// }
}
TEST_F(HnswRabitqStreamerTest, TestDimensions) {
std::vector<size_t> dimensions = {1, 2, 4, 8, 16, 32, 33,
63, 64, 128, 256, 512, 1024, 2047,
2048, 2049, 4095, 4096, 4097, 8192, 16384};
size_t doc_cnt = 100;
for (size_t test_dim : dimensions) {
std::cout << "Testing dimension: " << test_dim << std::endl;
IndexMeta index_meta(IndexMeta::DataType::DT_FP32, test_dim);
index_meta.set_metric("SquaredEuclidean", 0, ailego::Params());
auto holder =
make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(
test_dim);
IndexStreamer::Pointer streamer =
std::make_shared<HnswRabitqStreamer>(holder);
ailego::Params params;
params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
params.set("proxima.hnsw_rabitq.general.dimension", test_dim);
int ret = streamer->init(index_meta, params);
// dimension <= 63 or >= 4096: init() should return -31
if (test_dim <= 63 || test_dim >= 4096) {
ASSERT_EQ(-31, ret) << "expected init to fail with -31, dim=" << test_dim;
std::cout << "Dimension " << test_dim
<< " correctly rejected with ret=" << ret << std::endl;
continue;
}
// Valid dimensions: verify full streaming build succeeds
ASSERT_EQ(0, ret) << "init failed, dim=" << test_dim;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(test_dim);
for (size_t j = 0; j < test_dim; ++j) {
vec[j] = static_cast<float>(i * test_dim + j) / 1000.0f;
}
ASSERT_TRUE(holder->emplace(i, std::move(vec))) << "dim=" << test_dim;
}
RabitqConverter converter;
converter.init(index_meta, ailego::Params());
ASSERT_EQ(0, converter.train(holder))
<< "converter train failed, dim=" << test_dim;
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(0, converter.to_reformer(&index_reformer)) << "dim=" << test_dim;
auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
// Recreate streamer with reformer
streamer = std::make_shared<HnswRabitqStreamer>(holder, reformer);
ASSERT_EQ(0, streamer->init(index_meta, params))
<< "init with reformer failed, dim=" << test_dim;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ailego::Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
std::string storage_path =
dir_ + "/TestDimensions_" + std::to_string(test_dim);
ASSERT_EQ(0, storage->open(storage_path, true))
<< "storage open failed, dim=" << test_dim;
ASSERT_EQ(0, streamer->open(storage))
<< "streamer open failed, dim=" << test_dim;
auto context = streamer->create_context();
IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, test_dim);
for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
ASSERT_EQ(0,
streamer->add_impl(it->key(), it->data(), query_meta, context))
<< "add failed, dim=" << test_dim << ", key=" << it->key();
}
ASSERT_EQ(0, streamer->flush(0UL)) << "flush failed, dim=" << test_dim;
std::cout << "Dimension " << test_dim << " passed" << std::endl;
}
}
} // namespace core
} // namespace zvec
+1 -1
View File
@@ -8,7 +8,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_metric core_interface core_knn_flat core_utility core_quantizer sparsehash core_knn_hnsw core_mix_reducer
core_knn_flat_sparse core_knn_hnsw_sparse core_knn_ivf
core_knn_flat_sparse core_knn_hnsw_sparse core_knn_ivf core_knn_hnsw_rabitq
SRCS ${CC_SRCS}
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
)
@@ -17,6 +17,10 @@
#include <iostream>
#include <unordered_map>
#include <gtest/gtest.h>
#if RABITQ_SUPPORTED
#include "core/algorithm/hnsw_rabitq/rabitq_converter.h"
#include "zvec/core/framework/index_provider.h"
#endif
#include "zvec/ailego/buffer/buffer_manager.h"
#include "zvec/core/interface/index.h"
#include "zvec/core/interface/index_factory.h"
@@ -1345,6 +1349,116 @@ TEST(IndexInterface, Score) {
MetricType::kInnerProduct);
}
#if RABITQ_SUPPORTED
TEST(IndexInterface, HNSWRabitqGeneral) {
constexpr uint32_t kDimension = 64;
const std::string index_name{"test_rabitq.index"};
char cmd_buf[256];
snprintf(cmd_buf, sizeof(cmd_buf), "rm -f %s*", index_name.c_str());
auto func = [&](const BaseIndexParam::Pointer &param,
const BaseIndexQueryParam::Pointer &query_param) {
system(cmd_buf);
auto index = IndexFactory::CreateAndInitIndex(*param);
ASSERT_NE(nullptr, index);
index->Open(index_name, {StorageOptions::StorageType::kMMAP, true});
std::vector<float> vector(kDimension);
vector[1] = 1.0f;
vector[2] = 2.0f;
VectorData vector_data;
vector_data.vector = DenseVector{vector.data()};
ASSERT_TRUE(0 == index->Add(vector_data, 233));
ASSERT_TRUE(0 == index->Train());
SearchResult result;
VectorData query;
query.vector = DenseVector{vector.data()};
index->Search(query, query_param, &result);
ASSERT_EQ(1, result.doc_list_.size());
ASSERT_EQ(233, result.doc_list_[0].key());
// Fetch is meaningless for HNSWRabitq
index->Close();
system(cmd_buf);
};
using namespace zvec::core;
using namespace zvec::ailego;
auto holder = std::make_shared<
zvec::core::MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(
kDimension);
size_t doc_cnt = 500UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(kDimension);
for (size_t j = 0; j < kDimension; ++j) {
vec[j] = static_cast<float>(i);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
std::shared_ptr<IndexMeta> index_meta_ptr_;
index_meta_ptr_.reset(
new (std::nothrow) IndexMeta(IndexMeta::DataType::DT_FP32, kDimension));
index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
RabitqConverter converter;
converter.init(*index_meta_ptr_, Params());
ASSERT_EQ(converter.train(holder), 0);
std::shared_ptr<IndexReformer> index_reformer;
ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
// HNSWRabitq with default total_bits
func(HNSWRabitqIndexParamBuilder()
.WithMetricType(MetricType::kL2sq)
.WithDataType(DataType::DT_FP32)
.WithDimension(kDimension)
.WithIsSparse(false)
.WithEFConstruction(100)
.WithProvider(holder)
.WithReformer(index_reformer)
.Build(),
HNSWRabitqQueryParamBuilder()
.with_topk(10)
.with_fetch_vector(false)
.with_ef_search(50)
.build());
// HNSWRabitq with InnerProduct metric
func(HNSWRabitqIndexParamBuilder()
.WithMetricType(MetricType::kInnerProduct)
.WithDataType(DataType::DT_FP32)
.WithDimension(kDimension)
.WithIsSparse(false)
.WithEFConstruction(100)
.WithProvider(holder)
.WithReformer(index_reformer)
.Build(),
HNSWRabitqQueryParamBuilder()
.with_topk(10)
.with_fetch_vector(false)
.with_ef_search(50)
.build());
// HNSWRabitq with custom total_bits
func(HNSWRabitqIndexParamBuilder()
.WithMetricType(MetricType::kL2sq)
.WithDataType(DataType::DT_FP32)
.WithDimension(kDimension)
.WithIsSparse(false)
.WithEFConstruction(100)
.WithTotalBits(2)
.WithProvider(holder)
.WithReformer(index_reformer)
.Build(),
HNSWRabitqQueryParamBuilder()
.with_topk(10)
.with_fetch_vector(false)
.with_ef_search(50)
.build());
}
#endif
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
+1
View File
@@ -31,6 +31,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
core_knn_flat
core_knn_flat_sparse
core_knn_hnsw
core_knn_hnsw_rabitq
core_knn_hnsw_sparse
core_knn_ivf
core_mix_reducer
+78
View File
@@ -4104,6 +4104,84 @@ TEST_F(CollectionTest, Feature_Column_MixOperation_Empty) {
}
}
#if RABITQ_SUPPORTED
TEST_F(CollectionTest, Feature_Optimize_HNSW_RABITQ) {
auto func = [](MetricType metric_type, int concurrency) {
FileHelper::RemoveDirectory(col_path);
int doc_count = 1000;
// create simple schema with only FP32 dense vector for HNSW_RABITQ
auto schema = std::make_shared<CollectionSchema>("demo");
schema->set_max_doc_count_per_segment(MAX_DOC_COUNT_PER_SEGMENT);
auto hnsw_rabitq_params = std::make_shared<HnswRabitqIndexParams>(
metric_type, 7, 256, 16, 200, 0);
schema->add_field(std::make_shared<FieldSchema>(
"dense_fp32", DataType::VECTOR_FP32, 128, false, hnsw_rabitq_params));
auto options = CollectionOptions{false, true, 64 * 1024 * 1024};
auto collection = TestHelper::CreateCollectionWithDoc(
col_path, *schema, options, 0, doc_count, false);
auto check_doc = [&]() {
for (int i = 0; i < doc_count; i++) {
auto expect_doc = TestHelper::CreateDoc(i, *schema);
auto result = collection->Fetch({expect_doc.pk()});
ASSERT_TRUE(result.has_value());
ASSERT_EQ(result.value().size(), 1);
ASSERT_EQ(result.value().count(expect_doc.pk()), 1);
auto doc = result.value()[expect_doc.pk()];
ASSERT_NE(doc, nullptr);
if (*doc != expect_doc) {
std::cout << " doc:" << doc->to_detail_string() << std::endl;
std::cout << "expect_doc:" << expect_doc.to_detail_string()
<< std::endl;
}
ASSERT_EQ(*doc, expect_doc);
}
};
check_doc();
std::cout << "check success 1" << std::endl;
ASSERT_TRUE(collection->Flush().ok());
auto stats = collection->Stats().value();
ASSERT_EQ(stats.doc_count, doc_count);
ASSERT_EQ(stats.index_completeness["dense_fp32"], 0);
auto s = collection->Optimize(OptimizeOptions{concurrency});
if (!s.ok()) {
std::cout << s.message() << std::endl;
}
ASSERT_TRUE(s.ok());
stats = collection->Stats().value();
ASSERT_EQ(stats.doc_count, doc_count);
ASSERT_EQ(stats.index_completeness["dense_fp32"], 1);
check_doc();
std::cout << "check success 2" << std::endl;
collection.reset();
auto result = Collection::Open(col_path, options);
ASSERT_TRUE(result.has_value());
collection = std::move(result.value());
check_doc();
std::cout << "check success 3" << std::endl;
};
func(MetricType::L2, 0);
func(MetricType::L2, 4);
func(MetricType::IP, 0);
func(MetricType::IP, 4);
// TODO: cosine dense not match, may be accuracy issue
// func(MetricType::COSINE, 0);
// func(MetricType::COSINE, 4);
}
#endif
// **** CORNER CASES **** //
TEST_F(CollectionTest, CornerCase_CreateAndOpen) {
// Collection::CreateAndOpen
+2
View File
@@ -27,6 +27,7 @@ cc_binary(
core_knn_hnsw
core_knn_hnsw_sparse
core_knn_ivf
core_knn_hnsw_rabitq
core_mix_reducer
core_metric
core_utility
@@ -52,6 +53,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
core_knn_hnsw
core_knn_hnsw_sparse
core_knn_ivf
core_knn_hnsw_rabitq
core_mix_reducer
core_metric
core_utility
+1 -1
View File
@@ -29,7 +29,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
core_quantizer_static
core_knn_hnsw core_knn_hnsw_sparse sparsehash
core_knn_flat core_knn_flat_sparse core_knn_ivf
core_mix_reducer
core_knn_hnsw_rabitq core_mix_reducer
Arrow::arrow_dataset
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
+186 -10
View File
@@ -248,15 +248,6 @@ TEST(FieldSchemaTest, ComparisonOperators) {
// Different name
EXPECT_FALSE(field1 == field5);
EXPECT_TRUE(field1 != field5);
// Compare with nullptr index params
FieldSchema field6("no_index", DataType::STRING);
FieldSchema field7("no_index", DataType::STRING);
FieldSchema field8("no_index", DataType::STRING, false, 0, index_params1);
EXPECT_TRUE(field6 == field7);
EXPECT_FALSE(field6 == field8);
EXPECT_TRUE(field6 != field8);
}
TEST(FieldSchemaTest, Validate) {
@@ -823,4 +814,189 @@ TEST(CollectionSchemaTest, Validate) {
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
}
}
#if RABITQ_SUPPORTED
TEST(FieldSchemaTest, HnswRabitqIndexValidationMetricTypes) {
// Test supported combinations: FP32 + (L2/IP/COSINE)
// FP32 + L2
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 128, false,
index_params);
auto status = field.validate();
EXPECT_TRUE(status.ok())
<< "FP32 + L2 should be supported, but got error: " << status.message();
}
// FP32 + IP
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::IP, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 128, false,
index_params);
auto status = field.validate();
EXPECT_TRUE(status.ok())
<< "FP32 + IP should be supported, but got error: " << status.message();
}
// FP32 + COSINE
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::COSINE, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 128, false,
index_params);
auto status = field.validate();
EXPECT_TRUE(status.ok())
<< "FP32 + COSINE should be supported, but got error: "
<< status.message();
}
// FP32 + MIPSL2
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::MIPSL2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 128, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "FP32 + MIPSL2 should not be supported, but got error: "
<< status.message();
}
}
TEST(FieldSchemaTest, HnswRabitqIndexValidation_Dimension) {
// Dimension less than 64 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 63, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "Dimension 63 should not be supported with HNSW_RABITQ";
EXPECT_NE(
status.message().find("HNSW_RABITQ index only support dimension in"),
std::string::npos)
<< "Error message should mention dimension range, got: "
<< status.message();
}
// Dimension equal to 1 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 1, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "Dimension 1 should not be supported with HNSW_RABITQ";
}
// Dimension greater than 4095 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 4096, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "Dimension 4096 should not be supported with HNSW_RABITQ";
EXPECT_NE(
status.message().find("HNSW_RABITQ index only support dimension in"),
std::string::npos)
<< "Error message should mention dimension range, got: "
<< status.message();
}
// Boundary: dimension 64 should be supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 64, false,
index_params);
auto status = field.validate();
EXPECT_TRUE(status.ok())
<< "Dimension 64 should be supported, but got error: "
<< status.message();
}
// Boundary: dimension 4095 should be supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP32, 4095, false,
index_params);
auto status = field.validate();
EXPECT_TRUE(status.ok())
<< "Dimension 4095 should be supported, but got error: "
<< status.message();
}
}
#endif
TEST(FieldSchemaTest, HnswRabitqIndexValidation_UnsupportedDataTypes) {
// Test unsupported data types with HNSW_RABITQ index
// FP16 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP16, 128, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "FP16 should not be supported with HNSW_RABITQ";
EXPECT_NE(
status.message().find("HNSW_RABITQ index only support FP32 data type"),
std::string::npos)
<< "Error message should mention FP32 support only, got: "
<< status.message();
}
// INT8 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_INT8, 128, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "INT8 should not be supported with HNSW_RABITQ";
EXPECT_NE(
status.message().find("HNSW_RABITQ index only support FP32 data type"),
std::string::npos)
<< "Error message should mention FP32 support only, got: "
<< status.message();
}
// FP64 is not supported
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::L2, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::VECTOR_FP64, 128, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "FP64 should not be supported with HNSW_RABITQ";
}
// Sparse vector is not supported with HNSW_RABITQ
{
auto index_params = std::make_shared<HnswRabitqIndexParams>(
MetricType::IP, 7, 256, 16, 200, 0);
FieldSchema field("vector_field", DataType::SPARSE_VECTOR_FP32, 128, false,
index_params);
auto status = field.validate();
EXPECT_FALSE(status.ok())
<< "Sparse vector should not be supported with HNSW_RABITQ";
EXPECT_NE(
status.message().find("sparse_vector's index_params only support"),
std::string::npos)
<< "Error message should mention sparse vector index support, got: "
<< status.message();
}
}
+1 -1
View File
@@ -31,7 +31,7 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
core_quantizer
core_knn_hnsw core_knn_hnsw_sparse sparsehash
core_knn_flat core_knn_flat_sparse core_knn_ivf
core_mix_reducer
core_knn_hnsw_rabitq core_mix_reducer
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
SRCS ${CC_SRCS}
+1
View File
@@ -24,4 +24,5 @@ add_subdirectory(rocksdb rocksdb EXCLUDE_FROM_ALL)
add_subdirectory(CRoaring CRoaring EXCLUDE_FROM_ALL)
add_subdirectory(arrow arrow EXCLUDE_FROM_ALL)
add_subdirectory(magic_enum magic_enum EXCLUDE_FROM_ALL)
add_subdirectory(RaBitQ-Library RaBitQ-Library EXCLUDE_FROM_ALL)
+4
View File
@@ -0,0 +1,4 @@
add_library(rabitqlib INTERFACE)
target_include_directories(
rabitqlib INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/RaBitQ-Library-0.1/include"
)
+7 -7
View File
@@ -6,7 +6,7 @@ cc_binary(
STRICT PACKED
SRCS txt2vecs.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags core_framework zvec_ailego
LIBS gflags core_framework zvec_ailego
)
cc_binary(
@@ -14,7 +14,7 @@ cc_binary(
STRICT PACKED
SRCS local_builder.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf core_interface
)
cc_binary(
@@ -22,7 +22,7 @@ cc_binary(
STRICT PACKED
SRCS recall.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf roaring core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf roaring core_interface
)
cc_binary(
@@ -30,7 +30,7 @@ cc_binary(
STRICT PACKED
SRCS bench.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf roaring core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf roaring core_interface
)
@@ -39,7 +39,7 @@ cc_binary(
STRICT PACKED
SRCS recall_original.cc flow.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf roaring core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf roaring core_interface
)
cc_binary(
@@ -47,7 +47,7 @@ cc_binary(
STRICT PACKED
SRCS bench_original.cc flow.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf roaring core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf roaring core_interface
)
cc_binary(
@@ -55,5 +55,5 @@ cc_binary(
STRICT PACKED
SRCS local_builder_original.cc
INCS ${PROJECT_ROOT_DIR}/src/core/
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_cluster core_knn_ivf core_interface
LIBS gflags yaml-cpp magic_enum core_framework core_metric core_quantizer core_utility core_knn_flat core_knn_flat_sparse core_knn_hnsw core_knn_hnsw_sparse core_knn_hnsw_rabitq core_knn_cluster core_knn_ivf core_interface
)
+12
View File
@@ -108,6 +108,18 @@ class IndexMetaHelper {
std::cerr << "Not supported type(" << type << ") for IP" << std::endl;
return false;
}
} else if (method == std::string("Cosine")) {
if (feature_type == IndexMeta::DataType::DT_FP32) {
meta.set_metric("Cosine", 0, std::move(params));
} else if (feature_type == IndexMeta::DataType::DT_INT8) {
meta.set_metric("Cosine", 0, std::move(params));
} else if (feature_type == IndexMeta::DataType::DT_FP16) {
meta.set_metric("Cosine", 0, std::move(params));
} else {
std::cerr << "Not supported type(" << type << ") for Cosine"
<< std::endl;
return false;
}
} else if (method == std::string("HAMMING")) {
if (feature_type == IndexMeta::DataType::DT_BINARY32) {
meta.set_metric("Hamming", 0, std::move(params));
+73 -2
View File
@@ -15,16 +15,23 @@
#include <libgen.h>
#include <signal.h>
#include <iostream>
#include <memory>
#include <ailego/pattern/defer.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/utility/time_helper.h>
#include "algorithm/flat/flat_utility.h"
#include "algorithm/hnsw_rabitq/hnsw_rabitq_params.h"
#if RABITQ_SUPPORTED
#include "algorithm/hnsw_rabitq/hnsw_rabitq_streamer.h"
#include "algorithm/hnsw_rabitq/rabitq_converter.h"
#endif
#include "algorithm/hnsw/hnsw_params.h"
#include "zvec/ailego/logger/logger.h"
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_plugin.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_streamer.h"
#include "index_meta_helper.h"
@@ -113,6 +120,57 @@ bool prepare_params(YAML::Node &&config_params, ailego::Params &params) {
return true;
}
int setup_hnsw_rabitq_streamer(const IndexStreamer::Pointer &streamer,
const IndexMeta &meta, YAML::Node &config_root,
const std::string &converter_name,
IndexHolder::Pointer *build_holder) {
#if RABITQ_SUPPORTED
RabitqConverter rabitq_converter;
ailego::Params rabitq_converter_params;
if (config_root["RabitqConverterParams"]) {
auto rabitq_params_node = config_root["RabitqConverterParams"];
if (!prepare_params(std::move(rabitq_params_node),
rabitq_converter_params)) {
cerr << "Failed to prepare rabitq converter params" << endl;
return -1;
}
}
if (rabitq_converter.init(meta, rabitq_converter_params) != 0) {
cerr << "rabitq converter init failed" << std::endl;
return -1;
}
if (rabitq_converter.train(*build_holder) != 0) {
cerr << "rabitq converter train failed" << std::endl;
return -1;
}
IndexReformer::Pointer rabitq_reformer;
rabitq_converter.to_reformer(&rabitq_reformer);
HnswRabitqStreamer *hnsw_rabitq_streamer =
dynamic_cast<HnswRabitqStreamer *>(streamer.get());
hnsw_rabitq_streamer->set_reformer(std::move(rabitq_reformer));
IndexProvider::Pointer provider;
if (converter_name.empty()) {
// build_holder is VecsIndexHolder
provider = std::dynamic_pointer_cast<IndexProvider>(*build_holder);
} else {
// build_holder is ordinary IndexHolder, need to convert
provider = convert_holder_to_provider(*build_holder);
// reuse provider to release memory
*build_holder = provider;
}
if (!provider) {
cerr << "Failed to cast build holder to provider" << endl;
return -1;
}
hnsw_rabitq_streamer->set_provider(provider);
return 0;
#else
cerr << "HNSW RaBitQ is not supported on this platform" << endl;
return -1;
#endif
}
bool check_config(YAML::Node &config_root) {
auto common = config_root["BuilderCommon"];
if (!common) {
@@ -465,7 +523,8 @@ int do_build_by_streamer(IndexStreamer::Pointer &streamer,
uint64_t key = holder->get_key(id);
if (retrieval_mode == RM_DENSE) {
if (reformer) {
ret = reformer->convert(holder->get_vector(id), qmeta, &ovec, &ometa);
ret = reformer->convert(holder->get_vector_by_index(id), qmeta, &ovec,
&ometa);
if (ret != 0) {
LOG_ERROR("Failed to convert vector for %s", IndexError::What(ret));
errcode = ret;
@@ -473,7 +532,8 @@ int do_build_by_streamer(IndexStreamer::Pointer &streamer,
}
ret = add_to_streamer(key, ovec.data(), ometa, ctx);
} else {
ret = add_to_streamer(key, holder->get_vector(id), qmeta, ctx);
ret =
add_to_streamer(key, holder->get_vector_by_index(id), qmeta, ctx);
}
} else {
LOG_ERROR("Retrieval mode not supported");
@@ -874,6 +934,7 @@ int do_build(YAML::Node &config_root, YAML::Node &config_common) {
build_holder->set_metric(metric_name, metric_params);
meta.set_metric(metric_name, 0, metric_params);
}
IndexMeta input_meta = meta;
string converter_name;
ailego::Params converter_params;
if (config_common["ConverterName"] &&
@@ -939,6 +1000,7 @@ int do_build(YAML::Node &config_root, YAML::Node &config_common) {
std::vector<std::string> id_map_param_list = {
PARAM_HNSW_STREAMER_USE_ID_MAP,
PARAM_FLAT_USE_ID_MAP,
PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP,
};
for (auto &param : id_map_param_list) {
params.set(param, !g_disable_id_map);
@@ -1089,6 +1151,15 @@ int do_build(YAML::Node &config_root, YAML::Node &config_common) {
cout << "Skip train procedure" << endl;
}
if (builder_class == "HnswRabitqStreamer") {
if (setup_hnsw_rabitq_streamer(streamer, input_meta, config_root,
converter_name, &cv_build_holder) != 0) {
return -1;
}
} else if (builder_class == "HnswRabitqBuilder" && !converter_name.empty()) {
cv_build_holder = convert_holder_to_provider(cv_build_holder);
}
// BUILD
holder = build_holder;
signal(SIGINT, stop);
+75 -2
View File
@@ -15,13 +15,20 @@
#include <libgen.h>
#include <signal.h>
#include <iostream>
#include <memory>
#include <ailego/pattern/defer.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/utility/time_helper.h>
#if RABITQ_SUPPORTED
#include "algorithm/hnsw_rabitq/hnsw_rabitq_streamer.h"
#include "algorithm/hnsw_rabitq/rabitq_converter.h"
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#endif
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_plugin.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_streamer.h"
#include "index_meta_helper.h"
@@ -108,6 +115,60 @@ bool prepare_params(YAML::Node &&config_params, ailego::Params &params) {
return true;
}
int setup_hnsw_rabitq_streamer(const IndexStreamer::Pointer &streamer,
const IndexMeta &meta, YAML::Node &config_root,
const std::string &converter_name,
IndexHolder::Pointer *build_holder) {
#if RABITQ_SUPPORTED
RabitqConverter rabitq_converter;
ailego::Params rabitq_converter_params;
if (config_root["RabitqConverterParams"] &&
!prepare_params(std::move(config_root["RabitqConverterParams"]),
rabitq_converter_params)) {
cerr << "Failed to prepare rabitq converter params" << endl;
return -1;
}
if (rabitq_converter.init(meta, rabitq_converter_params) != 0) {
cerr << "rabitq converter init failed" << std::endl;
return -1;
}
if (rabitq_converter.train(*build_holder) != 0) {
cerr << "rabitq converter train failed" << std::endl;
return -1;
}
IndexReformer::Pointer rabitq_reformer;
rabitq_converter.to_reformer(&rabitq_reformer);
HnswRabitqStreamer *hnsw_rabitq_streamer =
dynamic_cast<HnswRabitqStreamer *>(streamer.get());
hnsw_rabitq_streamer->set_reformer(std::move(rabitq_reformer));
IndexProvider::Pointer provider;
if (converter_name.empty()) {
// build_holder is VecsIndexHolder
provider = std::dynamic_pointer_cast<IndexProvider>(*build_holder);
} else {
// build_holder is ordinary IndexHolder, need to convert
provider = convert_holder_to_provider(*build_holder);
// reuse provider to release memory
*build_holder = provider;
}
if (!provider) {
cerr << "Failed to cast build holder to provider" << endl;
return -1;
}
hnsw_rabitq_streamer->set_provider(provider);
return 0;
#else
(void)streamer;
(void)meta;
(void)config_root;
(void)converter_name;
(void)build_holder;
cerr << "HNSW RaBitQ is not supported on this platform" << endl;
return -1;
#endif
}
bool check_config(YAML::Node &config_root) {
auto common = config_root["BuilderCommon"];
if (!common) {
@@ -421,7 +482,8 @@ int do_build_by_streamer(IndexStreamer::Pointer &streamer,
uint64_t key = holder->get_key(id);
if (retrieval_mode == RM_DENSE) {
if (reformer) {
ret = reformer->convert(holder->get_vector(id), qmeta, &ovec, &ometa);
ret = reformer->convert(holder->get_vector_by_index(id), qmeta, &ovec,
&ometa);
if (ret != 0) {
LOG_ERROR("Failed to convert vector for %s", IndexError::What(ret));
errcode = ret;
@@ -429,7 +491,8 @@ int do_build_by_streamer(IndexStreamer::Pointer &streamer,
}
ret = streamer->add_impl(key, ovec.data(), ometa, ctx);
} else {
ret = streamer->add_impl(key, holder->get_vector(id), qmeta, ctx);
ret = streamer->add_impl(key, holder->get_vector_by_index(id), qmeta,
ctx);
}
} else {
cerr << "Retrieval mode not supported";
@@ -828,6 +891,7 @@ int do_build(YAML::Node &config_root, YAML::Node &config_common) {
build_holder->set_metric(metric_name, metric_params);
meta.set_metric(metric_name, 0, metric_params);
}
IndexMeta input_meta = meta;
string converter_name;
ailego::Params converter_params;
if (config_common["ConverterName"] &&
@@ -1036,6 +1100,15 @@ int do_build(YAML::Node &config_root, YAML::Node &config_common) {
cout << "Skip train procedure" << endl;
}
if (builder_class == "HnswRabitqStreamer") {
if (setup_hnsw_rabitq_streamer(streamer, input_meta, config_root,
converter_name, &cv_build_holder) != 0) {
return -1;
}
} else if (builder_class == "HnswRabitqBuilder" && !converter_name.empty()) {
cv_build_holder = convert_holder_to_provider(cv_build_holder);
}
// BUILD
holder = build_holder;
signal(SIGINT, stop);
+54 -9
View File
@@ -15,8 +15,12 @@
#pragma once
#include <string>
#include <unordered_map>
#include <zvec/ailego/container/params.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_holder.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_storage.h"
#include "vecs_reader.h"
namespace zvec {
@@ -30,12 +34,16 @@ namespace core {
* data = iter->data();
* }
*/
class VecsIndexHolder : public IndexHybridHolder {
class VecsIndexHolder : public IndexProvider {
public:
typedef std::shared_ptr<VecsIndexHolder> Pointer;
bool load(const std::string &file_path) {
return vecs_reader_.load(file_path);
if (!vecs_reader_.load(file_path)) {
return false;
}
build_key_index_map();
return true;
}
const IndexMeta &index_meta(void) const {
@@ -110,8 +118,7 @@ class VecsIndexHolder : public IndexHybridHolder {
return iter;
}
virtual IndexHybridHolder::Iterator::Pointer create_hybrid_iterator(
void) override {
virtual IndexHybridHolder::Iterator::Pointer create_hybrid_iterator(void) {
// make sure iter has value whenn create_iterator finished
IndexHybridHolder::Iterator::Pointer iter(
new VecsIndexHolder::Iterator(*this, start_cursor_));
@@ -157,10 +164,6 @@ class VecsIndexHolder : public IndexHybridHolder {
return vecs_reader_.get_key(idx);
}
const void *get_vector(size_t idx) const {
return vecs_reader_.get_vector(idx);
}
uint32_t get_sparse_count(size_t idx) const {
return vecs_reader_.get_sparse_count(idx);
}
@@ -185,7 +188,7 @@ class VecsIndexHolder : public IndexHybridHolder {
return start_cursor_;
}
size_t total_sparse_count(void) const override {
size_t total_sparse_count(void) const {
return vecs_reader_.get_total_sparse_count();
}
@@ -209,11 +212,53 @@ class VecsIndexHolder : public IndexHybridHolder {
return vecs_reader_.key_base();
}
const void *get_vector_by_index(size_t idx) const {
return vecs_reader_.get_vector(idx);
}
public: // IndexProvider interface implementation
//! Retrieve a vector using a primary key
const void *get_vector(const uint64_t key) const override {
auto it = key_to_index_map_.find(key);
if (it == key_to_index_map_.end()) {
return nullptr;
}
return vecs_reader_.get_vector(it->second);
}
//! Retrieve a vector using a primary key
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
const void *vector = get_vector(key);
if (vector == nullptr) {
return IndexError_NoExist;
}
block.reset((void *)vector);
return 0;
}
//! Retrieve the owner class
virtual const std::string &owner_class(void) const override {
static std::string owner_class_name = "VecsIndexHolder";
return owner_class_name;
}
private:
//! Build key to index mapping
void build_key_index_map() {
key_to_index_map_.clear();
size_t num_vecs = vecs_reader_.num_vecs();
for (size_t i = 0; i < num_vecs; ++i) {
uint64_t key = vecs_reader_.get_key(i);
key_to_index_map_[key] = i;
}
}
bool stop_{false};
uint32_t start_cursor_{0};
VecsReader vecs_reader_;
size_t max_doc_count_{0};
std::unordered_map<uint64_t, size_t> key_to_index_map_;
};