refactor(diskann): decouple from libaio via dlopen (#532)

Co-authored-by: Zefeng Yin <yinzefeng.yzf@alibaba-inc.com>

python/tests/detail/fixture_helper.py

@@ -13,19 +13,14 @@ from typing import Any, Generator
 from zvec.typing import DataType, StatusCode, MetricType, QuantizeType
 import zvec
 
-
-# Cache the DiskAnn plugin preload status so we pay the load cost once per
-# test session. The plugin normally auto-loads on first DiskAnn use, but we
-# preload it explicitly here so a missing libaio / misplaced plugin .so
-# surfaces as a clear pytest skip instead of a confusing
-# "Create vector column indexer failed" deep inside the collection code path.
 _DISKANN_PRELOAD_REASON: str | None = None
 _DISKANN_PRELOAD_DONE: bool = False
 
 
 def _ensure_diskann_runtime_or_reason() -> str | None:
-    """Preload the DiskAnn plugin and return None on success or a human-readable
-    skip reason on failure. Idempotent across calls."""
+    """Check whether DiskAnn is available on this platform and return None
+    on success or a human-readable skip reason on failure. Idempotent across
+    calls."""
     global _DISKANN_PRELOAD_DONE, _DISKANN_PRELOAD_REASON
     if _DISKANN_PRELOAD_DONE:
         return _DISKANN_PRELOAD_REASON
@@ -35,22 +30,6 @@ def _ensure_diskann_runtime_or_reason() -> str | None:
         _DISKANN_PRELOAD_REASON = "DiskAnn only supported on Linux x86_64"
         return _DISKANN_PRELOAD_REASON
 
-    if not zvec.is_libaio_available():
-        _DISKANN_PRELOAD_REASON = (
-            "libaio is not available on this host; DiskAnn cannot run. "
-            "Install libaio1 (or libaio1t64 on Ubuntu 24.04+) and retry."
-        )
-        return _DISKANN_PRELOAD_REASON
-
-    status = zvec.load_diskann_plugin()
-    if status != zvec.DISKANN_PLUGIN_OK:
-        _DISKANN_PRELOAD_REASON = (
-            f"Failed to load DiskAnn plugin (status={status}); "
-            "check that libzvec_diskann_plugin.so is installed alongside "
-            "_zvec.so in the Python site-packages directory."
-        )
-        return _DISKANN_PRELOAD_REASON
-
     _DISKANN_PRELOAD_REASON = None
     return None
 
@@ -148,8 +127,7 @@ def full_schema_new(request) -> CollectionSchema:
     else:
         nullable, has_index, vector_index = True, False, HnswIndexParam()
 
-    # Skip DiskAnn tests on unsupported platforms or when the runtime cannot
-    # be brought up (missing libaio, plugin .so not installed, etc.).
+    # Skip DiskAnn tests on unsupported platforms.
     from zvec.model.param import DiskAnnIndexParam
 
     if isinstance(vector_index, DiskAnnIndexParam):

python/tests/test_collection_diskann.py

@@ -13,17 +13,15 @@
 # limitations under the License.
 """End-to-end collection tests for the DiskAnn index.
 
-Mirrors ``test_collection_hnsw_rabitq.py`` but targets the DiskAnn plugin.
+Mirrors ``test_collection_hnsw_rabitq.py`` but targets the DiskAnn index.
 
 Two platform-level prerequisites are enforced at module import time:
 
 1. DiskAnn is currently built only for Linux x86_64 — other platforms are
    skipped wholesale.
-2. The DiskAnn backend lives in a *runtime-loaded* plugin
-   (``libzvec_diskann_plugin.so``). It must be loaded with ``RTLD_GLOBAL |
-   RTLD_NOW`` BEFORE ``import zvec`` so that the plugin's ``IndexFactory``
-   singleton is unified with the one inside ``_zvec.so``. After ``import
-   zvec`` we must also call ``zvec.load_diskann_plugin()`` exactly once.
+2. libaio is loaded eagerly (via dlopen) inside DiskAnnBuilder::init() /
+   DiskAnnStreamer::init(). If libaio is missing, DiskAnn falls back to
+   synchronous pread() — the tests still run but with degraded performance.
 
 If either prerequisite fails the whole module is skipped so the rest of the
 test-suite is not affected.
@@ -32,7 +30,6 @@ test-suite is not affected.
 from __future__ import annotations
 
 import math
-import os
 import platform
 import sys
 
@@ -46,13 +43,6 @@ pytestmark = pytest.mark.skipif(
     reason="DiskAnn plugin is only supported on Linux x86_64",
 )
 
-# Promote all symbols in subsequently-loaded DSOs to the global namespace and
-# resolve relocations eagerly. This is REQUIRED so the DiskAnn plugin can see
-# the ``IndexFactory`` singleton that lives in ``_zvec.so`` and vice versa.
-# See: DiskAnn RTLD_GLOBAL + RTLD_NOW Requirement.
-if sys.platform == "linux":
-    sys.setdlopenflags(sys.getdlopenflags() | os.RTLD_GLOBAL | os.RTLD_NOW)
-
 import zvec  # noqa: E402
 
 from zvec import (  # noqa: E402

python/tests/test_typing.py

@@ -17,6 +17,7 @@ import pytest
 from zvec import (
     DataType,
     IndexType,
+    IOBackendType,
     MetricType,
     QuantizeType,
     Status,
@@ -32,6 +33,7 @@ from zvec import (
     [
         (DataType.FLOAT, "FLOAT"),
         (IndexType.HNSW, "HNSW"),
+        (IOBackendType.PREAD, "PREAD"),
         (MetricType.COSINE, "COSINE"),
         (QuantizeType.INT8, "INT8"),
         (StatusCode.OK, "OK"),
@@ -46,6 +48,7 @@ def test_enum_names(member, name):
     [
         (DataType.FLOAT, 8),
         (IndexType.HNSW, 1),
+        (IOBackendType.PREAD, 0),
         (MetricType.COSINE, 3),
         (QuantizeType.INT8, 2),
         (StatusCode.OK, 0),
@@ -91,11 +94,29 @@ def test_data_type_has_member(member):
     assert member in DataType.__members__
 
 
-@pytest.mark.parametrize("member", ["HNSW", "IVF", "FLAT", "INVERT"])
+@pytest.mark.parametrize(
+    "member",
+    [
+        "UNDEFINED",
+        "HNSW",
+        "IVF",
+        "FLAT",
+        "HNSW_RABITQ",
+        "DISKANN",
+        "VAMANA",
+        "INVERT",
+        "FTS",
+    ],
+)
 def test_index_type_has_member(member):
     assert member in IndexType.__members__
 
 
+@pytest.mark.parametrize("member", ["PREAD", "LIBAIO"])
+def test_io_backend_type_has_member(member):
+    assert member in IOBackendType.__members__
+
+
 @pytest.mark.parametrize("member", ["FP16", "INT8", "INT4", "UNDEFINED"])
 def test_quantize_type_has_member(member):
     assert member in QuantizeType.__members__