Initial commit

python/tests/detail/distance_helper.py

@@ -0,0 +1,302 @@
+import logging
+import math
+import numpy as np
+
+from zvec import (
+    MetricType,
+    DataType,
+    QuantizeType,
+    Doc,
+    CollectionSchema,
+    FieldSchema,
+    VectorSchema,
+)
+
+from typing import Dict
+
+
+def is_float_equal(actual, expected, rel_tol=1e-5, abs_tol=1e-8):
+    if actual is None and expected is None:
+        return True
+    return math.isclose(actual, expected, rel_tol=rel_tol, abs_tol=abs_tol)
+
+
+def is_dense_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
+    """Compare two dense vectors with tolerance."""
+    return np.allclose(vec1, vec2, rtol=rtol, atol=atol)
+
+
+def is_sparse_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
+    """Compare two sparse vectors with tolerance."""
+    # Check if they have the same keys
+    if set(vec1.keys()) != set(vec2.keys()):
+        return False
+
+    # Check if all values are close
+    for key in vec1:
+        if not math.isclose(vec1[key], vec2[key], rel_tol=rtol, abs_tol=atol):
+            return False
+
+    return True
+
+
+def is_float_array_equal(arr1, arr2, rtol=1e-5, atol=1e-8):
+    """Compare two float arrays with tolerance."""
+    return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
+
+
+def is_double_array_equal(arr1, arr2, rtol=1e-9, atol=1e-12):
+    """Compare two double arrays with tolerance."""
+    return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
+
+
+def is_int_array_equal(arr1, arr2):
+    """Compare two integer arrays with exact equality."""
+    return np.array_equal(arr1, arr2)
+
+
+def cosine_distance_dense(
+    vec1,
+    vec2,
+    dtype: DataType = DataType.VECTOR_FP32,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
+        vec1 = [np.float16(a) for a in vec1]
+        vec2 = [np.float16(b) for b in vec2]
+
+    dot_product = sum(a * b for a, b in zip(vec1, vec2))
+
+    magnitude1 = math.sqrt(sum(a * a for a in vec1))
+    magnitude2 = math.sqrt(sum(b * b for b in vec2))
+
+    if magnitude1 == 0 or magnitude2 == 0:
+        return 0.0
+
+    return 1 - dot_product / (magnitude1 * magnitude2)
+
+
+def dp_distance_dense(
+    vec1,
+    vec2,
+    dtype: DataType = DataType.VECTOR_FP32,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
+        return sum(np.float16(a) * np.float16(b) for a, b in zip(vec1, vec2))
+    return sum(a * b for a, b in zip(vec1, vec2))
+
+
+def euclidean_distance_dense(
+    vec1,
+    vec2,
+    dtype: DataType = DataType.VECTOR_FP32,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
+        return sum((np.float16(a) - np.float16(b)) ** 2 for a, b in zip(vec1, vec2))
+    return sum((a - b) ** 2 for a, b in zip(vec1, vec2))
+
+
+def distance_dense(
+    vec1,
+    vec2,
+    metric: MetricType,
+    data_type: DataType = DataType.VECTOR_FP32,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    if metric == MetricType.COSINE:
+        return cosine_distance_dense(vec1, vec2, data_type, quantize_type)
+    elif metric == MetricType.L2:
+        return euclidean_distance_dense(vec1, vec2, data_type, quantize_type)
+    elif metric == MetricType.IP:
+        return dp_distance_dense(vec1, vec2, data_type, quantize_type)
+    else:
+        raise ValueError("Unsupported metric type")
+
+
+def dp_distance_sparse(
+    vec1,
+    vec2,
+    data_type: DataType = DataType.SPARSE_VECTOR_FP32,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    dot_product = 0.0
+    for dim in set(vec1.keys()) & set(vec2.keys()):
+        if (
+            data_type == DataType.SPARSE_VECTOR_FP16
+            or quantize_type == QuantizeType.FP16
+        ):
+            vec1[dim] = np.float16(vec1[dim])
+            vec2[dim] = np.float16(vec2[dim])
+        dot_product += vec1[dim] * vec2[dim]
+    return dot_product
+
+
+def distance(
+    vec1,
+    vec2,
+    metric: MetricType,
+    data_type: DataType,
+    quantize_type: QuantizeType = QuantizeType.UNDEFINED,
+):
+    is_sparse = (
+        data_type == DataType.SPARSE_VECTOR_FP32
+        or data_type == DataType.SPARSE_VECTOR_FP16
+    )
+
+    if is_sparse:
+        if metric != MetricType.IP:
+            raise ValueError("Unsupported metric type for sparse vectors")
+
+    if is_sparse:
+        return dp_distance_sparse(vec1, vec2, data_type, quantize_type)
+    else:
+        return distance_dense(vec1, vec2, metric, data_type, quantize_type)
+
+
+def calculate_rrf_score(rank, k=60):
+    return 1.0 / (k + rank + 1)
+
+
+def calculate_multi_vector_rrf_scores(query_results: Dict[str, Doc], k=60):
+    rrf_scores = {}
+
+    for vector_name, docs in query_results.items():
+        for rank, doc in enumerate(docs):
+            doc_id = doc.id
+            rrf_score = calculate_rrf_score(rank, k)
+            if doc_id in rrf_scores:
+                rrf_scores[doc_id] += rrf_score
+            else:
+                rrf_scores[doc_id] = rrf_score
+
+    return rrf_scores
+
+
+def calculate_multi_vector_weighted_scores(
+    query_results: Dict[str, Doc], weights: Dict[str, float], metric: MetricType
+):
+    def _normalize_score(score: float, metric: MetricType) -> float:
+        if metric == MetricType.L2:
+            return 1.0 - 2 * math.atan(score) / math.pi
+        if metric == MetricType.IP:
+            return 0.5 + math.atan(score) / math.pi
+        if metric == MetricType.COSINE:
+            return 1.0 - score / 2.0
+        raise ValueError("Unsupported metric type")
+
+    weighted_scores = {}
+
+    for vector_name, docs in query_results.items():
+        weight = weights.get(vector_name, 1.0)
+
+        for doc in docs:
+            doc_id = doc.id
+            weighted_score = (_normalize_score(doc.score, metric)) * weight
+            if doc_id in weighted_scores:
+                weighted_scores[doc_id] += weighted_score
+            else:
+                weighted_scores[doc_id] = weighted_score
+
+    return weighted_scores
+
+
+def is_field_equal(field1, field2, schema: FieldSchema) -> bool:
+    if field1 is None and field2 is None:
+        return True
+    if field1 is None or field2 is None:
+        return False
+
+    if schema.data_type == DataType.ARRAY_FLOAT:
+        return is_float_array_equal(field1, field2)
+    elif schema.data_type == DataType.ARRAY_DOUBLE:
+        return is_double_array_equal(field1, field2)
+    elif schema.data_type in [
+        DataType.ARRAY_INT32,
+        DataType.ARRAY_INT64,
+        DataType.ARRAY_BOOL,
+        DataType.ARRAY_STRING,
+        DataType.ARRAY_UINT32,
+        DataType.ARRAY_UINT64,
+        DataType.ARRAY_INT64,
+    ]:
+        return is_int_array_equal(field1, field2)
+    elif schema.data_type in [DataType.FLOAT, DataType.DOUBLE]:
+        return is_float_equal(field1, field2)
+
+    return field1 == field2
+
+
+def is_vector_equal(vec1, vec2, schema: VectorSchema) -> bool:
+    if (
+        schema.data_type == DataType.SPARSE_VECTOR_FP16
+        or schema.data_type == DataType.VECTOR_FP16
+    ):
+        # skip fp16 vector equal
+        return True
+
+    is_sparse = (
+        schema.data_type == DataType.SPARSE_VECTOR_FP32
+        or schema.data_type == DataType.SPARSE_VECTOR_FP16
+    )
+
+    if is_sparse:
+        return is_sparse_vector_equal(vec1, vec2)
+    else:
+        return is_dense_vector_equal(vec1, vec2)
+
+
+def is_doc_equal(
+    doc1: Doc,
+    doc2: Doc,
+    schema: CollectionSchema,
+    except_score: bool = True,
+    include_vector: bool = True,
+):
+    if doc1.id != doc2.id:
+        logging.error("doc ids are not equal")
+        return False
+
+    reduce_field_names = set(doc1.field_names() + doc2.field_names())
+    reduce_vector_names = set(doc1.vector_names() + doc2.vector_names())
+
+    is_doc1_fields_empty = doc1.fields is None or doc1.fields == {}
+    is_doc2_fields_empty = doc2.fields is None or doc2.fields == {}
+
+    if is_doc1_fields_empty or is_doc2_fields_empty:
+        if is_doc1_fields_empty != is_doc2_fields_empty:
+            return False
+    else:
+        for field_name in reduce_field_names:
+            field_schema = schema.field(field_name)
+            if field_schema is None:
+                return False
+            if is_field_equal(
+                doc1.field(field_name), doc2.field(field_name), field_schema
+            ):
+                continue
+            else:
+                logging.error(f"{field_name} are not equal")
+                return False
+
+    if include_vector:
+        is_doc1_vectors_empty = doc1.vectors is None or doc1.vectors == {}
+        is_doc2_vectors_empty = doc2.vectors is None or doc2.vectors == {}
+
+        if is_doc1_vectors_empty or is_doc2_vectors_empty:
+            if is_doc1_fields_empty != is_doc2_vectors_empty:
+                return False
+        else:
+            for vector_name in reduce_vector_names:
+                vector_schema = schema.vector(vector_name)
+                if vector_schema is None:
+                    return False
+                if is_vector_equal(
+                    doc1.vector(vector_name), doc2.vector(vector_name), vector_schema
+                ):
+                    continue
+                else:
+                    return False
+
+    return True

python/tests/detail/doc_helper.py

@@ -0,0 +1,377 @@
+from zvec import CollectionSchema, Doc
+
+from support_helper import *
+
+import numpy as np
+from typing import Literal, Optional, Union, Tuple
+
+import random
+import string
+
+
+def generate_constant_vector(
+    i: int, dimension: int, dtype: Literal["int8", "float16", "float32"] = "float32"
+):
+    if dtype == "int8":
+        vec = [i % 128] * dimension
+        vec[i % dimension] = (i + 1) % 128
+    else:
+        vec = [i / 256.0] * dimension
+        vec[i % dimension] = (i + 1) / 256.0
+
+    return vec
+
+
+def generate_sparse_vector(i: int):
+    return {i: i + 0.1}
+
+
+def generate_vectordict(i: int, schema: CollectionSchema) -> Doc:
+    doc_fields = {}
+    doc_vectors = {}
+    doc_fields = {}
+    doc_vectors = {}
+    for field in schema.fields:
+        if field.data_type == DataType.BOOL:
+            doc_fields[field.name] = i % 2 == 0
+        elif field.data_type == DataType.INT32:
+            doc_fields[field.name] = i
+        elif field.data_type == DataType.UINT32:
+            doc_fields[field.name] = i
+        elif field.data_type == DataType.INT64:
+            doc_fields[field.name] = i
+        elif field.data_type == DataType.UINT64:
+            doc_fields[field.name] = i
+        elif field.data_type == DataType.FLOAT:
+            doc_fields[field.name] = float(i) + 0.1
+        elif field.data_type == DataType.DOUBLE:
+            doc_fields[field.name] = float(i) + 0.11
+        elif field.data_type == DataType.STRING:
+            doc_fields[field.name] = f"test_{i}"
+        elif field.data_type == DataType.ARRAY_BOOL:
+            doc_fields[field.name] = [i % 2 == 0, i % 3 == 0]
+        elif field.data_type == DataType.ARRAY_INT32:
+            doc_fields[field.name] = [i, i + 1, i + 2]
+        elif field.data_type == DataType.ARRAY_UINT32:
+            doc_fields[field.name] = [i, i + 1, i + 2]
+        elif field.data_type == DataType.ARRAY_INT64:
+            doc_fields[field.name] = [i, i + 1, i + 2]
+        elif field.data_type == DataType.ARRAY_UINT64:
+            doc_fields[field.name] = [i, i + 1, i + 2]
+        elif field.data_type == DataType.ARRAY_FLOAT:
+            doc_fields[field.name] = [float(i + 0.1), float(i + 1.1), float(i + 2.1)]
+        elif field.data_type == DataType.ARRAY_DOUBLE:
+            doc_fields[field.name] = [float(i + 0.11), float(i + 1.11), float(i + 2.11)]
+        elif field.data_type == DataType.ARRAY_STRING:
+            doc_fields[field.name] = [f"test_{i}", f"test_{i + 1}", f"test_{i + 2}"]
+        else:
+            raise ValueError(f"Unsupported field type: {field.data_type}")
+    for vector in schema.vectors:
+        if vector.data_type == DataType.VECTOR_FP16:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i, vector.dimension, "float16"
+            )
+        elif vector.data_type == DataType.VECTOR_FP32:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i, vector.dimension, "float32"
+            )
+        elif vector.data_type == DataType.VECTOR_INT8:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i,
+                vector.dimension,
+                "int8",
+            )
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
+            doc_vectors[vector.name] = generate_sparse_vector(i)
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
+            doc_vectors[vector.name] = generate_sparse_vector(i)
+        else:
+            raise ValueError(f"Unsupported vector type: {vector.data_type}")
+    return doc_fields, doc_vectors
+
+
+def generate_doc(i: int, schema: CollectionSchema) -> Doc:
+    doc_fields = {}
+    doc_vectors = {}
+    doc_fields, doc_vectors = generate_vectordict(i, schema)
+    doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
+    return doc
+
+
+def generate_update_doc(i: int, schema: CollectionSchema) -> Doc:
+    doc_fields = {}
+    doc_vectors = {}
+    for field in schema.fields:
+        if field.data_type == DataType.BOOL:
+            doc_fields[field.name] = (i + 1) % 2 == 0
+        elif field.data_type == DataType.INT32:
+            doc_fields[field.name] = i + 1
+        elif field.data_type == DataType.UINT32:
+            doc_fields[field.name] = i + 1
+        elif field.data_type == DataType.INT64:
+            doc_fields[field.name] = i + 1
+        elif field.data_type == DataType.UINT64:
+            doc_fields[field.name] = i + 1
+        elif field.data_type == DataType.FLOAT:
+            doc_fields[field.name] = float(i + 1) + 0.1
+        elif field.data_type == DataType.DOUBLE:
+            doc_fields[field.name] = float(i) + 0.11
+        elif field.data_type == DataType.STRING:
+            doc_fields[field.name] = f"test_{i + 1}"
+        elif field.data_type == DataType.ARRAY_BOOL:
+            doc_fields[field.name] = [(i + 1) % 2 == 0, (i + 1) % 3 == 0]
+        elif field.data_type == DataType.ARRAY_INT32:
+            doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
+        elif field.data_type == DataType.ARRAY_UINT32:
+            doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
+        elif field.data_type == DataType.ARRAY_INT64:
+            doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
+        elif field.data_type == DataType.ARRAY_UINT64:
+            doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
+        elif field.data_type == DataType.ARRAY_FLOAT:
+            doc_fields[field.name] = [
+                float((i + 1) + 0.1),
+                float((i + 1) + 1.1),
+                float((i + 1) + 2.1),
+            ]
+        elif field.data_type == DataType.ARRAY_DOUBLE:
+            doc_fields[field.name] = [
+                float((i + 1) + 0.11),
+                float((i + 1) + 1.11),
+                float((i + 1) + 2.11),
+            ]
+        elif field.data_type == DataType.ARRAY_STRING:
+            doc_fields[field.name] = [
+                f"test_{i + 1}",
+                f"test_{(i + 1) + 1}",
+                f"test_{(i + 1) + 2}",
+            ]
+        else:
+            raise ValueError(f"Unsupported field type: {field.data_type}")
+    for vector in schema.vectors:
+        if vector.data_type == DataType.VECTOR_FP16:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i + 1, DEFAULT_VECTOR_DIMENSION, "float16"
+            )
+        elif vector.data_type == DataType.VECTOR_FP32:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i + 1, DEFAULT_VECTOR_DIMENSION, "float32"
+            )
+        elif vector.data_type == DataType.VECTOR_INT8:
+            doc_vectors[vector.name] = generate_constant_vector(
+                i + 1,
+                DEFAULT_VECTOR_DIMENSION,
+                "int8",
+            )
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
+            doc_vectors[vector.name] = generate_sparse_vector(i)
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
+            doc_vectors[vector.name] = generate_sparse_vector(i)
+        else:
+            raise ValueError(f"Unsupported vector type: {vector.data_type}")
+    doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
+    return doc
+
+
+def generate_doc_random(i, schema: CollectionSchema) -> Doc:
+    doc_fields = {}
+    doc_vectors = {}
+
+    random.seed(i)
+
+    for field in schema.fields:
+        if field.data_type == DataType.BOOL:
+            doc_fields[field.name] = random.choice([True, False])
+        elif field.data_type == DataType.INT32:
+            doc_fields[field.name] = random.randint(-2147483648, 2147483647)
+        elif field.data_type == DataType.UINT32:
+            doc_fields[field.name] = random.randint(0, 4294967295)
+        elif field.data_type == DataType.INT64:
+            doc_fields[field.name] = random.randint(
+                -9223372036854775808, 9223372036854775807
+            )
+        elif field.data_type == DataType.UINT64:
+            doc_fields[field.name] = random.randint(0, 18446744073709551615)
+        elif field.data_type == DataType.FLOAT:
+            doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
+        elif field.data_type == DataType.DOUBLE:
+            doc_fields[field.name] = random.uniform(
+                -1.7976931348623157e308, 1.7976931348623157e308
+            )
+        elif field.data_type == DataType.STRING:
+            length = random.randint(1, 999)
+            doc_fields[field.name] = "".join(
+                random.choices(string.ascii_letters + string.digits, k=length)
+            )
+        elif field.data_type == DataType.ARRAY_BOOL:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.choice([True, False]) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_INT32:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(-2147483648, 2147483647) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_UINT32:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(0, 4294967295) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_INT64:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(-9223372036854775808, 9223372036854775807)
+                for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_UINT64:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(0, 18446744073709551615) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_FLOAT:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_DOUBLE:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
+                for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_STRING:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                "".join(
+                    random.choices(
+                        string.ascii_letters + string.digits, k=random.randint(1, 100)
+                    )
+                )
+                for _ in range(array_length)
+            ]
+        else:
+            raise ValueError(f"Unsupported field type: {field.data_type}")
+
+    for vector in schema.vectors:
+        if vector.data_type == DataType.VECTOR_FP16:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
+            )
+        elif vector.data_type == DataType.VECTOR_FP32:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
+            )
+        elif vector.data_type == DataType.VECTOR_INT8:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
+            )
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
+            doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
+            doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
+        else:
+            raise ValueError(f"Unsupported vector type: {vector.data_type}")
+
+    doc = Doc(id=i, fields=doc_fields, vectors=doc_vectors)
+    return doc
+
+
+def generate_vectordict_random(schema: CollectionSchema):
+    doc_fields = {}
+    doc_vectors = {}
+    for field in schema.fields:
+        if field.data_type == DataType.BOOL:
+            doc_fields[field.name] = random.choice([True, False])
+        elif field.data_type == DataType.INT32:
+            doc_fields[field.name] = random.randint(-2147483648, 2147483647)
+        elif field.data_type == DataType.UINT32:
+            doc_fields[field.name] = random.randint(0, 4294967295)
+        elif field.data_type == DataType.INT64:
+            doc_fields[field.name] = random.randint(
+                -9223372036854775808, 9223372036854775807
+            )
+        elif field.data_type == DataType.UINT64:
+            doc_fields[field.name] = random.randint(0, 18446744073709551615)
+        elif field.data_type == DataType.FLOAT:
+            doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
+        elif field.data_type == DataType.DOUBLE:
+            doc_fields[field.name] = random.uniform(
+                -1.7976931348623157e308, 1.7976931348623157e308
+            )
+        elif field.data_type == DataType.STRING:
+            length = random.randint(1, 999)
+            doc_fields[field.name] = "".join(
+                random.choices(string.ascii_letters + string.digits, k=length)
+            )
+        elif field.data_type == DataType.ARRAY_BOOL:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.choice([True, False]) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_INT32:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(-2147483648, 2147483647) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_UINT32:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(0, 4294967295) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_INT64:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(-9223372036854775808, 9223372036854775807)
+                for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_UINT64:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.randint(0, 18446744073709551615) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_FLOAT:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_DOUBLE:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
+                for _ in range(array_length)
+            ]
+        elif field.data_type == DataType.ARRAY_STRING:
+            array_length = random.randint(0, 10)
+            doc_fields[field.name] = [
+                "".join(
+                    random.choices(
+                        string.ascii_letters + string.digits, k=random.randint(1, 100)
+                    )
+                )
+                for _ in range(array_length)
+            ]
+        else:
+            raise ValueError(f"Unsupported field type: {field.data_type}")
+
+    for vector in schema.vectors:
+        if vector.data_type == DataType.VECTOR_FP16:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
+            )
+        elif vector.data_type == DataType.VECTOR_FP32:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
+            )
+        elif vector.data_type == DataType.VECTOR_INT8:
+            doc_vectors[vector.name] = generate_constant_vector(
+                random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
+            )
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
+            doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
+        elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
+            doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
+        else:
+            raise ValueError(f"Unsupported vector type: {vector.data_type}")
+
+    return doc_fields, doc_vectors

python/tests/detail/fixture_helper.py

@@ -0,0 +1,319 @@
+import pytest
+import logging
+
+from typing import Any, Generator
+
+import zvec
+from zvec import (
+    CollectionOption,
+    InvertIndexParam,
+    HnswIndexParam,
+    IVFIndexParam,
+    FieldSchema,
+    VectorSchema,
+    CollectionSchema,
+    Collection,
+    Doc,
+    VectorQuery,
+)
+
+from support_helper import *
+
+
+@pytest.fixture(scope="session")
+def basic_schema(collection_name="test_collection") -> CollectionSchema:
+    return CollectionSchema(
+        name=collection_name if len(collection_name) > 0 else "test_collection",
+        fields=[
+            FieldSchema(
+                "id",
+                DataType.INT64,
+                nullable=False,
+                index_param=InvertIndexParam(enable_range_optimization=True),
+            ),
+            FieldSchema(
+                "name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
+            ),
+            FieldSchema("weight", DataType.FLOAT, nullable=True),
+        ],
+        vectors=[
+            VectorSchema(
+                "dense",
+                DataType.VECTOR_FP32,
+                dimension=128,
+                index_param=HnswIndexParam(),
+            ),
+            VectorSchema(
+                "sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
+            ),
+        ],
+    )
+
+
+@pytest.fixture(scope="session")
+def full_schema(
+    nullable: bool = False,
+    has_index: bool = False,
+) -> CollectionSchema:
+    scalar_index_param = None
+    vector_index_param = None
+    if has_index:
+        scalar_index_param = InvertIndexParam(enable_range_optimization=True)
+        vector_index_param = HnswIndexParam()
+
+    fields = []
+    for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
+        fields.append(
+            FieldSchema(
+                v,
+                k,
+                nullable=nullable,
+                index_param=scalar_index_param,
+            )
+        )
+    vetors = []
+    for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
+        vetors.append(
+            VectorSchema(
+                v,
+                k,
+                dimension=DEFAULT_VECTOR_DIMENSION,
+                index_param=vector_index_param,
+            )
+        )
+
+    return CollectionSchema(
+        name="full_collection",
+        fields=fields,
+        vectors=vetors,
+    )
+
+
+@pytest.fixture(scope="function")
+def full_schema_new(request) -> CollectionSchema:
+    if hasattr(request, "param"):
+        nullable, has_index, vector_index = request.param
+    else:
+        nullable, has_index, vector_index = True, False, HnswIndexParam()
+
+    scalar_index_param = None
+    vector_index_param = None
+    if has_index:
+        scalar_index_param = InvertIndexParam(enable_range_optimization=True)
+        vector_index_param = vector_index
+
+    fields = []
+    for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
+        fields.append(
+            FieldSchema(
+                v,
+                k,
+                nullable=nullable,
+                index_param=scalar_index_param,
+            )
+        )
+    vectors = []
+    for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
+        vectors.append(
+            VectorSchema(
+                v,
+                k,
+                dimension=DEFAULT_VECTOR_DIMENSION,
+                index_param=vector_index_param,
+            )
+        )
+
+    return CollectionSchema(
+        name="full_collection_new",
+        fields=fields,
+        vectors=vectors,
+    )
+
+
+@pytest.fixture(scope="function")
+def full_schema_ivf(request) -> CollectionSchema:
+    if hasattr(request, "param"):
+        nullable, has_index, vector_index = request.param
+    else:
+        nullable, has_index, vector_index = True, False, IVFIndexParam()
+
+    scalar_index_param = None
+    vector_index_param = None
+    if has_index:
+        scalar_index_param = InvertIndexParam(enable_range_optimization=True)
+        vector_index_param = vector_index
+
+    fields = []
+    for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
+        fields.append(
+            FieldSchema(
+                v,
+                k,
+                nullable=nullable,
+                index_param=scalar_index_param,
+            )
+        )
+    vectors = []
+    for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
+        if v in ["vector_fp16_field", "vector_fp32_field"]:
+            vectors.append(
+                VectorSchema(
+                    v,
+                    k,
+                    dimension=DEFAULT_VECTOR_DIMENSION,
+                    index_param=vector_index_param,
+                )
+            )
+
+    return CollectionSchema(
+        name="full_collection_ivf",
+        fields=fields,
+        vectors=vectors,
+    )
+
+
+@pytest.fixture(scope="function")
+def single_vector_schema(
+    data_type: DataType,
+) -> CollectionSchema:
+    vector_schema = [
+        VectorSchema(
+            DEFAULT_VECTOR_FIELD_NAME[data_type],
+            data_type,
+            DEFAULT_VECTOR_DIMENSION,
+        )
+    ]
+
+    return CollectionSchema(
+        name="full_collection",
+        vectors=vector_schema,
+    )
+
+
+@pytest.fixture(scope="function")
+def single_vector_schema_with_index_param(
+    data_type: DataType, index_param
+) -> CollectionSchema:
+    vector_schema = [
+        VectorSchema(
+            DEFAULT_VECTOR_FIELD_NAME[data_type],
+            data_type,
+            DEFAULT_VECTOR_DIMENSION,
+            index_param,
+        )
+    ]
+
+    return CollectionSchema(
+        name="full_collection",
+        vectors=vector_schema,
+    )
+
+
+def create_collection_fixture(
+    collection_temp_dir, schema: CollectionSchema, collection_option: CollectionOption
+) -> Generator[Any, Any, Collection]:
+    """Common helper function to create and manage collection fixtures."""
+    coll = zvec.create_and_open(
+        path=str(collection_temp_dir),
+        schema=schema,
+        option=collection_option,
+    )
+
+    assert coll is not None, "Failed to create and open collection"
+    assert coll.path == str(collection_temp_dir)
+    assert coll.schema.name == schema.name
+    assert list(coll.schema.fields) == list(schema.fields)
+    assert list(coll.schema.vectors) == list(schema.vectors)
+    assert coll.option.read_only == collection_option.read_only
+    assert coll.option.enable_mmap == collection_option.enable_mmap
+
+    try:
+        yield coll
+    finally:
+        if hasattr(coll, "destroy") and coll is not None:
+            try:
+                coll.destroy()
+            except Exception as e:
+                logging.warning(f"Warning: failed to destroy collection: {e}")
+
+
+@pytest.fixture(scope="function")
+def basic_collection(
+    collection_temp_dir, basic_schema, collection_option
+) -> Generator[Any, Any, Collection]:
+    yield from create_collection_fixture(
+        collection_temp_dir, basic_schema, collection_option
+    )
+
+
+@pytest.fixture(scope="function")
+def collection_option():
+    return CollectionOption(read_only=False, enable_mmap=True)
+
+
+@pytest.fixture(scope="function")
+def collection_temp_dir(tmp_path_factory):
+    temp_dir = tmp_path_factory.mktemp("zvec")
+    collection_path = temp_dir / "test_collection_path"
+    return str(collection_path)
+
+
+@pytest.fixture(scope="function")
+def full_collection(
+    collection_temp_dir,
+    full_schema,
+    collection_option,
+    nullable: bool = True,
+    has_index: bool = False,
+) -> Generator[Any, Any, Collection]:
+    yield from create_collection_fixture(
+        collection_temp_dir, full_schema, collection_option
+    )
+
+
+@pytest.fixture(scope="function")
+def full_collection_new(
+    collection_temp_dir, full_schema_new, collection_option
+) -> Generator[Any, Any, Collection]:
+    yield from create_collection_fixture(
+        collection_temp_dir, full_schema_new, collection_option
+    )
+
+
+@pytest.fixture(scope="function")
+def full_collection_ivf(
+    collection_temp_dir, full_schema_ivf, collection_option
+) -> Generator[Any, Any, Collection]:
+    yield from create_collection_fixture(
+        collection_temp_dir, full_schema_ivf, collection_option
+    )
+
+
+@pytest.fixture
+def sample_field_list(nullable: bool = True, scalar_index_param=None, name_prefix=""):
+    field_list = []
+    for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
+        field_list.append(
+            FieldSchema(
+                f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
+                k,
+                nullable=nullable,
+                index_param=scalar_index_param,
+            )
+        )
+    return field_list
+
+
+@pytest.fixture
+def sample_vector_list(vector_index_param=None, name_prefix=""):
+    vector_list = []
+    for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
+        vector_list.append(
+            VectorSchema(
+                f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
+                k,
+                dimension=DEFAULT_VECTOR_DIMENSION,
+                index_param=vector_index_param,
+            )
+        )
+    return vector_list

python/tests/detail/params_helper.py

@@ -0,0 +1,209 @@
+from zvec import (
+    CollectionOption,
+    IndexOption,
+    OptimizeOption,
+    InvertIndexParam,
+    HnswIndexParam,
+    IVFIndexParam,
+    FlatIndexParam,
+    AlterColumnOption,
+    AddColumnOption,
+    DataType,
+    MetricType,
+    QuantizeType,
+)
+
+
+VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
+    DataType.VECTOR_FP32: [
+        HnswIndexParam(),
+        HnswIndexParam(
+            metric_type=MetricType.IP,
+            m=16,
+            ef_construction=100,
+            quantize_type=QuantizeType.INT8,
+        ),
+        HnswIndexParam(
+            metric_type=MetricType.COSINE,
+            m=24,
+            ef_construction=150,
+            quantize_type=QuantizeType.INT4,
+        ),
+        HnswIndexParam(
+            metric_type=MetricType.L2,
+            m=32,
+            ef_construction=200,
+            quantize_type=QuantizeType.FP16,
+        ),
+        FlatIndexParam(),
+        FlatIndexParam(metric_type=MetricType.IP, quantize_type=QuantizeType.INT4),
+        FlatIndexParam(metric_type=MetricType.L2, quantize_type=QuantizeType.INT8),
+        FlatIndexParam(metric_type=MetricType.COSINE, quantize_type=QuantizeType.FP16),
+        IVFIndexParam(),
+        IVFIndexParam(
+            metric_type=MetricType.IP,
+            quantize_type=QuantizeType.INT4,
+            n_list=100,
+            n_iters=10,
+            use_soar=False,
+        ),
+        IVFIndexParam(
+            metric_type=MetricType.L2,
+            quantize_type=QuantizeType.INT8,
+            n_list=200,
+            n_iters=20,
+            use_soar=True,
+        ),
+        IVFIndexParam(
+            metric_type=MetricType.COSINE,
+            quantize_type=QuantizeType.FP16,
+            n_list=150,
+            n_iters=15,
+            use_soar=False,
+        ),
+    ],
+    DataType.VECTOR_FP16: [
+        HnswIndexParam(),
+        FlatIndexParam(),
+        # IVFIndexParam(),
+    ],
+    DataType.VECTOR_INT8: [
+        HnswIndexParam(),
+        FlatIndexParam(),
+        # IVFIndexParam(),
+    ],
+    DataType.SPARSE_VECTOR_FP32: [
+        HnswIndexParam(),
+        FlatIndexParam(),
+        HnswIndexParam(
+            metric_type=MetricType.IP,
+            m=16,
+            ef_construction=100,
+            quantize_type=QuantizeType.FP16,
+        ),
+    ],
+    DataType.SPARSE_VECTOR_FP16: [
+        HnswIndexParam(),
+        FlatIndexParam(),
+        HnswIndexParam(
+            metric_type=MetricType.IP,
+            m=16,
+            ef_construction=100,
+        ),
+    ],
+}
+
+VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
+    (data_type, param)
+    for data_type, params in VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
+    for param in params
+]
+
+INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
+    DataType.VECTOR_FP32: [
+        InvertIndexParam(),
+    ],
+    DataType.VECTOR_FP16: [
+        InvertIndexParam(),
+    ],
+    DataType.VECTOR_INT8: [
+        InvertIndexParam(),
+    ],
+    DataType.SPARSE_VECTOR_FP32: [
+        HnswIndexParam(metric_type=MetricType.L2),
+        FlatIndexParam(metric_type=MetricType.COSINE),
+        IVFIndexParam(),
+        InvertIndexParam(),
+    ],
+    DataType.SPARSE_VECTOR_FP16: [
+        HnswIndexParam(metric_type=MetricType.L2),
+        FlatIndexParam(metric_type=MetricType.COSINE),
+        IVFIndexParam(),
+        InvertIndexParam(),
+    ],
+}
+
+INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
+    (data_type, param)
+    for data_type, params in INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
+    for param in params
+]
+
+COLLECTION_NAME_MAX_LENGTH = 64
+
+COLLECTION_NAME_VALID_LIST = [
+    "col",
+    "C0llECTION",
+    "Collection1",
+    "collection_2",
+    "123collection-",
+    "a" * COLLECTION_NAME_MAX_LENGTH,
+]
+
+COLLECTION_NAME_INVALID_LIST = [
+    "l",
+    "1C",
+    "",
+    " ",
+    None,
+    "abcdefghijklmnopqrstuvwxzy123456abcdefghijklmnopqrstuvwxzy1234561",
+    "test/",
+    "!@#$%^&*()test",
+]
+
+FIELD_NAME_VALID_LIST = [
+    "1",
+    "12",
+    "col",
+    "ID",
+    "name1",
+    "Weigt_12-",
+    "123age",
+    "name_with_underscores",
+    "123numeric_start",
+    "name-with-dashes",
+]
+
+FIELD_NAME_INVALID_LIST = [
+    "",
+    " ",
+    None,
+    "abcdefghijklmnopqrstuvwxzy1234561",
+    "test/",
+    "!@#$%^&*()test",
+    "name@with#special$chars",
+    "name with spaces",
+]
+
+FIELD_LIST_MAX_LENGTH = 1024
+VECTOR_LIST_MAX_LENGTH = 5
+DENSE_VECTOR_MAX_DIMENSION = 20000
+SPARSE_VECTOR_MAX_DIMENSION = 4096
+
+FIELD_VECTOR_LIST_DIMENSION_VALID_LIST = [
+    # field_list_len, vector_list_len, dimension
+    (1, 1, 1),
+    (2, 2, 512),
+    (512, 3, 1024),
+    (1024, 4, 20000),
+]
+
+FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST = [
+    # field_list_len, vector_list_len, dimension
+    (1, 1, 0),
+    (1, 1, -1),
+    (1, 1, "1"),
+    (1, 1, 20001),
+]
+
+
+INCOMPATIBLE_CONSTRUCTOR_ERROR_MSG = "incompatible constructor arguments"
+SCHEMA_VALIDATE_ERROR_MSG = "schema validate failed"
+CREATE_READ_ONLY_ERROR_MSG = "Unable to create collection with read-only mode"
+INCOMPATIBLE_FUNCTION_ERROR_MSG = "incompatible function arguments"
+INVALID_PATH_ERROR_MSG = "path validate failed"
+INDEX_NON_EXISTENT_COLUMN_ERROR_MSG = "not found in schema"
+ACCESS_DESTROYED_COLLECTION_ERROR_MSG = "is already destroyed"
+COLLECTION_PATH_NOT_EXIST_ERROR_MSG = "not exist"
+NOT_SUPPORT_ADD_COLUMN_ERROR_MSG = "Only support basic numeric data type"
+NOT_EXIST_COLUMN_TO_DROP_ERROR_MSG = "Column not exists"

python/tests/detail/support_helper.py

@@ -0,0 +1,126 @@
+from zvec import (
+    CollectionOption,
+    IndexOption,
+    OptimizeOption,
+    InvertIndexParam,
+    HnswIndexParam,
+    IVFIndexParam,
+    FlatIndexParam,
+    DataType,
+    IndexType,
+    QuantizeType,
+)
+
+SUPPORT_SCALAR_DATA_TYPES = [
+    DataType.BOOL,
+    DataType.FLOAT,
+    DataType.DOUBLE,
+    DataType.INT32,
+    DataType.INT64,
+    DataType.UINT32,
+    DataType.UINT64,
+    DataType.STRING,
+    DataType.ARRAY_BOOL,
+    DataType.ARRAY_FLOAT,
+    DataType.ARRAY_DOUBLE,
+    DataType.ARRAY_INT32,
+    DataType.ARRAY_INT64,
+    DataType.ARRAY_UINT32,
+    DataType.ARRAY_UINT64,
+    DataType.ARRAY_STRING,
+]
+
+DEFAULT_SCALAR_FIELD_NAME = {
+    DataType.BOOL: "bool_field",
+    DataType.FLOAT: "float_field",
+    DataType.DOUBLE: "double_field",
+    DataType.INT32: "int32_field",
+    DataType.INT64: "int64_field",
+    DataType.UINT32: "uint32_field",
+    DataType.UINT64: "uint64_field",
+    DataType.STRING: "string_field",
+    DataType.ARRAY_BOOL: "array_bool_field",
+    DataType.ARRAY_FLOAT: "array_float_field",
+    DataType.ARRAY_DOUBLE: "array_double_field",
+    DataType.ARRAY_INT32: "array_int32_field",
+    DataType.ARRAY_INT64: "array_int64_field",
+    DataType.ARRAY_UINT32: "array_uint32_field",
+    DataType.ARRAY_UINT64: "array_uint64_field",
+    DataType.ARRAY_STRING: "array_string_field",
+}
+
+SUPPORT_SCALAR_INDEX_TYPES = [
+    IndexType.INVERT,
+]
+
+SUPPORT_VECTOR_DATA_TYPES = [
+    DataType.VECTOR_FP16,
+    DataType.VECTOR_FP32,
+    DataType.VECTOR_INT8,
+    DataType.SPARSE_VECTOR_FP32,
+    DataType.SPARSE_VECTOR_FP16,
+]
+
+SUPPORT_VECTOR_INDEX_TYPES = [
+    IndexType.FLAT,
+    IndexType.HNSW,
+    IndexType.IVF,
+]
+
+DEFAULT_VECTOR_FIELD_NAME = {
+    DataType.VECTOR_FP16: "vector_fp16_field",
+    DataType.VECTOR_FP32: "vector_fp32_field",
+    DataType.VECTOR_INT8: "vector_int8_field",
+    DataType.SPARSE_VECTOR_FP32: "sparse_vector_fp32_field",
+    DataType.SPARSE_VECTOR_FP16: "sparse_vector_fp16_field",
+}
+
+DEFAULT_VECTOR_DIMENSION = 128
+
+SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP = {
+    DataType.VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
+    DataType.VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
+    DataType.VECTOR_INT8: [IndexType.FLAT, IndexType.HNSW],
+    DataType.SPARSE_VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW],
+    DataType.SPARSE_VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW],
+}
+
+SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP_PARAMS = [
+    (data_type, index_type)
+    for data_type, index_types in SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP.items()
+    for index_type in index_types
+]
+
+DEFAULT_INDEX_PARAMS = {
+    IndexType.FLAT: FlatIndexParam(),
+    IndexType.HNSW: HnswIndexParam(),
+    IndexType.IVF: IVFIndexParam(),
+    IndexType.INVERT: InvertIndexParam(),
+}
+
+SUPPORT_VECTOR_DATA_TYPE_QUANT_MAP = {
+    DataType.VECTOR_FP32: [QuantizeType.FP16, QuantizeType.INT8, QuantizeType.INT4],
+    DataType.SPARSE_VECTOR_FP32: [QuantizeType.FP16],
+}
+
+SUPPORT_ADD_COLUMN_DATA_TYPE = [
+    DataType.INT32,
+    DataType.UINT32,
+    DataType.INT64,
+    DataType.UINT64,
+    DataType.FLOAT,
+    DataType.DOUBLE,
+]
+
+NOT_SUPPORT_ADD_COLUMN_DATA_TYPE = [
+    DataType.BOOL,
+    DataType.STRING,
+    DataType.ARRAY_BOOL,
+    DataType.ARRAY_INT32,
+    DataType.ARRAY_INT64,
+    DataType.ARRAY_UINT32,
+    DataType.ARRAY_UINT64,
+    DataType.ARRAY_FLOAT,
+    DataType.ARRAY_DOUBLE,
+    DataType.ARRAY_STRING,
+]

python/tests/detail/test_collection_concurrency.py

@@ -0,0 +1,429 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import pytest
+import threading
+import numpy as np
+import zvec
+
+from zvec import (
+    CollectionOption,
+    InvertIndexParam,
+    HnswIndexParam,
+    Collection,
+    Doc,
+    DataType,
+    FieldSchema,
+    VectorSchema,
+)
+
+
+class TestCollectionConcurrency:
+    @pytest.fixture(scope="function")
+    def test_collection(self, tmp_path_factory):
+        """Fixture to create a test collection"""
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+                FieldSchema("weight", DataType.FLOAT, nullable=True),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                ),
+                VectorSchema(
+                    "sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
+                ),
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert coll is not None, "Failed to create and open collection"
+
+        yield coll
+
+        # Clean up
+        if hasattr(coll, "destroy") and coll is not None:
+            try:
+                coll.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+    def test_concurrent_read_write(self, test_collection: Collection):
+        results = []
+
+        def insert_docs(thread_id):
+            try:
+                docs = [
+                    Doc(
+                        id=f"{thread_id}_{i}",
+                        fields={
+                            "id": int(f"{thread_id}{i}"),
+                            "name": f"thread_{thread_id}_doc_{i}",
+                            "weight": float(i),
+                        },
+                        vectors={
+                            "dense": np.random.random(128).tolist(),
+                            "sparse": {1: float(i), 2: float(i * 2)},
+                        },
+                    )
+                    for i in range(5)
+                ]
+
+                result = test_collection.insert(docs)
+                results.append((thread_id, "insert", len(result)))
+            except Exception as e:
+                results.append((thread_id, "insert_exception", str(e)))
+
+        def query_docs(thread_id):
+            try:
+                result = test_collection.query(filter="id > 0", topk=10)
+                results.append((thread_id, "query", len(result)))
+            except Exception as e:
+                results.append((thread_id, "query_exception", str(e)))
+
+        # Create threads for concurrent operations
+        threads = []
+
+        # Start insert threads
+        for i in range(3):
+            thread = threading.Thread(target=insert_docs, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Start query threads
+        for i in range(3):
+            thread = threading.Thread(target=query_docs, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Analyze results
+        insert_results = [r for r in results if r[1] == "insert"]
+        query_results = [r for r in results if r[1] == "query"]
+
+        logging.info(
+            f"Concurrent read/write results - Inserts: {len(insert_results)}, Queries: {len(query_results)}"
+        )
+
+        # At least some operations should succeed
+        assert len(insert_results) + len(query_results) > 0
+
+    def test_concurrent_query(self, test_collection: Collection):
+        # First insert some data
+        docs = [
+            Doc(
+                id=f"{i}",
+                fields={"id": i, "name": f"test_{i}", "weight": float(i)},
+                vectors={
+                    "dense": np.random.random(128).tolist(),
+                    "sparse": {1: float(i), 2: float(i * 2)},
+                },
+            )
+            for i in range(20)
+        ]
+
+        insert_result = test_collection.insert(docs)
+        assert len(insert_result) == 20
+
+        results = []
+
+        def query_operation(thread_id):
+            """Perform query operation from a thread"""
+            try:
+                result = test_collection.query(filter=f"id > {thread_id}", topk=5)
+                results.append((thread_id, "query", len(result)))
+            except Exception as e:
+                results.append((thread_id, "query_exception", str(e)))
+
+        # Create multiple threads for concurrent queries
+        threads = []
+        for i in range(5):
+            thread = threading.Thread(target=query_operation, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Analyze results
+        query_results = [r for r in results if r[1] == "query"]
+        logging.info(f"Concurrent query results - Queries: {len(query_results)}")
+
+        # All query operations should succeed
+        assert len(query_results) == 5
+
+    def test_concurrent_modifications(self, test_collection: Collection):
+        # First insert some data
+        docs = [
+            Doc(
+                id=f"{i}",
+                fields={"id": i, "name": f"test_{i}", "weight": float(i)},
+                vectors={
+                    "dense": np.random.random(128).tolist(),
+                    "sparse": {1: float(i), 2: float(i * 2)},
+                },
+            )
+            for i in range(10)
+        ]
+
+        insert_result = test_collection.insert(docs)
+        assert len(insert_result) == 10
+
+        results = []
+
+        def update_operation(thread_id):
+            """Perform update operation from a thread"""
+            try:
+                # Each thread updates different documents
+                update_docs = [
+                    Doc(
+                        id=f"{i}",
+                        fields={
+                            "id": i,
+                            "name": f"updated_by_thread_{thread_id}",
+                            "weight": float(i + thread_id),
+                        },
+                        vectors={
+                            "dense": np.random.random(128).tolist(),
+                            "sparse": {1: float(i) + 0.5, 2: float(i * 2) + 0.5},
+                        },
+                    )
+                    for i in range(thread_id * 2, thread_id * 2 + 2)
+                ]
+
+                result = test_collection.update(update_docs)
+                results.append((thread_id, "update", len(result)))
+            except Exception as e:
+                results.append((thread_id, "update_exception", str(e)))
+
+        def delete_operation(thread_id):
+            """Perform delete operation from a thread"""
+            try:
+                # Each thread deletes different documents
+                delete_ids = [f"{thread_id * 2 + 2}", f"{thread_id * 2 + 3}"]
+                result = test_collection.delete(delete_ids)
+                results.append((thread_id, "delete", len(result)))
+            except Exception as e:
+                results.append((thread_id, "delete_exception", str(e)))
+
+        # Create threads for concurrent operations
+        threads = []
+
+        # Start update threads
+        for i in range(3):
+            thread = threading.Thread(target=update_operation, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Start delete threads
+        for i in range(2):
+            thread = threading.Thread(target=delete_operation, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Analyze results
+        update_results = [r for r in results if r[1] == "update"]
+        delete_results = [r for r in results if r[1] == "delete"]
+
+        logging.info(
+            f"Concurrent modification results - Updates: {len(update_results)}, Deletes: {len(delete_results)}"
+        )
+
+        # At least some operations should succeed
+        assert len(update_results) + len(delete_results) > 0
+
+    def test_read_write_locking(self, test_collection: Collection):
+        # Perform operations that should be thread-safe
+        docs = [
+            Doc(
+                id=f"{i}",
+                fields={"id": i, "name": f"test_{i}", "weight": float(i)},
+                vectors={
+                    "dense": np.random.random(128).tolist(),
+                    "sparse": {1: float(i), 2: float(i * 2)},
+                },
+            )
+            for i in range(5)
+        ]
+
+        # Insert data
+        insert_result = test_collection.insert(docs)
+        assert len(insert_result) == 5
+
+        # Concurrent operations should not cause data corruption
+        results = []
+
+        def mixed_operation(thread_id):
+            """Perform mixed operations from a thread"""
+            try:
+                # Mix of read and write operations
+                if thread_id % 2 == 0:
+                    # Read operation
+                    result = test_collection.fetch([f"{thread_id % 5}"])
+                    results.append((thread_id, "read", len(result)))
+                else:
+                    # Write operation
+                    doc = Doc(
+                        id=f"{thread_id % 5}",
+                        fields={
+                            "id": thread_id % 5,
+                            "name": f"mixed_op_{thread_id}",
+                            "weight": float(thread_id),
+                        },
+                        vectors={
+                            "dense": np.random.random(128).tolist(),
+                            "sparse": {1: float(thread_id), 2: float(thread_id * 2)},
+                        },
+                    )
+                    result = test_collection.upsert(doc)
+                    results.append((thread_id, "write", len(result)))
+            except Exception as e:
+                results.append((thread_id, "exception", str(e)))
+
+        # Create multiple threads
+        threads = []
+        for i in range(10):
+            thread = threading.Thread(target=mixed_operation, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Verify that the collection is still in a consistent state
+        final_result = test_collection.query()
+        assert len(final_result) >= 0  # Should not crash or return corrupted data
+
+    def test_race_condition_detection(self, test_collection: Collection):
+        # Insert initial data
+        docs = [
+            Doc(
+                id=f"{i}",
+                fields={"id": i, "name": f"initial_{i}", "weight": float(i)},
+                vectors={
+                    "dense": np.random.random(128).tolist(),
+                    "sparse": {1: float(i), 2: float(i * 2)},
+                },
+            )
+            for i in range(10)
+        ]
+
+        insert_result = test_collection.insert(docs)
+        assert len(insert_result) == 10
+
+        # Perform many rapid concurrent operations
+        operation_count = 100
+        results = []
+
+        def rapid_operation(op_id):
+            """Perform rapid operations"""
+            try:
+                # Alternate between different types of operations
+                if op_id % 4 == 0:
+                    # Insert
+                    doc = Doc(
+                        id=f"rapid_{op_id}",
+                        fields={
+                            "id": op_id,
+                            "name": f"rapid_{op_id}",
+                            "weight": float(op_id),
+                        },
+                        vectors={
+                            "dense": np.random.random(128).tolist(),
+                            "sparse": {1: float(op_id), 2: float(op_id * 2)},
+                        },
+                    )
+                    result = test_collection.insert(doc)
+                    results.append(("insert", len(result)))
+                elif op_id % 4 == 1:
+                    # Update
+                    doc = Doc(
+                        id=f"{op_id % 10}",
+                        fields={
+                            "id": op_id % 10,
+                            "name": f"rapid_update_{op_id}",
+                            "weight": float(op_id),
+                        },
+                        vectors={
+                            "dense": np.random.random(128).tolist(),
+                            "sparse": {1: float(op_id), 2: float(op_id * 2)},
+                        },
+                    )
+                    result = test_collection.update(doc)
+                    results.append(("update", len(result)))
+                elif op_id % 4 == 2:
+                    # Query
+                    result = test_collection.query(filter=f"id > {op_id % 5}", topk=3)
+                    results.append(("query", len(result)))
+                else:
+                    # Fetch
+                    result = test_collection.fetch([f"{op_id % 10}"])
+                    results.append(("fetch", len(result)))
+            except Exception as e:
+                results.append(("exception", str(e)))
+
+        # Create many threads for rapid concurrent operations
+        threads = []
+        for i in range(operation_count):
+            thread = threading.Thread(target=rapid_operation, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Verify collection is still functional
+        final_query = test_collection.query()
+        assert len(final_query) >= 0  # Should not be corrupted
+
+        logging.info(
+            f"Rapid concurrent operations completed - Total operations: {len(results)}"
+        )

python/tests/detail/test_collection_create_and_open.py

@@ -0,0 +1,792 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import threading
+import os
+
+from distance_helper import *
+from fixture_helper import *
+from doc_helper import *
+from params_helper import *
+
+
+def check_collection_info(
+    coll: Collection, schema: CollectionSchema, option: CollectionOption, path: str
+):
+    assert coll is not None, "Failed to create and open collection"
+    assert coll.path == path
+    assert coll.schema.name == schema.name
+    assert list(coll.schema.fields) == list(schema.fields)
+    assert list(coll.schema.vectors) == list(schema.vectors)
+    assert coll.option.read_only == option.read_only
+    assert coll.option.enable_mmap == option.enable_mmap
+
+
+def check_collection_basic(coll: Collection, optimize: bool = False):
+    schema = coll.schema
+
+    docs = [generate_doc(i, schema) for i in range(10)]
+
+    results = coll.insert(docs=docs)
+    assert len(results) == len(docs)
+    for result in results:
+        assert result.ok()
+
+    assert coll.stats.doc_count == len(docs)
+
+    def check_fetch_query():
+        results = coll.fetch([str(i) for i in range(len(docs))])
+        assert len(results) == len(docs)
+        for i in range(len(docs)):
+            assert str(i) in results
+
+        results = coll.query()
+        assert len(results) == len(docs)
+
+    check_fetch_query()
+
+    if optimize:
+        coll.optimize()
+        check_fetch_query()
+
+
+def check_collection_full(coll: Collection):
+    test_doc = generate_doc(1, coll.schema)
+
+    insert_result = coll.insert(test_doc)
+    assert insert_result.ok()
+
+    stats = coll.stats
+    assert stats.doc_count == 1
+
+    fetched_docs = coll.fetch(ids=["1"])
+    assert len(fetched_docs) == 1
+    assert "1" in fetched_docs
+    assert fetched_docs["1"] is not None
+    assert is_doc_equal(fetched_docs["1"], test_doc, coll.schema)
+
+    query_result = coll.query()
+    assert len(query_result) == 1
+
+    updated_doc = Doc(
+        id="1",
+        fields={"int32_field": 1},
+        vectors={"vector_fp32_field": [0.2] * 128},
+    )
+    update_result = coll.update(updated_doc)
+    assert update_result.ok()
+
+    upserted_doc = generate_doc(1, coll.schema)
+    upsert_result = coll.upsert(upserted_doc)
+    assert upsert_result.ok()
+
+    # 8. Delete document
+    delete_result = coll.delete("1")
+    assert delete_result.ok()
+
+    # Verify document was deleted
+    stats = coll.stats
+    assert stats.doc_count == 0
+
+
+valid_collection_options = [
+    # (read_only, enable_mmap)
+    (False, True),
+    (False, False),
+]
+invalid_collection_options = [
+    # (read_only, enable_mmap)
+    (True, True),
+    (True, False),
+]
+duplicate_names_test = [
+    ("field1", "field1", "vector1", "vector2"),
+    ("field1", "field2", "vector1", "vector1"),
+    (
+        "shared_name1",
+        "shared_name2",
+        "shared_name1",
+        "shared_name2",
+    ),
+]
+long_names = [
+    "a" * 100,  # 100 characters
+    "b" * 200,  # 200 characters
+]
+
+valid_path_list = [
+    "/tmp/nonexistent/directory/test_collection",
+    "test/collection/with/slashes",
+]
+invalid_path_list = [
+    "invalid:path",
+    "",
+    "test_collection_with_spaces ",
+    "test@#$%collection",
+]
+
+
+class TestCreateAndOpen:
+    @pytest.mark.parametrize("collection_name", COLLECTION_NAME_VALID_LIST)
+    def test_valid_collection_name(
+        self,
+        collection_temp_dir,
+        collection_name,
+        collection_option,
+        sample_field_list,
+        sample_vector_list,
+    ):
+        collection_schema = zvec.CollectionSchema(
+            name=collection_name,
+            fields=sample_field_list,
+            vectors=sample_vector_list,
+        )
+
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        check_collection_info(
+            coll, collection_schema, collection_option, collection_temp_dir
+        )
+        check_collection_basic(coll)
+
+        coll.destroy()
+
+    @pytest.mark.parametrize("collection_name", COLLECTION_NAME_INVALID_LIST)
+    def test_invalid_collection_name(
+        self,
+        collection_temp_dir,
+        collection_name,
+        collection_option,
+        sample_field_list,
+        sample_vector_list,
+    ):
+        with pytest.raises(Exception) as exc_info:
+            collection_schema = zvec.CollectionSchema(
+                name=collection_name,
+                fields=sample_field_list,
+                vectors=sample_vector_list,
+            )
+
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize("name_prefix", FIELD_NAME_VALID_LIST)
+    def test_valid_field_vector_name(
+        self,
+        collection_temp_dir,
+        collection_option,
+        name_prefix,
+        sample_field_list,
+        sample_vector_list,
+    ):
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=sample_field_list,
+            vectors=sample_vector_list,
+        )
+
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        check_collection_info(
+            coll, collection_schema, collection_option, collection_temp_dir
+        )
+        check_collection_basic(coll)
+
+        coll.destroy()
+
+    @pytest.mark.parametrize("field_name", FIELD_NAME_INVALID_LIST)
+    def test_invalid_field_name(
+        self, collection_temp_dir, collection_option, field_name
+    ):
+        with pytest.raises(Exception) as exc_info:
+            field_list = [FieldSchema(field_name, DataType.STRING)]
+            vector_list = [
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ]
+
+            collection_schema = zvec.CollectionSchema(
+                name="collection_name", fields=field_list, vectors=vector_list
+            )
+
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize("vector_name", FIELD_NAME_INVALID_LIST)
+    def test_invalid_vector_name(
+        self, collection_temp_dir, collection_option, vector_name
+    ):
+        with pytest.raises(Exception) as exc_info:
+            field_list = [
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                )
+            ]
+            vector_list = [
+                VectorSchema(vector_name, DataType.VECTOR_FP32, dimension=128)
+            ]
+
+            collection_schema = zvec.CollectionSchema(
+                name="collection_name", fields=field_list, vectors=vector_list
+            )
+
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize(
+        "field_list_len,vector_list_len,dimension",
+        FIELD_VECTOR_LIST_DIMENSION_VALID_LIST,
+    )
+    def test_valid_field_vector_size_dimension(
+        self,
+        collection_temp_dir,
+        collection_option,
+        field_list_len,
+        vector_list_len,
+        dimension,
+    ):
+        field_list = []
+        vector_list = []
+        for i in range(0, field_list_len):
+            field_list.append(
+                FieldSchema("id_" + str(i), DataType.INT64, nullable=True)
+            )
+
+        for i in range(0, vector_list_len):
+            vector_list.append(
+                VectorSchema(
+                    "dense_vector_" + str(i),
+                    DataType.VECTOR_FP32,
+                    dimension=dimension,
+                    index_param=HnswIndexParam(),
+                )
+            )
+
+        collection_schema = zvec.CollectionSchema(
+            name="test_dense_vector_list", fields=field_list, vectors=vector_list
+        )
+
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        check_collection_info(
+            coll, collection_schema, collection_option, collection_temp_dir
+        )
+        check_collection_basic(coll)
+
+        coll.destroy()
+
+    @pytest.mark.parametrize(
+        "field_list_len,vector_list_len,dimension",
+        FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST,
+    )
+    def test_invalid_field_vector_size_dimension(
+        self,
+        collection_temp_dir,
+        collection_option,
+        vector_list_len,
+        field_list_len,
+        dimension,
+    ):
+        with pytest.raises(Exception) as exc_info:
+            field_list = []
+            vector_list = []
+            for i in range(0, field_list_len):
+                field_list.append(
+                    FieldSchema(
+                        "id_" + str(i),
+                        DataType.INT64,
+                        nullable=False,
+                    )
+                )
+
+            for i in range(0, vector_list_len):
+                vector_list.append(
+                    VectorSchema(
+                        "dense_vector_" + str(i),
+                        DataType.VECTOR_FP32,
+                        dimension=dimension,
+                        index_param=HnswIndexParam(),
+                    )
+                )
+
+            collection_schema = zvec.CollectionSchema(
+                name="test_dense_vector_list", fields=field_list, vectors=vector_list
+            )
+
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    def test_valid_single_vector_field_construction(
+        self, collection_temp_dir, collection_option
+    ):
+        field = FieldSchema(
+            "id",
+            DataType.INT64,
+            nullable=True,
+            index_param=InvertIndexParam(enable_range_optimization=True),
+        )
+
+        vector = VectorSchema(
+            "dense_vector",
+            DataType.VECTOR_FP32,
+            dimension=128,
+            index_param=HnswIndexParam(),
+        )
+
+        collection_schema = zvec.CollectionSchema(
+            name="test_single_dense_vector_non_list",
+            fields=field,
+            vectors=vector,  # Non-list form
+        )
+
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        check_collection_info(
+            coll, collection_schema, collection_option, collection_temp_dir
+        )
+        check_collection_basic(coll)
+        coll.destroy()
+
+    def test_collection_concurrent_create(
+        self, collection_temp_dir, basic_schema, collection_option
+    ):
+        results = []
+        errors = []
+        lock = threading.Lock()
+
+        # Function to be executed by each thread
+        def create_collection_thread(thread_id):
+            try:
+                coll = zvec.create_and_open(
+                    path=collection_temp_dir,
+                    schema=basic_schema,
+                    option=collection_option,
+                )
+                with lock:
+                    results.append((thread_id, coll))
+            except Exception as e:
+                with lock:
+                    errors.append((thread_id, str(e)))
+
+        threads = []
+        for i in range(5):
+            thread = threading.Thread(target=create_collection_thread, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        for thread in threads:
+            thread.join()
+        assert len(results) == 1, (
+            f"Expected exactly one successful creation, but got {len(results)}"
+        )
+        assert len(errors) == 4, (
+            f"Expected exactly four failures, but got {len(errors)}"
+        )
+
+        successful_thread_id, successful_collection = results[0]
+        assert successful_collection is not None, (
+            "Successful creation should return a valid collection"
+        )
+        assert successful_collection.path == collection_temp_dir, (
+            "Collection path mismatch"
+        )
+
+    def test_create_open_loop(
+        self, collection_temp_dir, collection_option, full_schema
+    ):
+        for cycle in range(10):
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=full_schema,
+                option=collection_option,
+            )
+            assert coll is not None, (
+                f"Failed to create and open collection in cycle {cycle}"
+            )
+            assert coll.path == collection_temp_dir, (
+                f"Collection path mismatch in cycle {cycle}"
+            )
+
+            del coll
+
+            reopened_coll = zvec.open(
+                path=collection_temp_dir, option=collection_option
+            )
+            assert reopened_coll is not None, (
+                f"Failed to reopen collection in cycle {cycle}"
+            )
+            assert reopened_coll.path == collection_temp_dir, (
+                f"Reopened collection path mismatch in cycle {cycle}"
+            )
+
+            check_collection_full(reopened_coll)
+
+            reopened_coll.destroy()
+
+    @pytest.mark.parametrize(
+        "data_type, index_param", VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
+    )
+    def test_valid_vector_index_params(
+        self,
+        data_type,
+        index_param,
+        single_vector_schema_with_index_param,
+        collection_temp_dir,
+        collection_option,
+    ):
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=single_vector_schema_with_index_param,
+            option=collection_option,
+        )
+
+        check_collection_info(
+            coll,
+            single_vector_schema_with_index_param,
+            collection_option,
+            collection_temp_dir,
+        )
+
+        check_collection_basic(coll, True)
+
+    @pytest.mark.parametrize(
+        "data_type, index_param", INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
+    )
+    def test_invalid_vector_index_params(
+        self,
+        data_type,
+        index_param,
+        single_vector_schema_with_index_param,
+        collection_temp_dir,
+        collection_option,
+    ):
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=single_vector_schema_with_index_param,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    def test_open_concurrent_same_path(self, tmp_path_factory, collection_option):
+        """Test concurrent opening of the same collection path.
+
+        - Multi-threading concurrency: 5 threads simultaneously open the same collection
+        - Result verification: Verify that only one can open successfully, others must fail
+        """
+        # Create a temporary directory and path for the collection
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "concurrent_open_test_collection"
+
+        # First, create a collection that we'll try to open concurrently
+        field_list = [
+            FieldSchema(
+                "id",
+                DataType.INT64,
+                nullable=False,
+                index_param=InvertIndexParam(enable_range_optimization=True),
+            ),
+            FieldSchema(
+                "name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
+            ),
+        ]
+
+        vector_list = [
+            VectorSchema(
+                "dense_vector",
+                DataType.VECTOR_FP32,
+                dimension=128,
+                index_param=HnswIndexParam(),
+            )
+        ]
+
+        collection_schema = zvec.CollectionSchema(
+            name="concurrent_open_test_collection",
+            fields=field_list,
+            vectors=vector_list,
+        )
+
+        # Create the collection first
+        coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        # Close the collection so we can test opening it
+        if hasattr(coll, "close") and coll is not None:
+            coll.close()
+
+        # Shared variables to collect results from threads
+        results = []
+        errors = []
+
+        # Lock for thread-safe operations
+        lock = threading.Lock()
+        # Clean up the created collection reference
+        del coll
+
+        # Function to be executed by each thread
+        def open_collection_thread(thread_id):
+            try:
+                reopened_coll = zvec.open(
+                    path=str(collection_path), option=collection_option
+                )
+                with lock:
+                    results.append((thread_id, reopened_coll))
+                # Clean up the collection if opened successfully
+                if hasattr(reopened_coll, "close") and reopened_coll is not None:
+                    reopened_coll.close()
+            except Exception as e:
+                with lock:
+                    errors.append((thread_id, str(e)))
+
+        # Create and start 5 threads
+        threads = []
+        for i in range(5):
+            thread = threading.Thread(target=open_collection_thread, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Verify results:
+        # 1. Only one open should succeed (exactly one collection in results)
+        # 2. Others should fail (4 errors in errors)
+        assert len(results) == 1, (
+            f"Expected exactly one successful open, but got {len(results)}"
+        )
+        assert len(errors) == 4, (
+            f"Expected exactly four failures, but got {len(errors)}"
+        )
+
+        # Additional verification: check that the successful open has a valid collection
+        successful_thread_id, successful_collection = results[0]
+        assert successful_collection is not None, (
+            "Successful open should return a valid collection"
+        )
+        assert successful_collection.path == str(collection_path), (
+            "Collection path mismatch"
+        )
+
+    @pytest.mark.parametrize("read_only,enable_mmap", valid_collection_options)
+    def test_valid_option(
+        self, collection_temp_dir, basic_schema, read_only, enable_mmap
+    ):
+        option = CollectionOption(read_only=read_only, enable_mmap=enable_mmap)
+
+        coll = zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=basic_schema,
+            option=option,
+        )
+
+        check_collection_info(coll, basic_schema, option, collection_temp_dir)
+        check_collection_basic(coll)
+
+        coll.destroy()
+
+    def test_valid_none_option(self, collection_temp_dir, basic_schema):
+        zvec.create_and_open(
+            path=collection_temp_dir,
+            schema=basic_schema,
+            option=None,
+        )
+
+    @pytest.mark.parametrize("read_only,enable_mmap", invalid_collection_options)
+    def test_invalid_option(
+        self, collection_temp_dir, basic_schema, read_only, enable_mmap
+    ):
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=basic_schema,
+                option=CollectionOption(read_only=read_only, enable_mmap=enable_mmap),
+            )
+
+        assert CREATE_READ_ONLY_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize(
+        "field_name1,field_name2,vector_name1,vector_name2",
+        duplicate_names_test,
+    )
+    def test_duplicate_field_names(
+        self,
+        collection_temp_dir,
+        collection_option,
+        field_name1,
+        field_name2,
+        vector_name1,
+        vector_name2,
+    ):
+        with pytest.raises(Exception) as exc_info:
+            collection_schema = zvec.CollectionSchema(
+                name="test_collection",
+                fields=[
+                    FieldSchema(
+                        field_name1,
+                        DataType.INT64,
+                        nullable=False,
+                        index_param=InvertIndexParam(enable_range_optimization=True),
+                    ),
+                    FieldSchema(
+                        field_name2,
+                        DataType.INT64,
+                        nullable=False,
+                        index_param=InvertIndexParam(enable_range_optimization=True),
+                    ),
+                ],
+                vectors=[
+                    VectorSchema(
+                        vector_name1,
+                        DataType.VECTOR_FP32,
+                        dimension=128,
+                        index_param=HnswIndexParam(),
+                    ),
+                    VectorSchema(
+                        vector_name2,
+                        DataType.VECTOR_FP32,
+                        dimension=128,
+                        index_param=HnswIndexParam(),
+                    ),
+                ],
+            )
+
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize("long_name", long_names)
+    def test_invalid_long_field_names(
+        self, collection_option, collection_temp_dir, long_name
+    ):
+        collection_schema = zvec.CollectionSchema(
+            name=long_name,
+            fields=[
+                FieldSchema(
+                    long_name + "_field",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    long_name + "_vector",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    def test_invalid_empty_fields_and_vectors(
+        self, collection_temp_dir, collection_option
+    ):
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[],  # Empty fields
+            vectors=[],  # Empty vectors
+        )
+
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=collection_temp_dir,
+                schema=collection_schema,
+                option=collection_option,
+            )
+
+        assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
+
+    @pytest.mark.parametrize("valid_path", valid_path_list)
+    def test_valid_path(self, basic_schema, collection_option, valid_path):
+        if os.path.exists(valid_path):
+            import shutil
+
+            shutil.rmtree(valid_path)
+
+        coll = zvec.create_and_open(
+            path=valid_path, schema=basic_schema, option=collection_option
+        )
+
+        check_collection_info(coll, basic_schema, collection_option, valid_path)
+
+        coll.destroy()
+
+    @pytest.mark.parametrize("invalid_path", invalid_path_list)
+    def test_invalid_path(self, basic_schema, collection_option, invalid_path):
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=invalid_path, schema=basic_schema, option=collection_option
+            )
+
+        assert INVALID_PATH_ERROR_MSG in str(exc_info.value), str(exc_info.value)

python/tests/detail/test_collection_exception.py

@@ -0,0 +1,328 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import pytest
+import numpy as np
+import zvec
+
+from zvec import (
+    CollectionOption,
+    InvertIndexParam,
+    HnswIndexParam,
+    DataType,
+    Collection,
+    Doc,
+    FieldSchema,
+    VectorSchema,
+    VectorQuery,
+)
+
+
+class TestCollectionExceptionHandling:
+    @pytest.fixture(scope="function")
+    def test_collection(self, tmp_path_factory):
+        """Fixture to create a test collection"""
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+                FieldSchema("weight", DataType.FLOAT, nullable=True),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                ),
+                VectorSchema(
+                    "sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
+                ),
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert coll is not None, "Failed to create and open collection"
+
+        yield coll
+
+        # Clean up
+        if hasattr(coll, "destroy") and coll is not None:
+            try:
+                coll.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+    def test_create_and_open_missing_path(self, tmp_path_factory):
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                schema=collection_schema, option=collection_option
+            )
+        assert exc_info.value is not None, (
+            "Expected exception for missing path parameter"
+        )
+
+    def test_create_and_open_missing_schema(self, tmp_path_factory):
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.create_and_open(
+                path=str(collection_path), option=collection_option
+            )
+        assert exc_info.value is not None, (
+            "Expected exception for missing schema parameter"
+        )
+
+    def test_open_missing_path(self):
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        with pytest.raises(Exception) as exc_info:
+            coll = zvec.open(option=collection_option)
+        assert exc_info.value is not None, (
+            "Expected exception for missing path parameter"
+        )
+
+    def test_insert_missing_docs(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.insert()
+        assert exc_info.value is not None, (
+            "Expected exception for missing docs parameter"
+        )
+
+    def test_update_missing_docs(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.update()
+        assert exc_info.value is not None, (
+            "Expected exception for missing docs parameter"
+        )
+
+    def test_upsert_missing_docs(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.upsert()
+        assert exc_info.value is not None, (
+            "Expected exception for missing docs parameter"
+        )
+
+    def test_delete_missing_ids(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.delete()
+        assert exc_info.value is not None, (
+            "Expected exception for missing ids parameter"
+        )
+
+    def test_fetch_missing_ids(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.fetch()
+        assert exc_info.value is not None, (
+            "Expected exception for missing ids parameter"
+        )
+
+    def test_query_missing_vectorquery_field_name(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            result = test_collection.query(vectors=[VectorQuery()])
+        assert exc_info.value is not None, (
+            "Expected exception for missing VectorQuery field_name parameter"
+        )
+
+    def test_add_column_missing_field_schema(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            test_collection.add_column()
+        assert exc_info.value is not None, (
+            "Expected exception for missing field_schema parameter"
+        )
+
+    def test_alter_column_missing_old_name(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            test_collection.alter_column(new_name="new_name")
+        assert exc_info.value is not None, (
+            "Expected exception for missing old_name parameter"
+        )
+
+    def test_alter_column_missing_new_name(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            test_collection.alter_column(old_name="old_name")
+        assert exc_info.value is not None, (
+            "Expected exception for missing new_name parameter"
+        )
+
+    def test_drop_column_missing_field_name(self, test_collection: Collection):
+        with pytest.raises(Exception) as exc_info:
+            test_collection.drop_column()
+        assert exc_info.value is not None, (
+            "Expected exception for missing field_name parameter"
+        )
+
+    def test_invalid_parameter_types(self, test_collection: Collection):
+        # This test depends on specific implementation details
+        # Generally, we would expect TypeErrors or similar exceptions
+        pass
+
+    def test_missing_required_parameters(self, test_collection: Collection):
+        # This test depends on specific implementation details
+        # Generally, we would expect TypeErrors or similar exceptions
+        pass
+
+    def test_empty_collection_operations(self, tmp_path_factory):
+        collection_schema = zvec.CollectionSchema(
+            name="empty_test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "empty_test_collection"
+
+        coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert coll is not None, "Failed to create and open collection"
+
+        # Test fetch on empty collection
+        result = coll.fetch(["1"])
+        assert len(result) >= 0  # May be empty or have special handling
+
+        # Test query on empty collection
+        result = coll.query()
+        assert len(result) == 0
+
+        # Test update on empty collection
+        doc = Doc(
+            id="1",
+            fields={"id": 1, "name": "test"},
+            vectors={"dense": np.random.random(128).tolist()},
+        )
+
+        result = coll.update(doc)
+        # Should handle gracefully, possibly with NOT_FOUND status
+
+        # Clean up
+        if hasattr(coll, "destroy") and coll is not None:
+            try:
+                coll.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+    def test_resource_management(self, test_collection: Collection):
+        doc = Doc(
+            id="1",
+            fields={"id": 1, "name": "test", "weight": 80.5},
+            vectors={
+                "dense": np.random.random(128).tolist(),
+                "sparse": {1: 1.0, 2: 2.0},
+            },
+        )
+
+        # Insert
+        result = test_collection.insert(doc)
+        assert result.ok()
+
+        # Fetch
+        result = test_collection.fetch(["1"])
+        assert len(result) == 1
+
+        # Query
+        result = test_collection.query()
+        assert len(result) >= 0
+
+        # Update
+        result = test_collection.update(doc)
+        assert result.ok()
+
+        # Delete
+        result = test_collection.delete("1")
+        assert result.ok()
+
+    def test_exception_resource_cleanup(self, test_collection: Collection):
+        # This test would need to simulate exception conditions
+        # which is difficult without specific failure injection points
+        pass

python/tests/detail/test_collection_open.py

@@ -0,0 +1,960 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import threading
+import numpy as np
+
+from fixture_helper import *
+
+COLLECTION_OPTION_TEST_CASES_VALID = [
+    # (read_only, enable_mmap, description)
+    (False, True, "Read-write with mmap enabled"),
+    (False, False, "Read-write with mmap disabled"),
+    (True, True, "Read-only with mmap enabled"),
+    (True, False, "Read-only with mmap disabled"),
+]
+
+# Test data for invalid paths
+INVALID_PATH_LIST = [
+    "/nonexistent/directory/test_collection",
+    "invalid:path",
+    "",  # Empty path
+]
+
+
+@pytest.fixture(scope="session")
+def collection_schema():
+    return zvec.CollectionSchema(
+        name="test_collection",
+        fields=[
+            FieldSchema(
+                "id",
+                DataType.INT64,
+                nullable=False,
+                index_param=InvertIndexParam(enable_range_optimization=True),
+            ),
+            FieldSchema(
+                "name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
+            ),
+            FieldSchema(
+                "weight", DataType.FLOAT, nullable=False, index_param=InvertIndexParam()
+            ),
+        ],
+        vectors=[
+            VectorSchema(
+                "dense",
+                DataType.VECTOR_FP32,
+                dimension=128,
+                index_param=HnswIndexParam(),
+            ),
+            VectorSchema(
+                "sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
+            ),
+        ],
+    )
+
+
+@pytest.fixture
+def single_doc():
+    id = 0
+    return Doc(
+        id=f"{id}",
+        fields={"id": id, "name": "test"},
+        vectors={
+            "dense": [id + 0.1] * 128,
+        },
+    )
+
+
+@pytest.fixture(scope="function")
+def test_collection(
+    tmp_path_factory, collection_schema, collection_option
+) -> Generator[Any, Any, Collection]:
+    temp_dir = tmp_path_factory.mktemp("zvec")
+    collection_path = temp_dir / "test_collection"
+
+    coll = zvec.create_and_open(
+        path=str(collection_path), schema=collection_schema, option=collection_option
+    )
+
+    assert coll is not None, "Failed to create and open collection"
+    assert coll.path == str(collection_path)
+    assert coll.schema.name == collection_schema.name
+    assert list(coll.schema.fields) == list(collection_schema.fields)
+    assert list(coll.schema.vectors) == list(collection_schema.vectors)
+    assert coll.option.read_only == collection_option.read_only
+    assert coll.option.enable_mmap == collection_option.enable_mmap
+
+    try:
+        yield coll
+    finally:
+        if hasattr(coll, "destroy") and coll is not None:
+            try:
+                coll.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+
+class TestCollectionOpen:
+    def test_open_basic_functionality(
+        self, tmp_path_factory, collection_schema, collection_option
+    ):
+        import sys
+        import time
+        import os
+
+        # Create unique temp directory
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        # Ensure the path exists
+        collection_path_str = str(collection_path)
+        print(f"DEBUG: Collection path: {collection_path_str}")
+        print(f"DEBUG: Temp directory exists: {temp_dir.exists()}")
+
+        # Create and open collection first
+        created_coll = zvec.create_and_open(
+            path=collection_path_str, schema=collection_schema, option=collection_option
+        )
+
+        assert created_coll is not None, (
+            f"Failed to create collection, returned None instead of valid Collection object. Path: {collection_path_str}"
+        )
+        assert created_coll.path == collection_path_str, (
+            f"Collection path mismatch. Expected: {collection_path_str}, Actual: {created_coll.path}"
+        )
+        assert created_coll.schema.name == "test_collection", (
+            f"Collection schema name mismatch. Expected: test_collection, Actual: {created_coll.schema.name}"
+        )
+
+        # Insert multiple documents to verify persistence
+        docs = []
+        for i in range(3):
+            doc = Doc(
+                id=f"{i}",
+                fields={"id": i, "name": f"test_{i}", "weight": float(i * 10)},
+                vectors={
+                    "dense": [float(j + i) for j in range(128)],
+                    "sparse": {j: float(j + i) for j in range(5)},
+                },
+            )
+            docs.append(doc)
+
+        result = created_coll.insert(docs)
+        assert len(result) == 3, f"Expected 3 insertion results, but got {len(result)}"
+        for i, res in enumerate(result):
+            assert res.ok(), (
+                f"Insertion result {i} is not OK. Status code: {res.code()}, Message: {res.message()}"
+            )
+
+        # Verify documents were inserted using fetch interface
+        fetched_docs_after_insert = created_coll.fetch(["0", "1", "2"])
+        assert len(fetched_docs_after_insert) == 3, (
+            f"Expected 3 fetched documents after insertion, but got {len(fetched_docs_after_insert)}"
+        )
+        assert "0" in fetched_docs_after_insert, (
+            "Document with ID '0' not found in fetched results after insertion"
+        )
+        assert "1" in fetched_docs_after_insert, (
+            "Document with ID '1' not found in fetched results after insertion"
+        )
+        assert "2" in fetched_docs_after_insert, (
+            "Document with ID '2' not found in fetched results after insertion"
+        )
+
+        # Verify fetched document content after insertion
+        for i in range(3):
+            doc = fetched_docs_after_insert[f"{i}"]
+            assert doc is not None, (
+                f"Fetched document with ID '{i}' is None after insertion"
+            )
+            assert doc.id == f"{i}", (
+                f"Document ID mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.id}"
+            )
+            assert doc.field("id") == i, (
+                f"Document id field mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.field('id')}"
+            )
+            assert doc.field("name") == f"test_{i}", (
+                f"Document name field mismatch for document '{i}' after insertion. Expected: test_{i}, Actual: {doc.field('name')}"
+            )
+            assert doc.field("weight") == float(i * 10), (
+                f"Document weight field mismatch for document '{i}' after insertion. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
+            )
+
+            # Verify vector access after insertion
+            assert doc.vector("dense") is not None, (
+                f"Document {i} should have dense vector after insertion"
+            )
+            assert doc.vector("sparse") is not None, (
+                f"Document {i} should have sparse vector after insertion"
+            )
+
+            # Verify vector types after insertion
+            assert isinstance(doc.vector("dense"), list), (
+                f"Document {i} dense vector should be dict after insertion, got {type(doc.vector('dense'))}"
+            )
+            assert isinstance(doc.vector("sparse"), dict), (
+                f"Document {i} sparse vector should be dict after insertion, got {type(doc.vector('sparse'))}"
+            )
+
+        # Verify documents were inserted using stats
+        stats = created_coll.stats
+        assert stats is not None, "Collection stats should not be None"
+        assert stats.doc_count == 3, (
+            f"Document count mismatch after insertion. Expected: 3, Actual: {stats.doc_count}"
+        )
+
+        # Store the collection path before cleanup
+        collection_path = created_coll.path
+
+        # Clean up the created collection reference
+        del created_coll
+
+        # Wait and verify the path still exists
+        print(f"DEBUG: Collection path after destroy: {collection_path}")
+        print(f"DEBUG: Path exists after destroy: {os.path.exists(collection_path)}")
+
+        # Now open the existing collection
+        try:
+            print(f"DEBUG: Path exists before open: {os.path.exists(collection_path)}")
+
+            # List contents of parent directory for debugging
+            parent_dir = os.path.dirname(collection_path)
+            if os.path.exists(parent_dir):
+                print(f"DEBUG: Parent directory contents: {os.listdir(parent_dir)}")
+
+            opened_coll = zvec.open(path=collection_path, option=collection_option)
+
+            assert opened_coll is not None, (
+                f"Failed to open existing collection at path: {collection_path}. Returned None instead of valid Collection object"
+            )
+            assert opened_coll.path == collection_path, (
+                f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
+            )
+            assert opened_coll.schema.name == "test_collection", (
+                f"Opened collection schema name mismatch. Expected: test_collection, Actual: {opened_coll.schema.name}"
+            )
+
+            # Check reference count of opened collection
+            opened_ref_count = sys.getrefcount(opened_coll)
+            print(f"DEBUG: Reference count of opened collection: {opened_ref_count}")
+
+            # Verify data persistence
+            # Verify data persistence using fetch interface
+            fetched_docs = opened_coll.fetch(["0", "1", "2"])
+            assert len(fetched_docs) == 3, (
+                f"Expected 3 fetched documents after reopening, but got {len(fetched_docs)}"
+            )
+            assert "0" in fetched_docs, (
+                "Document with ID '0' not found in fetched results after reopening"
+            )
+            assert "1" in fetched_docs, (
+                "Document with ID '1' not found in fetched results after reopening"
+            )
+            assert "2" in fetched_docs, (
+                "Document with ID '2' not found in fetched results after reopening"
+            )
+
+            # Verify fetched document content after reopening collection
+            for i in range(3):
+                doc = fetched_docs[f"{i}"]
+                assert doc is not None, (
+                    f"Fetched document with ID '{i}' is None after reopening collection"
+                )
+                assert doc.id == f"{i}", (
+                    f"Document ID mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.id}"
+                )
+                assert doc.field("id") == i, (
+                    f"Document id field mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.field('id')}"
+                )
+                assert doc.field("name") == f"test_{i}", (
+                    f"Document name field mismatch for document '{i}' after reopening. Expected: test_{i}, Actual: {doc.field('name')}"
+                )
+                assert doc.field("weight") == float(i * 10), (
+                    f"Document weight field mismatch for document '{i}' after reopening. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
+                )
+
+                # Verify vector access after reopening
+                assert doc.vector("dense") is not None, (
+                    f"Document {i} should have dense vector after reopening"
+                )
+                assert doc.vector("sparse") is not None, (
+                    f"Document {i} should have sparse vector after reopening"
+                )
+
+                # Verify vector types after reopening
+                assert isinstance(doc.vector("dense"), list), (
+                    f"Document {i} dense vector should be dict after reopening, got {type(doc.vector('dense'))}"
+                )
+                assert isinstance(doc.vector("sparse"), dict), (
+                    f"Document {i} sparse vector should be dict after reopening, got {type(doc.vector('sparse'))}"
+                )
+
+                # Verify score attribute exists
+                assert hasattr(doc, "score"), (
+                    f"Document {i} should have a score attribute after reopening"
+                )
+                assert isinstance(doc.score, (int, float)), (
+                    f"Document {i} score should be numeric after reopening, got {type(doc.score)}"
+                )
+                # For fetch operations, score is typically 0.0
+                assert doc.score == 0.0, (
+                    f"Document {i} score should be 0.0 for fetch operation after reopening, but got {doc.score}"
+                )
+
+            # Test query functionality
+            query_result = opened_coll.query(include_vector=True)
+            assert len(query_result) == 3, (
+                f"Expected 3 query results, but got {len(query_result)}"
+            )
+
+            # Verify query results have proper structure and content with detailed validation
+            returned_doc_ids = set()
+            for doc in query_result:
+                # Verify basic document structure
+                assert doc.id is not None, f"Query result document should have an ID"
+                assert doc.id in ["0", "1", "2"], (
+                    f"Query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
+                )
+                returned_doc_ids.add(doc.id)
+
+                # Verify field access
+                assert doc.field("id") is not None, (
+                    f"Document {doc.id} should have id field"
+                )
+                assert doc.field("name") is not None, (
+                    f"Document {doc.id} should have name field"
+                )
+                assert doc.field("weight") is not None, (
+                    f"Document {doc.id} should have weight field"
+                )
+
+                # Verify field values
+                expected_id = int(doc.id)
+                assert doc.field("id") == expected_id, (
+                    f"Document {doc.id} id field mismatch. Expected: {expected_id}, Actual: {doc.field('id')}"
+                )
+                assert doc.field("name") == f"test_{expected_id}", (
+                    f"Document {doc.id} name field mismatch. Expected: test_{expected_id}, Actual: {doc.field('name')}"
+                )
+                assert doc.field("weight") == float(expected_id * 10), (
+                    f"Document {doc.id} weight field mismatch. Expected: {float(expected_id * 10)}, Actual: {doc.field('weight')}"
+                )
+
+                # Verify vector access
+                assert doc.vector("dense") is not None, (
+                    f"Document {doc.id} should have dense vector"
+                )
+                assert doc.vector("sparse") is not None, (
+                    f"Document {doc.id} should have sparse vector"
+                )
+
+                # Verify vector types
+                assert isinstance(doc.vector("dense"), list), (
+                    f"Document {doc.id} dense vector should be list, got {type(doc.vector('dense'))}"
+                )
+                assert isinstance(doc.vector("sparse"), dict), (
+                    f"Document {doc.id} sparse vector should be dict, got {type(doc.vector('sparse'))}"
+                )
+
+                # Verify score attribute exists
+                assert hasattr(doc, "score"), (
+                    f"Document {doc.id} should have a score attribute"
+                )
+                assert isinstance(doc.score, (int, float)), (
+                    f"Document {doc.id} score should be numeric, got {type(doc.score)}"
+                )
+
+            # Verify all expected documents are returned
+            expected_doc_ids = {"0", "1", "2"}
+            assert returned_doc_ids == expected_doc_ids, (
+                f"Query should return all expected documents. Expected: {expected_doc_ids}, Actual: {returned_doc_ids}"
+            )
+
+            # === Enhanced validation based on test_collection_dql_operations.py ===
+
+            # Verify vector field names accessibility for all documents
+            for doc in query_result:
+                vector_names = doc.vector_names()
+                expected_vector_names = {"dense", "sparse"}
+                assert set(vector_names) == expected_vector_names, (
+                    f"Document {doc.id} vector names mismatch. Expected: {expected_vector_names}, Actual: {set(vector_names)}"
+                )
+
+                # Verify all vector fields can be accessed
+                for vector_name in expected_vector_names:
+                    vector_data = doc.vector(vector_name)
+                    assert vector_data is not None, (
+                        f"Document {doc.id} should have accessible vector '{vector_name}'"
+                    )
+                    if vector_name == "dense":
+                        assert isinstance(vector_data, list), (
+                            f"Document {doc.id} vector '{vector_name}' should be list, got {type(vector_data)}"
+                        )
+                    else:
+                        assert isinstance(vector_data, dict), (
+                            f"Document {doc.id} vector '{vector_name}' should be dict, got {type(vector_data)}"
+                        )
+
+            # Test query with filter
+            filtered_result = opened_coll.query(filter="id >= 1", include_vector=True)
+            assert len(filtered_result) == 2, (
+                f"Expected 2 filtered query results (id >= 1), but got {len(filtered_result)}"
+            )
+
+            # Verify filtered query results
+            filtered_doc_ids = set()
+            for doc in filtered_result:
+                assert doc.id is not None, (
+                    f"Filtered query result document should have an ID"
+                )
+                assert doc.id in ["1", "2"], (
+                    f"Filtered query result document ID should be one of ['1', '2'], but got {doc.id}"
+                )
+                filtered_doc_ids.add(doc.id)
+
+                # Verify filter condition is satisfied
+                doc_id = int(doc.id)
+                assert doc_id >= 1, (
+                    f"Document {doc.id} should satisfy filter condition id >= 1"
+                )
+
+                # Verify document structure
+                assert doc.field("id") is not None, (
+                    f"Document {doc.id} should have id field"
+                )
+                assert doc.field("name") is not None, (
+                    f"Document {doc.id} should have name field"
+                )
+                assert doc.field("weight") is not None, (
+                    f"Document {doc.id} should have weight field"
+                )
+
+                # Verify field values
+                assert doc.field("id") == doc_id, (
+                    f"Document {doc.id} id field mismatch. Expected: {doc_id}, Actual: {doc.field('id')}"
+                )
+                assert doc.field("name") == f"test_{doc_id}", (
+                    f"Document {doc.id} name field mismatch. Expected: test_{doc_id}, Actual: {doc.field('name')}"
+                )
+                assert doc.field("weight") == float(doc_id * 10), (
+                    f"Document {doc.id} weight field mismatch. Expected: {float(doc_id * 10)}, Actual: {doc.field('weight')}"
+                )
+
+                # Verify vector access
+                assert doc.vector("dense") is not None, (
+                    f"Document {doc.id} should have dense vector"
+                )
+                assert doc.vector("sparse") is not None, (
+                    f"Document {doc.id} should have sparse vector"
+                )
+
+                # Verify score attribute exists
+                assert hasattr(doc, "score"), (
+                    f"Document {doc.id} should have a score attribute"
+                )
+                assert isinstance(doc.score, (int, float)), (
+                    f"Document {doc.id} score should be numeric, got {type(doc.score)}"
+                )
+
+            # Verify filtered documents
+            expected_filtered_ids = {"1", "2"}
+            assert filtered_doc_ids == expected_filtered_ids, (
+                f"Filtered query should return expected documents. Expected: {expected_filtered_ids}, Actual: {filtered_doc_ids}"
+            )
+
+            # Test vector query functionality for dense vectors
+            query_vector_dense = [0.1] * 128
+            vector_query_result = opened_coll.query(
+                VectorQuery(field_name="dense", vector=query_vector_dense)
+            )
+            assert len(vector_query_result) > 0, (
+                f"Expected at least 1 vector query result, but got {len(vector_query_result)}"
+            )
+
+            # Verify vector query results structure
+            for doc in vector_query_result[:3]:  # Check first 3 results
+                assert doc.id is not None, (
+                    f"Vector query result document should have an ID"
+                )
+                assert doc.id in ["0", "1", "2"], (
+                    f"Vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
+                )
+
+                # Verify document structure
+                assert doc.field("id") is not None, (
+                    f"Document {doc.id} should have id field"
+                )
+                assert doc.field("name") is not None, (
+                    f"Document {doc.id} should have name field"
+                )
+                assert doc.field("weight") is not None, (
+                    f"Document {doc.id} should have weight field"
+                )
+
+                # Verify vector access
+                assert doc.vector("dense") is not None, (
+                    f"Document {doc.id} should have dense vector"
+                )
+                assert doc.vector("sparse") is not None, (
+                    f"Document {doc.id} should have sparse vector"
+                )
+
+                # Verify score attribute exists and is numeric
+                assert hasattr(doc, "score"), (
+                    f"Document {doc.id} should have a score attribute"
+                )
+                assert isinstance(doc.score, (int, float)), (
+                    f"Document {doc.id} score should be numeric, got {type(doc.score)}"
+                )
+
+                # For dense vector queries, score should typically be non-negative (depending on metric)
+                # Note: This may vary based on the metric type used
+                assert doc.score >= 0 or doc.score < 0, (
+                    f"Document {doc.id} score should be a valid number"
+                )
+
+            # Test vector query functionality for sparse vectors
+            query_vector_sparse = {1: 1.0, 2: 2.0, 3: 3.0}
+            sparse_vector_query_result = opened_coll.query(
+                VectorQuery(field_name="sparse", vector=query_vector_sparse)
+            )
+            assert len(sparse_vector_query_result) > 0, (
+                f"Expected at least 1 sparse vector query result, but got {len(sparse_vector_query_result)}"
+            )
+
+            # Verify sparse vector query results structure
+            for doc in sparse_vector_query_result[:3]:  # Check first 3 results
+                assert doc.id is not None, (
+                    f"Sparse vector query result document should have an ID"
+                )
+                assert doc.id in ["0", "1", "2"], (
+                    f"Sparse vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
+                )
+
+                # Verify document structure
+                assert doc.field("id") is not None, (
+                    f"Document {doc.id} should have id field"
+                )
+                assert doc.field("name") is not None, (
+                    f"Document {doc.id} should have name field"
+                )
+                assert doc.field("weight") is not None, (
+                    f"Document {doc.id} should have weight field"
+                )
+
+                # Verify vector access
+                assert doc.vector("dense") is not None, (
+                    f"Document {doc.id} should have dense vector"
+                )
+                assert doc.vector("sparse") is not None, (
+                    f"Document {doc.id} should have sparse vector"
+                )
+
+                # Verify score attribute exists and is numeric
+                assert hasattr(doc, "score"), (
+                    f"Document {doc.id} should have a score attribute"
+                )
+                assert isinstance(doc.score, (int, float)), (
+                    f"Document {doc.id} score should be numeric, got {type(doc.score)}"
+                )
+
+            # Clean up
+            if hasattr(opened_coll, "destroy") and opened_coll is not None:
+                opened_coll.destroy()
+                print("DEBUG: Opened collection destroyed successfully")
+
+        except Exception as e:
+            logging.error("Exception occurred: [{}]".format(e))
+            raise e
+
+    @pytest.mark.parametrize(
+        "read_only,enable_mmap,description", COLLECTION_OPTION_TEST_CASES_VALID
+    )
+    @pytest.mark.parametrize("createAndopen_enable_mmap", [True, False])
+    def test_open_with_different_collection_options_valid(
+        self,
+        tmp_path_factory,
+        createAndopen_enable_mmap,
+        read_only,
+        enable_mmap,
+        description,
+        collection_schema,
+    ):
+        # Create collection with initial option
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        initial_option = CollectionOption(
+            read_only=False, enable_mmap=createAndopen_enable_mmap
+        )
+
+        # Create and open collection first
+        created_coll = zvec.create_and_open(
+            path=str(collection_path), schema=collection_schema, option=initial_option
+        )
+
+        assert created_coll is not None, "Failed to create collection"
+
+        # Clean up the created collection reference
+        del created_coll
+
+        # Now open with different options
+        collection_option = CollectionOption(
+            read_only=read_only, enable_mmap=enable_mmap
+        )
+
+        try:
+            opened_coll = zvec.open(path=str(collection_path), option=collection_option)
+
+            assert opened_coll is not None, (
+                f"Failed to open collection with option: {description}. Returned None instead of valid Collection object. Path: {collection_path}"
+            )
+            assert opened_coll.path == str(collection_path), (
+                f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
+            )
+            assert opened_coll.schema.name == collection_schema.name, (
+                f"Opened collection schema name mismatch. Expected: {collection_schema.name}, Actual: {opened_coll.schema.name}"
+            )
+            assert opened_coll.option.read_only == read_only, (
+                f"Opened collection read_only option mismatch. Expected: {read_only}, Actual: {opened_coll.option.read_only}"
+            )
+            assert opened_coll.option.enable_mmap == createAndopen_enable_mmap, (
+                f"Opened collection mmap option mismatch. Expected: {createAndopen_enable_mmap}, Actual: {opened_coll.option.enable_mmap}"
+            )
+
+            # Clean up
+            if (
+                hasattr(opened_coll, "destroy")
+                and opened_coll is not None
+                and read_only == False
+            ):
+                opened_coll.destroy()
+
+        except Exception as e:
+            logging.error("Exception occurred: [{}]".format(e))
+            pytest.fail(f"Failed to open collection with different options: {e}")
+
+    def test_open_with_none_option(self, tmp_path_factory, collection_schema):
+        # Create collection
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        initial_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        # Create and open collection first
+        created_coll = zvec.create_and_open(
+            path=str(collection_path), schema=collection_schema, option=initial_option
+        )
+
+        assert created_coll is not None, (
+            f"Failed to create collection. Returned None instead of valid Collection object. Path: {collection_path}"
+        )
+
+        # Clean up the created collection reference
+        del created_coll
+
+        # Now open with None option
+        with pytest.raises(Exception) as exc_info:
+            zvec.open(path=str(collection_path), option=None)
+
+        assert "incompatible function arguments" in str(exc_info.value), (
+            f"Expected 'incompatible function arguments' error, but got: {exc_info.value}"
+        )
+
+    def test_reopen_collection(self, tmp_path_factory):
+        # Prepare schema
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        # Create collection
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        # Create and open collection
+        coll1 = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert coll1 is not None, "Failed to create and open collection"
+
+        # Insert some data
+        doc = Doc(
+            id="1",
+            fields={"id": 1, "name": "test"},
+            vectors={"dense": np.random.random(128).tolist()},
+        )
+
+        result = coll1.insert(doc)
+        assert result.ok()
+
+        # Close the first collection (delete reference)
+        del coll1
+
+        # Reopen the collection
+        coll2 = zvec.open(path=str(collection_path), option=collection_option)
+
+        assert coll2 is not None, "Failed to reopen collection"
+        assert coll2.path == str(collection_path)
+        assert coll2.schema.name == collection_schema.name
+
+        # Verify data is still there
+        fetched_docs = coll2.fetch(["1"])
+        assert "1" in fetched_docs
+        fetched_doc = fetched_docs["1"]
+        assert fetched_doc.id == "1"
+        assert fetched_doc.field("name") == "test"
+
+        # Clean up
+        if hasattr(coll2, "destroy") and coll2 is not None:
+            try:
+                coll2.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+    def test_open_concurrent_same_path(self, tmp_path_factory):
+        # First create a collection
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        # Create collection path
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        # First create the collection
+        created_coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert created_coll is not None, "Failed to create collection"
+
+        # Close the collection so we can test concurrent opening
+        if hasattr(created_coll, "close") and created_coll is not None:
+            created_coll.close()
+
+        # Shared variables to collect results from threads
+        results = []
+        errors = []
+
+        # Lock for thread-safe operations
+        lock = threading.Lock()
+        # Clean up the created collection reference
+        del created_coll
+
+        # Function to be executed by each thread
+        def open_collection_thread(thread_id):
+            try:
+                coll = zvec.open(path=str(collection_path), option=collection_option)
+                with lock:
+                    results.append((thread_id, coll))
+                # Close the collection if opened successfully
+                if hasattr(coll, "close") and coll is not None:
+                    coll.close()
+            except Exception as e:
+                with lock:
+                    errors.append((thread_id, str(e)))
+
+        # Create 5 threads to call open concurrently
+        threads = []
+        for i in range(5):
+            thread = threading.Thread(target=open_collection_thread, args=(i,))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        # Verify concurrency safety: only one should succeed, others should fail
+        assert len(results) == 1, (
+            f"Expected exactly one successful open, but got {len(results)}"
+        )
+        assert len(errors) == 4, (
+            f"Expected exactly four failures, but got {len(errors)}"
+        )
+
+        # Additional verification: check that the successful open has a valid collection
+        successful_thread_id, successful_collection = results[0]
+        assert successful_collection is not None, (
+            "Successful open should return a valid collection"
+        )
+        assert successful_collection.path == str(collection_path), (
+            "Collection path mismatch"
+        )
+
+        # Clean up the successfully opened collection
+        if (
+            hasattr(successful_collection, "destroy")
+            and successful_collection is not None
+        ):
+            try:
+                successful_collection.destroy()
+            except Exception as e:
+                print(f"Warning: failed to destroy collection: {e}")
+
+    def test_open_with_corrupted_files(self, tmp_path_factory):
+        # First create a collection
+        collection_schema = zvec.CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                )
+            ],
+        )
+
+        collection_option = CollectionOption(read_only=False, enable_mmap=True)
+
+        # Create collection path
+        temp_dir = tmp_path_factory.mktemp("zvec")
+        collection_path = temp_dir / "test_collection"
+
+        # First create the collection
+        created_coll = zvec.create_and_open(
+            path=str(collection_path),
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert created_coll is not None, "Failed to create collection"
+
+        # Close the collection so we can manipulate its files
+        if hasattr(created_coll, "close") and created_coll is not None:
+            created_coll.close()
+
+        # Test case 1: Delete some files in the collection directory (simulate partial corruption)
+        import os
+        import shutil
+        import random
+
+        # Get the collection directory path
+        collection_dir = str(collection_path)
+
+        # List all files in the collection directory
+        files_in_dir = []
+        for root, dirs, files in os.walk(collection_dir):
+            for file in files:
+                files_in_dir.append(os.path.join(root, file))
+
+        # Randomly delete approximately half of the files to simulate partial corruption
+        if files_in_dir:
+            # Shuffle the list to randomly select files
+            random.shuffle(files_in_dir)
+            files_to_delete = files_in_dir[: len(files_in_dir) // 2]
+            for file_path in files_to_delete:
+                try:
+                    os.remove(file_path)
+                except Exception as e:
+                    pass  # Ignore errors during deletion
+
+        # Try to open the collection with missing files - should raise an exception
+        with pytest.raises(Exception):
+            zvec.open(path=str(collection_path), option=collection_option)
+
+        # Test case 2: Delete all files in the collection directory (simulate complete corruption)
+        # Recreate the collection
+        recreated_coll = zvec.create_and_open(
+            path=str(collection_path) + "_all",
+            schema=collection_schema,
+            option=collection_option,
+        )
+
+        assert recreated_coll is not None, "Failed to recreate collection"
+
+        # Close the collection so we can manipulate its files
+        if hasattr(recreated_coll, "close") and recreated_coll is not None:
+            recreated_coll.close()
+
+        # Delete all files in the collection directory
+        try:
+            shutil.rmtree(collection_dir)
+            os.makedirs(collection_dir)  # Recreate empty directory
+        except Exception as e:
+            pass  # Ignore errors during deletion
+
+        # Try to open the collection with missing files - should raise an exception
+        with pytest.raises(Exception):
+            zvec.open(path=str(collection_path), option=collection_option)

python/tests/detail/test_db_config.py

@@ -0,0 +1,307 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import pytest
+import tempfile
+import os
+import sys
+import subprocess
+
+import zvec
+import zvec
+from zvec import LogType, LogLevel
+
+# Error messages
+INITIALIZATION_ERROR_MSG = "initialization failed"
+RUNTIME_ERROR_MSG = "RuntimeError"
+VALUE_ERROR_MSG = "ValueError"
+TYPE_ERROR_MSG = "TypeError"
+
+
+# ==================== helper ====================
+def run_in_subprocess(func):
+    def wrapper(*args, **kwargs):
+        if os.getenv("RUNNING_IN_SUBPROCESS"):
+            return func(*args, **kwargs)
+
+        env = os.environ.copy()
+        env["RUNNING_IN_SUBPROCESS"] = "1"
+        env["PYTEST_CURRENT_TEST"] = func.__name__
+
+        import inspect
+
+        filepath = inspect.getfile(func)
+        qualname = func.__qualname__.replace(".", "::")
+        test_id = f"{filepath}::{qualname}"
+
+        project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        env["PYTHONPATH"] = project_root + ":" + env.get("PYTHONPATH", "")
+
+        cmd = [sys.executable, "-m", "pytest", "-v", "-s", test_id]
+
+        result = subprocess.run(cmd, env=env, capture_output=True, text=True)
+        if result.returncode != 0:
+            pytest.fail(
+                f"Subprocess test {func.__name__} failed with code {result.returncode}\n"
+                f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
+            )
+
+    return wrapper
+
+
+# ==================== Fixtures ====================
+@pytest.fixture(scope="function")
+def temp_log_dir(tmp_path_factory):
+    return tmp_path_factory.mktemp("logs")
+
+
+# ==================== Tests ====================
+class TestDbConfigInitialization:
+    @run_in_subprocess
+    def test_init_default(self):
+        # default config
+        # log_type: Optional[LogType] = LogType.CONSOLE,
+        # log_level: Optional[LogLevel] = LogLevel.WARN,
+        # log_dir: Optional[str] = "./logs",
+        # log_basename: Optional[str] = "zvec.log",
+        # log_file_size: Optional[int] = 2048,
+        # log_overdue_days: Optional[int] = 7,
+        zvec.init()
+
+    @run_in_subprocess
+    def test_init_file_logger(self):
+        from pathlib import Path
+        import shutil
+
+        zvec.init(
+            log_level=LogLevel.DEBUG,
+            log_type=LogType.FILE,
+        )
+        # assert logdir exist
+        log_dir = Path("./logs")
+        assert log_dir.exists()
+
+        # validate write log
+        col = zvec.create_and_open(
+            "/tmp/test/1",
+            zvec.CollectionSchema(
+                name="test",
+                vectors=zvec.VectorSchema(
+                    dimension=4,
+                    data_type=zvec.DataType.VECTOR_FP32,
+                    name="image",
+                ),
+            ),
+        )
+        col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
+        assert any(log_dir.glob("zvec.log.*"))
+
+        # clear
+        col.destroy()
+        shutil.rmtree(log_dir, ignore_errors=True)
+
+    @run_in_subprocess
+    def test_init_with_mixed_config(self):
+        zvec.init(
+            memory_limit_mb=128,
+            log_type=LogType.FILE,
+            query_threads=1,
+            log_level=LogLevel.WARN,
+        )
+
+    @run_in_subprocess
+    def test_repeated_initialization_runtime_error(self):
+        zvec.init()
+        with pytest.raises(RuntimeError):
+            zvec.init()
+
+
+class TestDbConfigMemoryLimitValidation:
+    @run_in_subprocess
+    def test_memory_limit_min_valid(self):
+        # MIN_MEMORY_LIMIT_BYTES is 100M
+        with pytest.raises(RuntimeError):
+            zvec.init(memory_limit_mb=99)
+
+    @run_in_subprocess
+    def test_memory_limit_invalid_value(self):
+        # memory_limit_mb must >= 0 and must be int and if None, set default value
+        with pytest.raises(ValueError):
+            zvec.init(memory_limit_mb=0)
+        with pytest.raises(ValueError):
+            zvec.init(memory_limit_mb=-1)
+        with pytest.raises(TypeError):
+            zvec.init(memory_limit_mb="512")
+        with pytest.raises(TypeError):
+            zvec.init(memory_limit_mb=512.5)
+
+
+class TestDbConfigThreadValidation:
+    @run_in_subprocess
+    def test_query_threads(self):
+        zvec.init(query_threads=1)
+
+    @run_in_subprocess
+    def test_query_threads_invalid(self):
+        # query_threads must >= 0 and must be int and if None, set default value
+        with pytest.raises(ValueError):
+            zvec.init(query_threads=0)
+        with pytest.raises(ValueError):
+            zvec.init(query_threads=-1)
+        with pytest.raises(TypeError):
+            zvec.init(query_threads="value")
+        with pytest.raises(TypeError):
+            zvec.init(query_threads=512.5)
+        with pytest.raises(TypeError):
+            zvec.init(query_threads="512")
+
+    @run_in_subprocess
+    def test_optimize_threads(self):
+        zvec.init(optimize_threads=1)
+
+    @run_in_subprocess
+    def test_optimize_threads_invalid(self):
+        # optimize_threads must >= 0 and must be int and if None, set default value
+        with pytest.raises(ValueError):
+            zvec.init(optimize_threads=0)
+        with pytest.raises(ValueError):
+            zvec.init(optimize_threads=-1)
+        with pytest.raises(TypeError):
+            zvec.init(optimize_threads="value")
+        with pytest.raises(TypeError):
+            zvec.init(optimize_threads=512.5)
+        with pytest.raises(TypeError):
+            zvec.init(optimize_threads="512")
+
+
+class TestDbConfigRatioValidation:
+    @run_in_subprocess
+    def test_init_invert_to_forward_scan_ratio(self):
+        # must be in [0,1]
+        zvec.init(invert_to_forward_scan_ratio=0.8)
+
+    @run_in_subprocess
+    def test_init_invert_to_forward_scan_ratio_invalid(self):
+        with pytest.raises(ValueError):
+            zvec.init(invert_to_forward_scan_ratio=1.1)
+        with pytest.raises(ValueError):
+            zvec.init(invert_to_forward_scan_ratio=-0.1)
+        with pytest.raises(TypeError):
+            zvec.init(invert_to_forward_scan_ratio="0.8")
+
+    @run_in_subprocess
+    def test_init_brute_force_by_keys_ratio(self):
+        zvec.init(brute_force_by_keys_ratio=0.8)
+
+    @run_in_subprocess
+    def test_init_brute_force_by_keys_ratio_invalid(self):
+        with pytest.raises(ValueError):
+            zvec.init(brute_force_by_keys_ratio=1.1)
+        with pytest.raises(ValueError):
+            zvec.init(brute_force_by_keys_ratio=-0.1)
+        with pytest.raises(TypeError):
+            zvec.init(brute_force_by_keys_ratio="0.8")
+
+
+class TestDbConfigLogValidation:
+    @run_in_subprocess
+    def test_log_type_valid(self):
+        zvec.init(log_type=LogType.CONSOLE)
+
+    @run_in_subprocess
+    def test_log_type_invalid(self):
+        with pytest.raises(TypeError):
+            zvec.init(log_type="FILE")
+        with pytest.raises(TypeError):
+            zvec.init(log_type="")
+        with pytest.raises(TypeError):
+            zvec.init(log_type="invalid")
+        with pytest.raises(TypeError):
+            zvec.init(log_type=123)
+
+    @run_in_subprocess
+    def test_log_level_valid(self):
+        zvec.init(log_level=LogLevel.ERROR)
+
+    @run_in_subprocess
+    def test_log_level_invalid(self):
+        with pytest.raises(TypeError):
+            zvec.init(log_level="WARN")
+        with pytest.raises(TypeError):
+            zvec.init(log_level="")
+        with pytest.raises(TypeError):
+            zvec.init(log_level="invalid")
+        with pytest.raises(TypeError):
+            zvec.init(log_level=123)
+
+    @run_in_subprocess
+    def test_init_file_logger(self):
+        from pathlib import Path
+        import shutil
+
+        temp_dir = tempfile.mkdtemp(prefix="log_test_")
+        abs_temp_dir = os.path.abspath(temp_dir)
+
+        zvec.init(
+            log_level=LogLevel.DEBUG,
+            log_type=LogType.FILE,
+            log_dir=abs_temp_dir,
+            log_basename="test",
+        )
+
+        # assert logdir exist
+        log_dir = Path(abs_temp_dir)
+        assert log_dir.exists()
+
+        # validate write log
+        col = zvec.create_and_open(
+            "/tmp/test/1",
+            zvec.CollectionSchema(
+                name="test",
+                vectors=zvec.VectorSchema(
+                    dimension=4,
+                    data_type=zvec.DataType.VECTOR_FP32,
+                    name="image",
+                ),
+            ),
+        )
+        col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
+        assert any(log_dir.glob("test.*"))
+
+        # clear
+        col.destroy()
+        shutil.rmtree(log_dir, ignore_errors=True)
+
+    @run_in_subprocess
+    def test_log_file_size_invalid(self):
+        with pytest.raises(TypeError):
+            zvec.init(log_type=LogType.FILE, log_file_size="df")
+
+        with pytest.raises(ValueError):
+            zvec.init(log_type=LogType.FILE, log_file_size=0)
+
+        with pytest.raises(ValueError):
+            zvec.init(log_type=LogType.FILE, log_file_size=-1)
+
+    @run_in_subprocess
+    def test_log_overdue_days_invalid(self):
+        with pytest.raises(TypeError):
+            zvec.init(log_type=LogType.FILE, log_overdue_days="df")
+
+        with pytest.raises(ValueError):
+            zvec.init(log_type=LogType.FILE, log_overdue_days=0)
+
+        with pytest.raises(ValueError):
+            zvec.init(log_type=LogType.FILE, log_overdue_days=-1)

python/tests/test_convert.py

@@ -0,0 +1,584 @@
+from __future__ import annotations
+
+import math
+
+import pytest
+from _zvec import _Doc
+from zvec.model.convert import convert_to_py_doc, convert_to_cpp_doc
+from zvec import Doc, CollectionSchema, DataType, FieldSchema, VectorSchema
+
+
+# ----------------------------
+# Convert Cpp Doc Test Case
+# ----------------------------
+class TestConvertCppDoc:
+    def test_default(self):
+        doc = Doc(id="1")
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=FieldSchema("name", DataType.STRING),
+        )
+
+        cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
+
+        assert cpp_doc is not None
+        assert cpp_doc.pk() == doc.id
+
+    def test_with_field_notin_schema(self):
+        doc = Doc(id="1", fields={"name": "Tom"})
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("id", DataType.UINT64),
+                FieldSchema("salary", DataType.UINT32),
+                FieldSchema("age", DataType.INT32),
+                FieldSchema("create_at", DataType.INT64),
+                FieldSchema("author", DataType.STRING),
+                FieldSchema("weight", DataType.FLOAT),
+            ],
+        )
+        with pytest.raises(ValueError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+    def test_with_scalar_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("id", DataType.UINT64),
+                FieldSchema("salary", DataType.UINT32),
+                FieldSchema("age", DataType.INT32),
+                FieldSchema("create_at", DataType.INT64),
+                FieldSchema("author", DataType.STRING),
+                FieldSchema("weight", DataType.FLOAT),
+                FieldSchema("bmi", DataType.DOUBLE),
+                FieldSchema("is_male", DataType.BOOL),
+            ],
+        )
+        doc = Doc(
+            id="1",
+            fields={
+                "id": 1,
+                "salary": 1000,
+                "age": 18,
+                "create_at": 1640995200,
+                "bmi": 80.0 / 200.0,
+                "author": "Tom",
+                "weight": 80.0,
+                "is_male": True,
+            },
+        )
+        cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
+        assert cpp_doc is not None
+        assert cpp_doc.pk() == doc.id
+        assert cpp_doc.get_any("id", DataType.UINT64) == 1
+        assert cpp_doc.get_any("salary", DataType.UINT32) == 1000
+        assert cpp_doc.get_any("age", DataType.INT32) == 18
+        assert cpp_doc.get_any("create_at", DataType.INT64) == 1640995200
+        assert cpp_doc.get_any("author", DataType.STRING) == "Tom"
+        assert math.isclose(
+            cpp_doc.get_any("weight", DataType.FLOAT), 80.0, rel_tol=1e-6
+        )
+        assert math.isclose(
+            cpp_doc.get_any("bmi", DataType.DOUBLE), 80.0 / 200.0, rel_tol=1e-6
+        )
+        assert cpp_doc.get_any("is_male", DataType.BOOL) == True
+
+    def test_with_array_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("tags", DataType.ARRAY_STRING),
+                FieldSchema("ids", DataType.ARRAY_UINT64),
+                FieldSchema("marks", DataType.ARRAY_UINT32),
+                FieldSchema("x", DataType.ARRAY_INT32),
+                FieldSchema("y", DataType.ARRAY_INT64),
+                FieldSchema("scores", DataType.ARRAY_FLOAT),
+                FieldSchema("ratios", DataType.ARRAY_DOUBLE),
+                FieldSchema("results", DataType.ARRAY_BOOL),
+            ],
+        )
+
+        doc = Doc(
+            id="1",
+            fields={
+                "tags": ["tag1", "tag2", "tag3"],
+                "ids": [111111111111, 222222222222, 333333333333],
+                "marks": [100, 200, 300],
+                "x": [1, 2, 3],
+                "y": [100, 200, 300],
+                "scores": [1.1, 2.2, 3.3],
+                "ratios": [0.1, 0.2, 0.3],
+                "results": [True, False, True],
+            },
+        )
+        cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
+
+        assert cpp_doc is not None
+        assert cpp_doc.pk() == doc.id
+        assert cpp_doc.get_any("tags", DataType.ARRAY_STRING) == doc.field("tags")
+        assert cpp_doc.get_any("ids", DataType.ARRAY_UINT64) == doc.field("ids")
+        assert cpp_doc.get_any("marks", DataType.ARRAY_UINT32) == doc.field("marks")
+        assert cpp_doc.get_any("x", DataType.ARRAY_INT32) == doc.field("x")
+        assert cpp_doc.get_any("y", DataType.ARRAY_INT64) == doc.field("y")
+        scores = cpp_doc.get_any("scores", DataType.ARRAY_FLOAT)
+        for i in range(len(doc.field("scores"))):
+            assert math.isclose(scores[i], doc.field("scores")[i], rel_tol=1e-1)
+        ratios = cpp_doc.get_any("ratios", DataType.ARRAY_DOUBLE)
+        for i in range(len(doc.field("ratios"))):
+            assert math.isclose(ratios[i], doc.field("ratios")[i], rel_tol=1e-1)
+        results = cpp_doc.get_any("results", DataType.ARRAY_BOOL)
+        for i in range(len(doc.field("results"))):
+            assert results[i] == doc.field("results")[i]
+
+    def test_with_dense_vector_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="embedding",
+                    data_type=DataType.VECTOR_FP16,
+                    dimension=4,
+                ),
+                VectorSchema(
+                    name="image",
+                    data_type=DataType.VECTOR_FP32,
+                    dimension=8,
+                ),
+                VectorSchema(
+                    name="text",
+                    data_type=DataType.VECTOR_INT8,
+                    dimension=32,
+                ),
+            ],
+        )
+
+        doc = Doc(
+            id="1",
+            vectors={
+                "embedding": [1.1] * 4,
+                "image": [2.2] * 8,
+                "text": [4] * 32,
+            },
+        )
+        cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
+        assert cpp_doc is not None
+        assert cpp_doc.pk() == doc.id
+
+        embedding_vector = cpp_doc.get_any("embedding", DataType.VECTOR_FP16)
+        assert len(embedding_vector) == 4
+        for i in range(4):
+            assert math.isclose(
+                embedding_vector[i], doc.vector("embedding")[i], rel_tol=1e-1
+            )
+
+        image_vector = cpp_doc.get_any("image", DataType.VECTOR_FP32)
+        assert len(image_vector) == 8
+        for i in range(8):
+            assert math.isclose(image_vector[i], doc.vector("image")[i], rel_tol=1e-1)
+
+        text_vector = cpp_doc.get_any("text", DataType.VECTOR_INT8)
+        assert len(text_vector) == 32
+        for i in range(32):
+            assert text_vector[i] == doc.vectors["text"][i]
+
+    def test_with_sparse_vector_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="author",
+                    data_type=DataType.SPARSE_VECTOR_FP32,
+                ),
+                VectorSchema(
+                    name="content",
+                    data_type=DataType.SPARSE_VECTOR_FP16,
+                ),
+            ],
+        )
+        doc = Doc(
+            id="1",
+            vectors={
+                "author": {1: 1.1, 2: 2.2, 3: 3.3},
+                "content": {4: 4.4, 5: 5.5, 6: 6.6},
+            },
+        )
+
+        cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
+        assert cpp_doc is not None
+        assert cpp_doc.pk() == doc.id
+
+        author_vector = cpp_doc.get_any("author", DataType.SPARSE_VECTOR_FP32)
+        assert isinstance(author_vector, dict)
+        for key, value in doc.vector("author").items():
+            assert math.isclose(author_vector[key], value, rel_tol=1e-1)
+
+        content_vector = cpp_doc.get_any("content", DataType.SPARSE_VECTOR_FP16)
+        assert isinstance(content_vector, dict)
+        for key, value in doc.vector("content").items():
+            assert math.isclose(content_vector[key], value, rel_tol=1e-1)
+
+    def test_with_scalar_fields_error_datatype(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("id", DataType.UINT64),
+                FieldSchema("salary", DataType.UINT32),
+                FieldSchema("age", DataType.INT32),
+                FieldSchema("create_at", DataType.INT64),
+                FieldSchema("author", DataType.STRING),
+                FieldSchema("weight", DataType.FLOAT),
+                FieldSchema("bmi", DataType.DOUBLE),
+                FieldSchema("is_male", DataType.BOOL),
+            ],
+        )
+        doc = Doc(
+            id="1",
+            fields={
+                "id": "1",
+            },
+        )
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"salary": "1000"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"age": "18"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"create_at": "2021-01-01"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"author": 1})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"weight": "80.5"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"bmi": "25.0"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"is_male": "true"})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+    def test_with_array_fields_error_datatype(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("tags", DataType.ARRAY_STRING),
+                FieldSchema("ids", DataType.ARRAY_UINT64),
+                FieldSchema("marks", DataType.ARRAY_UINT32),
+                FieldSchema("x", DataType.ARRAY_INT32),
+                FieldSchema("y", DataType.ARRAY_INT64),
+                FieldSchema("scores", DataType.ARRAY_FLOAT),
+                FieldSchema("ratios", DataType.ARRAY_DOUBLE),
+                FieldSchema("results", DataType.ARRAY_BOOL),
+            ],
+        )
+
+        doc = Doc(id="1", fields={"tags": [1, 2, 3]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"ids": ["1", "2", "3"]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"marks": [1.1, 2.2, 3.3]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"x": [1.1, 2.2, 3.3]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"y": [1.1, 2.2, 3.3]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"scores": ["1", "2", "3"]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"ratios": ["1", "2", "3"]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", fields={"results": ["1", "2", "3"]})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+    def test_with_vector_fields_error_datatype(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="embedding",
+                    data_type=DataType.VECTOR_FP16,
+                    dimension=4,
+                ),
+                VectorSchema(
+                    name="image",
+                    data_type=DataType.VECTOR_FP32,
+                    dimension=8,
+                ),
+                VectorSchema(
+                    name="text",
+                    data_type=DataType.VECTOR_INT8,
+                    dimension=32,
+                ),
+            ],
+        )
+
+        doc = Doc(id="1", vectors={"image": ["1.1"] * 4})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", vectors={"text": ["1"] * 4})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(id="1", vectors={"embedding": ["1"] * 4})
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+    def test_with_sparse_vector_error_datatype(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="author",
+                    data_type=DataType.SPARSE_VECTOR_FP32,
+                ),
+                VectorSchema(
+                    name="content",
+                    data_type=DataType.SPARSE_VECTOR_FP16,
+                ),
+            ],
+        )
+        doc = Doc(
+            id="1",
+            vectors={
+                "author": {"1": 1.1, "2": 2.2, "3": 3.3},
+            },
+        )
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(
+            id="1",
+            vectors={
+                "content": {"1": 1.1, "2": 2.2, "3": 3.3},
+            },
+        )
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+        doc = Doc(
+            id="1",
+            vectors={
+                "author": {1: "1", 2: "2", 3: "3"},
+            },
+        )
+        with pytest.raises(TypeError):
+            convert_to_cpp_doc(doc, collection_schema=schema)
+
+
+# ----------------------------
+# Convert Py Doc Test Case
+# ----------------------------
+class TestConvertPyDoc:
+    def test_default(self):
+        doc = _Doc()
+        doc.set_pk("1")
+        doc.set_score(1.0)
+
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=FieldSchema("name", DataType.STRING),
+        )
+
+        py_doc = convert_to_py_doc(doc, schema)
+        assert py_doc.id == "1"
+        assert py_doc.score == 1.0
+
+    def test_with_scalar_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("id", DataType.UINT64),
+                FieldSchema("salary", DataType.UINT32),
+                FieldSchema("age", DataType.INT32),
+                FieldSchema("create_at", DataType.INT64),
+                FieldSchema("author", DataType.STRING),
+                FieldSchema("weight", DataType.FLOAT),
+                FieldSchema("bmi", DataType.DOUBLE),
+                FieldSchema("is_male", DataType.BOOL),
+            ],
+        )
+        doc = _Doc()
+        doc.set_pk("1")
+        doc.set_any("id", schema.field("id")._get_object(), 1111111111111111)
+        doc.set_any("salary", schema.field("salary")._get_object(), 1000)
+        doc.set_any("age", schema.field("age")._get_object(), 18)
+        doc.set_any("create_at", schema.field("create_at")._get_object(), 1640995200)
+        doc.set_any("author", schema.field("author")._get_object(), "Tom")
+        doc.set_any("weight", schema.field("weight")._get_object(), 80.0)
+        doc.set_any("bmi", schema.field("bmi")._get_object(), 80.0 / 200.0)
+        doc.set_any("is_male", schema.field("is_male")._get_object(), True)
+
+        py_doc = convert_to_py_doc(doc, schema)
+        assert py_doc.id == "1"
+        assert py_doc.field("id") == 1111111111111111
+        assert py_doc.field("salary") == 1000
+        assert py_doc.field("age") == 18
+        assert py_doc.field("create_at") == 1640995200
+        assert py_doc.field("author") == "Tom"
+        assert py_doc.field("weight") == 80.0
+        assert py_doc.field("bmi") == 80.0 / 200.0
+        assert py_doc.field("is_male") == True
+
+    def test_with_array_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema("tags", DataType.ARRAY_STRING),
+                FieldSchema("ids", DataType.ARRAY_UINT64),
+                FieldSchema("marks", DataType.ARRAY_UINT32),
+                FieldSchema("x", DataType.ARRAY_INT32),
+                FieldSchema("y", DataType.ARRAY_INT64),
+                FieldSchema("scores", DataType.ARRAY_FLOAT),
+                FieldSchema("ratios", DataType.ARRAY_DOUBLE),
+                FieldSchema("results", DataType.ARRAY_BOOL),
+            ],
+        )
+
+        doc = _Doc()
+        doc.set_pk("1")
+        doc.set_any(
+            "tags", schema.field("tags")._get_object(), ["tag1", "tag2", "tag3"]
+        )
+        doc.set_any(
+            "ids",
+            schema.field("ids")._get_object(),
+            [111111111111, 222222222222, 3333333333333],
+        )
+        doc.set_any("marks", schema.field("marks")._get_object(), [1000, 2000, 3000])
+        doc.set_any("x", schema.field("x")._get_object(), [1, 2, 3])
+        doc.set_any("y", schema.field("y")._get_object(), [100, 200, 300])
+        doc.set_any("scores", schema.field("scores")._get_object(), [0.1, 0.2, 0.3])
+        doc.set_any("ratios", schema.field("ratios")._get_object(), [0.1, 0.2, 0.3])
+        doc.set_any(
+            "results", schema.field("results")._get_object(), [True, False, True]
+        )
+
+        py_doc = convert_to_py_doc(doc, schema)
+        assert py_doc.field("tags") == ["tag1", "tag2", "tag3"]
+        assert py_doc.field("ids") == [111111111111, 222222222222, 3333333333333]
+        assert py_doc.field("marks") == [1000, 2000, 3000]
+        assert py_doc.field("x") == [1, 2, 3]
+        assert py_doc.field("y") == [100, 200, 300]
+
+        scores = doc.get_any("scores", DataType.ARRAY_FLOAT)
+        for i in range(len(scores)):
+            assert math.isclose(scores[i], py_doc.field("scores")[i], rel_tol=1e-1)
+        ratios = doc.get_any("ratios", DataType.ARRAY_DOUBLE)
+        for i in range(len(ratios)):
+            assert math.isclose(ratios[i], py_doc.field("ratios")[i], rel_tol=1e-1)
+        results = doc.get_any("results", DataType.ARRAY_BOOL)
+        for i in range(len(results)):
+            assert results[i] == py_doc.field("results")[i]
+
+    def test_with_dense_vector_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="embedding",
+                    data_type=DataType.VECTOR_FP16,
+                    dimension=4,
+                ),
+                VectorSchema(
+                    name="image",
+                    data_type=DataType.VECTOR_FP32,
+                    dimension=8,
+                ),
+                VectorSchema(
+                    name="text",
+                    data_type=DataType.VECTOR_INT8,
+                    dimension=32,
+                ),
+            ],
+        )
+
+        doc = _Doc()
+        doc.set_pk("1")
+        doc.set_any("embedding", schema.vector("embedding")._get_object(), [1.1] * 4)
+        doc.set_any("image", schema.vector("image")._get_object(), [2.2] * 8)
+        doc.set_any("text", schema.vector("text")._get_object(), [4] * 32)
+
+        py_doc = convert_to_py_doc(doc, schema)
+        assert py_doc.id == "1"
+
+        embedding_vector = py_doc.vector("embedding")
+        assert len(embedding_vector) == 4
+        for i in range(4):
+            assert math.isclose(
+                py_doc.vector("embedding")[i], embedding_vector[i], rel_tol=1e-1
+            )
+
+        image_vector = py_doc.vector("image")
+        assert len(image_vector) == 8
+        for i in range(8):
+            assert math.isclose(
+                py_doc.vector("image")[i], image_vector[i], rel_tol=1e-1
+            )
+
+        text_vector = py_doc.vector("text")
+        assert len(text_vector) == 32
+        for i in range(32):
+            assert py_doc.vector("text")[i] == text_vector[i]
+
+    def test_with_sparse_vector_fields(self):
+        schema = CollectionSchema(
+            name="test_collection",
+            vectors=[
+                VectorSchema(
+                    name="author",
+                    data_type=DataType.SPARSE_VECTOR_FP32,
+                ),
+                VectorSchema(
+                    name="content",
+                    data_type=DataType.SPARSE_VECTOR_FP16,
+                ),
+            ],
+        )
+
+        doc = _Doc()
+        doc.set_pk("1")
+        doc.set_any(
+            "author", schema.vector("author")._get_object(), {1: 1.1, 2: 2.2, 3: 3.3}
+        )
+        doc.set_any(
+            "content", schema.vector("content")._get_object(), {4: 4.4, 5: 5.5, 6: 6.6}
+        )
+
+        py_doc = convert_to_py_doc(doc, schema)
+        assert py_doc.id == "1"
+
+        author_vector = py_doc.vector("author")
+        assert isinstance(author_vector, dict)
+        for key, value in doc.get_any("author", DataType.SPARSE_VECTOR_FP32).items():
+            assert math.isclose(author_vector[key], value, rel_tol=1e-1)
+
+        content_vector = py_doc.vector("content")
+        assert isinstance(content_vector, dict)
+        for key, value in doc.get_any("content", DataType.SPARSE_VECTOR_FP16).items():
+            assert math.isclose(content_vector[key], value, rel_tol=1e-1)

python/tests/test_doc.py

@@ -0,0 +1,269 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import math
+import pytest
+
+
+from _zvec import _Doc
+from zvec import FieldSchema, VectorSchema, Doc, DataType
+
+
+# ----------------------------
+# PyDoc Test Case
+# ----------------------------
+class TestPyDoc:
+    def test_default(self):
+        Doc(id="1")
+
+    def test_with_single_vector(self):
+        doc = Doc(id="1", vectors={"dense": [1, 2, 3]})
+        assert doc is not None
+        assert doc.id == "1"
+        assert doc.vector("dense") == [1, 2, 3]
+
+    def test_with_hybrid_vectors(self):
+        doc = Doc(
+            id="1", vectors={"dense": [1, 2, 3], "sparse": {1: 1.0, 2: 2.0, 3: 3.0}}
+        )
+        assert doc is not None
+        assert doc.id == "1"
+        assert doc.vector("dense") == [1, 2, 3]
+        assert doc.vector("sparse") == {1: 1.0, 2: 2.0, 3: 3.0}
+
+    def test_with_multi_vectors(self):
+        doc = Doc(
+            id="1",
+            vectors={
+                "image": [1, 2, 3],
+                "description": [4, 5, 6],
+                "keys": {1: 1.0, 2: 2.0, 3: 3.0},
+            },
+            fields={"author": "Tom", "age": 19, "is_male": True, "weight": 60.5},
+        )
+        assert doc is not None
+        assert doc.id == "1"
+        assert doc.vector("image") == [1, 2, 3]
+        assert doc.vector("description") == [4, 5, 6]
+        assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
+        assert doc.field("author") == "Tom"
+        assert doc.field("age") == 19
+        assert doc.field("is_male") == True
+        assert doc.field("weight") == 60.5
+
+    def test_with_numpy_array(self):
+        import numpy as np
+
+        doc = Doc._from_tuple(
+            (
+                "1",
+                0.0,
+                None,
+                {
+                    "image": np.array([1, 2, 3]),
+                    "description": np.random.random(512),
+                    "keys": {1: 1.0, 2: 2.0, 3: 3.0},
+                },
+            )
+        )
+        assert doc is not None
+        assert doc.id == "1"
+        assert doc.vector("image") == [1, 2, 3]
+        assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
+
+
+# ----------------------------
+# CppDoc Test Case
+# ----------------------------
+class TestCppDoc:
+    def test_default(self):
+        doc = _Doc()
+        assert doc is not None
+
+    def test_doc_set_pk(self):
+        doc = _Doc()
+        doc.set_pk("1")
+        assert doc.pk() == "1"
+
+    def test_doc_set_score(self):
+        doc = _Doc()
+        doc.set_score(0.9)
+        assert math.isclose(doc.score(), 0.9, rel_tol=1e-6)
+
+    def test_doc_get_null_field(self):
+        doc = _Doc()
+        schema = FieldSchema("author", DataType.STRING, nullable=True)
+        doc.set_any("author", schema._get_object(), None)
+        assert doc.has_field("author")
+        assert doc.get_any("author", schema.data_type) is None
+
+    def test_doc_get_set_has_null_field(self):
+        doc = _Doc()
+        schema = FieldSchema("author", DataType.STRING, nullable=False)
+        with pytest.raises(ValueError):
+            doc.set_any("author", schema._get_object(), None)
+
+    def test_doc_get_set_has_string_field(self):
+        doc = _Doc()
+        schema = FieldSchema("author", DataType.STRING)
+        doc.set_any("author", schema._get_object(), "Tom")
+        assert doc.has_field("author")
+        assert doc.get_any("author", DataType.STRING) == "Tom"
+
+    def test_doc_get_set_has_bool_field(self):
+        doc = _Doc()
+        schema = FieldSchema("is_male", DataType.BOOL)
+        doc.set_any("is_male", schema._get_object(), True)
+        assert doc.has_field("is_male")
+        assert doc.get_any("is_male", DataType.BOOL) == True
+
+    def test_doc_get_set_has_int32_field(self):
+        doc = _Doc()
+        schema = FieldSchema("age", DataType.INT32)
+        doc.set_any("age", schema._get_object(), 19)
+        assert doc.has_field("age")
+        assert doc.get_any("age", DataType.INT32) == 19
+
+    def test_doc_get_set_has_int64_field(self):
+        doc = _Doc()
+        schema = FieldSchema("id", DataType.INT64)
+        doc.set_any("id", schema._get_object(), 1111111111111111111)
+        assert doc.has_field("id")
+        assert doc.get_any("id", DataType.INT64) == 1111111111111111111
+
+    def test_doc_get_set_has_float_field(self):
+        doc = _Doc()
+        schema = FieldSchema("weight", DataType.FLOAT)
+        doc.set_any("weight", schema._get_object(), 60.5)
+        assert doc.has_field("weight")
+        assert math.isclose(doc.get_any("weight", DataType.FLOAT), 60.5, rel_tol=1e-6)
+
+    def test_doc_get_set_has_double_field(self):
+        doc = _Doc()
+        schema = FieldSchema("height", DataType.DOUBLE)
+        doc.set_any("height", schema._get_object(), 1.77777777777)
+        assert doc.has_field("height")
+        assert math.isclose(
+            doc.get_any("height", DataType.DOUBLE), 1.7777777777, rel_tol=1e-9
+        )
+
+    def test_doc_get_set_has_uint32_field(self):
+        doc = _Doc()
+        schema = FieldSchema("id", DataType.UINT32)
+        doc.set_any("id", schema._get_object(), 4294967295)
+        assert doc.has_field("id")
+        assert doc.get_any("id", DataType.UINT32) == 4294967295
+
+    def test_doc_get_set_has_uint64_field(self):
+        doc = _Doc()
+        schema = FieldSchema("id", DataType.UINT64)
+        doc.set_any("id", schema._get_object(), 18446744073709551615)
+        assert doc.has_field("id")
+        assert doc.get_any("id", DataType.UINT64) == 18446744073709551615
+
+    def test_doc_get_set_has_array_string_field(self):
+        doc = _Doc()
+        schema = FieldSchema("tags", DataType.ARRAY_STRING)
+        doc.set_any("tags", schema._get_object(), ["tag1", "tag2", "tag3"])
+        assert doc.has_field("tags")
+        assert doc.get_any("tags", DataType.ARRAY_STRING) == ["tag1", "tag2", "tag3"]
+
+    def test_doc_get_set_has_array_int32_field(self):
+        doc = _Doc()
+        schema = FieldSchema("ids", DataType.ARRAY_INT32)
+        doc.set_any("ids", schema._get_object(), [1, 2, 3])
+        assert doc.has_field("ids")
+        assert doc.get_any("ids", DataType.ARRAY_INT32) == [1, 2, 3]
+
+    def test_doc_get_set_has_array_int64_field(self):
+        doc = _Doc()
+        schema = FieldSchema("ids", DataType.ARRAY_INT64)
+        doc.set_any("ids", schema._get_object(), [1, 2, 3])
+        assert doc.has_field("ids")
+        assert doc.get_any("ids", DataType.ARRAY_INT64) == [1, 2, 3]
+
+    def test_doc_get_set_has_array_float_field(self):
+        doc = _Doc()
+        schema = FieldSchema("weights", DataType.ARRAY_FLOAT)
+        doc.set_any("weights", schema._get_object(), [1.0, 2.0, 3.0])
+        assert doc.has_field("weights")
+        assert doc.get_any("weights", DataType.ARRAY_FLOAT) == [1.0, 2.0, 3.0]
+
+    def test_doc_get_set_has_array_double_field(self):
+        doc = _Doc()
+        schema = FieldSchema("heights", DataType.ARRAY_DOUBLE)
+        doc.set_any("heights", schema._get_object(), [1.0, 2.0, 3.0])
+        assert doc.has_field("heights")
+        assert doc.get_any("heights", DataType.ARRAY_DOUBLE) == [1.0, 2.0, 3.0]
+
+    def test_doc_get_set_has_array_bool_field(self):
+        doc = _Doc()
+        schema = FieldSchema("bools", DataType.ARRAY_BOOL)
+        doc.set_any("bools", schema._get_object(), [True, False, True])
+        assert doc.has_field("bools")
+        assert doc.get_any("bools", DataType.ARRAY_BOOL) == [True, False, True]
+
+    def test_doc_get_set_has_vector_fp16(self):
+        doc = _Doc()
+        schema = VectorSchema("image", DataType.VECTOR_FP16)
+        doc.set_any("image", schema._get_object(), [1.0, 2.0, 3.0])
+        assert doc.has_field("image")
+        image_vector = doc.get_any("image", DataType.VECTOR_FP16)
+        assert image_vector is not None
+        for i in range(len(image_vector)):
+            assert math.isclose(image_vector[i], [1.0, 2.0, 3.0][i], rel_tol=1e-6)
+
+    def test_doc_get_set_has_vector_fp32(self):
+        doc = _Doc()
+        schema = VectorSchema("image", DataType.VECTOR_FP32)
+        doc.set_any("image", schema._get_object(), [1.111111, 2.222222, 3.333333])
+        assert doc.has_field("image")
+        vector = doc.get_any("image", DataType.VECTOR_FP32)
+        assert vector is not None
+        for i in range(len(vector)):
+            assert math.isclose(
+                vector[i], [1.111111, 2.222222, 3.333333][i], rel_tol=1e-6
+            )
+
+    def test_doc_get_set_has_vector_int8(self):
+        doc = _Doc()
+        schema = VectorSchema("image", DataType.VECTOR_INT8)
+        doc.set_any("image", schema._get_object(), [1, 2, 3])
+        assert doc.has_field("image")
+        assert doc.get_any("image", DataType.VECTOR_INT8) == [1, 2, 3]
+
+    def test_doc_get_set_has_sparse_vector_fp32(self):
+        doc = _Doc()
+        sparse = {1: 1.111111, 2: 2.222222, 3: 3.333333}
+        schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP32)
+        doc.set_any("key", schema._get_object(), sparse)
+        assert doc.has_field("key")
+        vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP32)
+        assert vector is not None
+        assert isinstance(vector, dict)
+        for key, value in sparse.items():
+            assert math.isclose(vector[key], value, rel_tol=1e-6)
+
+    def test_doc_get_set_has_sparse_vector_fp16(self):
+        doc = _Doc()
+        sparse = {1: 1.1, 2: 2.2, 3: 3.3}
+        schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP16)
+        doc.set_any("key", schema._get_object(), sparse)
+        assert doc.has_field("key")
+        vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP16)
+        assert vector is not None
+        assert isinstance(vector, dict)
+        for key, value in sparse.items():
+            assert math.isclose(vector[key], value, rel_tol=1e-1)

python/tests/test_embedding.py

@@ -0,0 +1,131 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+from http import HTTPStatus
+from unittest.mock import MagicMock, patch
+
+
+import pytest
+from zvec.extension import QwenEmbeddingFunction
+
+
+# ----------------------------
+# QwenEmbeddingFunction Test Case
+# ----------------------------
+class TestQwenEmbeddingFunction:
+    def test_init_with_api_key(self):
+        # Test initialization with explicit API key
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+        assert embedding_func.dimension == 128
+        assert embedding_func.model == "text-embedding-v4"
+        assert embedding_func._api_key == "test_key"
+
+    @patch.dict(os.environ, {"DASHSCOPE_API_KEY": "env_key"})
+    def test_init_with_env_api_key(self):
+        # Test initialization with API key from environment
+        embedding_func = QwenEmbeddingFunction(dimension=128)
+        assert embedding_func._api_key == "env_key"
+
+    def test_init_without_api_key(self):
+        # Test initialization without API key raises ValueError
+        with pytest.raises(ValueError, match="DashScope API key is required"):
+            QwenEmbeddingFunction(dimension=128)
+
+    @patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""})
+    def test_init_with_empty_env_api_key(self):
+        # Test initialization with empty API key from environment
+        with pytest.raises(ValueError, match="DashScope API key is required"):
+            QwenEmbeddingFunction(dimension=128)
+
+    def test_model_property(self):
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+        assert embedding_func.model == "text-embedding-v4"
+
+        embedding_func = QwenEmbeddingFunction(
+            dimension=128, model="custom-model", api_key="test_key"
+        )
+        assert embedding_func.model == "custom-model"
+
+    @patch("zvec.extension.embedding.require_module")
+    def test_embed_with_empty_text(self, mock_require_module):
+        # Test embed method with empty text raises ValueError
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+
+        with pytest.raises(
+            ValueError, match="Input text cannot be empty or whitespace only"
+        ):
+            embedding_func.embed("")
+
+        with pytest.raises(TypeError):
+            embedding_func.embed(None)
+
+    @patch("zvec.extension.embedding.require_module")
+    def test_embed_success(self, mock_require_module):
+        # Test successful embedding
+        mock_dashscope = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status_code = HTTPStatus.OK
+        mock_response.output = {"embeddings": [{"embedding": [0.1, 0.2, 0.3]}]}
+        mock_dashscope.TextEmbedding.call.return_value = mock_response
+        mock_require_module.return_value = mock_dashscope
+
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+        result = embedding_func.embed("test text")
+
+        assert result == [0.1, 0.2, 0.3]
+        mock_dashscope.TextEmbedding.call.assert_called_once_with(
+            model="text-embedding-v4",
+            input="test text",
+            dimension=128,
+            output_type="dense",
+        )
+
+    @patch("zvec.extension.embedding.require_module")
+    def test_embed_http_error(self, mock_require_module):
+        # Test embedding with HTTP error
+        mock_dashscope = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status_code = HTTPStatus.BAD_REQUEST
+        mock_response.message = "Bad Request"
+        mock_dashscope.TextEmbedding.call.return_value = mock_response
+        mock_require_module.return_value = mock_dashscope
+
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+
+        with pytest.raises(ValueError):
+            embedding_func.embed("test text")
+
+    @patch("zvec.extension.embedding.require_module")
+    def test_embed_invalid_response(self, mock_require_module):
+        # Test embedding with invalid response (wrong number of embeddings)
+        mock_dashscope = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status_code = HTTPStatus.OK
+        mock_response.output.embeddings = []
+        mock_dashscope.TextEmbedding.call.return_value = mock_response
+        mock_require_module.return_value = mock_dashscope
+
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
+
+        with pytest.raises(ValueError):
+            embedding_func.embed("test text")
+
+    @pytest.mark.skip(reason="Qwen Embedding is not available in CI")
+    def test_embed(self):
+        # Test embedding with invalid dimension
+        embedding_func = QwenEmbeddingFunction(dimension=128, api_key="xxx")
+        dense = embedding_func("test text")
+        assert len(dense) == 128

python/tests/test_params.py

@@ -0,0 +1,415 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import sys
+import time
+
+
+import numpy as np
+import pytest
+from zvec import (
+    AddColumnOption,
+    AlterColumnOption,
+    CollectionOption,
+    FlatIndexParam,
+    HnswIndexParam,
+    IndexOption,
+    InvertIndexParam,
+    IVFIndexParam,
+    OptimizeOption,
+    HnswQueryParam,
+    IVFQueryParam,
+    VectorQuery,
+    IndexType,
+    MetricType,
+    QuantizeType,
+    DataType,
+    VectorSchema,
+)
+
+from _zvec.param import _VectorQuery
+
+# ----------------------------
+# Invert Index Param Test Case
+# ----------------------------
+
+
+class TestInvertIndexParam:
+    def test_default(self):
+        param = InvertIndexParam()
+        assert param.enable_range_optimization is False
+        assert param.enable_extended_wildcard is False
+        assert param.type == IndexType.INVERT
+
+    def test_custom(self):
+        param = InvertIndexParam(
+            enable_range_optimization=True, enable_extended_wildcard=True
+        )
+        assert param.enable_range_optimization is True
+        assert param.enable_extended_wildcard is True
+
+    def test_readonly(self):
+        param = InvertIndexParam()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            param.enable_range_optimization = False
+            param.enable_extended_wildcard = False
+
+
+# ----------------------------
+# Hnsw Index Param Test Case
+# ----------------------------
+
+
+class TestHnswIndexParam:
+    def test_default(self):
+        param = HnswIndexParam()
+        assert param.metric_type == MetricType.IP
+        assert param.m == 100
+        assert param.ef_construction == 500
+        assert param.quantize_type == QuantizeType.UNDEFINED
+        assert param.type == IndexType.HNSW
+
+    def test_custom(self):
+        param = HnswIndexParam(
+            metric_type=MetricType.L2,
+            m=10,
+            ef_construction=1000,
+            quantize_type=QuantizeType.FP16,
+        )
+        assert param.metric_type == MetricType.L2
+        assert param.m == 10
+        assert param.ef_construction == 1000
+        assert param.quantize_type == QuantizeType.FP16
+
+    @pytest.mark.parametrize(
+        "attr", ["metric_type", "m", "ef_construction", "quantize_type"]
+    )
+    def test_readonly_attributes(self, attr):
+        param = HnswIndexParam()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# Flat Index Param Test Case
+# ----------------------------
+class TestFlatIndexParam:
+    def test_default(self):
+        param = FlatIndexParam()
+        assert param.type == IndexType.FLAT
+        assert param.quantize_type == QuantizeType.UNDEFINED
+        assert param.metric_type == MetricType.IP
+
+    def test_custom(self):
+        param = FlatIndexParam(
+            metric_type=MetricType.L2, quantize_type=QuantizeType.INT8
+        )
+        assert param.metric_type == MetricType.L2
+        assert param.quantize_type == QuantizeType.INT8
+
+    @pytest.mark.parametrize("attr", ["metric_type", "quantize_type"])
+    def test_readonly_attributes(self, attr):
+        param = FlatIndexParam()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# Ivf Index Param Test Case
+# ----------------------------
+class TestIVFIndexParam:
+    def test_default(self):
+        param = IVFIndexParam()
+        assert param.metric_type == MetricType.IP
+        assert param.n_list == 0
+        assert param.quantize_type == QuantizeType.UNDEFINED
+        assert param.type == IndexType.IVF
+
+    def test_custom(self):
+        param = IVFIndexParam(
+            metric_type=MetricType.L2, n_list=1000, quantize_type=QuantizeType.FP16
+        )
+        assert param.metric_type == MetricType.L2
+        assert param.n_list == 1000
+        assert param.quantize_type == QuantizeType.FP16
+        assert param.type == IndexType.IVF
+
+    @pytest.mark.parametrize("attr", ["metric_type", "n_list", "quantize_type"])
+    def test_readonly_attributes(self, attr):
+        param = IVFIndexParam()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# CollectionOption Test Case
+# ----------------------------
+class TestCollectionOption:
+    def test_default(self):
+        option = CollectionOption()
+        assert option is not None
+        assert option.read_only == False
+        assert option.enable_mmap == True
+
+    def test_custom(self):
+        option = CollectionOption(read_only=True, enable_mmap=False)
+        assert option.read_only == True
+        assert option.enable_mmap == False
+
+        option = CollectionOption(read_only=False, enable_mmap=True)
+        assert option.read_only == False
+        assert option.enable_mmap == True
+
+    @pytest.mark.parametrize("attr", ["read_only", "enable_mmap"])
+    def test_readonly_attributes(self, attr):
+        param = CollectionOption()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# IndexOption Test Case
+# ----------------------------
+class TestIndexOption:
+    def test_default(self):
+        option = IndexOption()
+        assert option is not None
+        assert option.concurrency == 0
+
+    def test_custom(self):
+        option = IndexOption(concurrency=10)
+        assert option.concurrency == 10
+
+    @pytest.mark.parametrize("attr", ["concurrency"])
+    def test_readonly_attributes(self, attr):
+        param = IndexOption()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# AddColumnOption Test Case
+# ----------------------------
+class TestAddColumnOption:
+    def test_default(self):
+        option = AddColumnOption()
+        assert option is not None
+        assert option.concurrency == 0
+
+    def test_custom(self):
+        option = AddColumnOption(concurrency=10)
+        assert option.concurrency == 10
+
+    @pytest.mark.parametrize("attr", ["concurrency"])
+    def test_readonly_attributes(self, attr):
+        param = AddColumnOption()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# AlterColumnOption Test Case
+# ----------------------------
+class TestAlterColumnOption:
+    def test_default(self):
+        option = AlterColumnOption()
+        assert option is not None
+        assert option.concurrency == 0
+
+    def test_custom(self):
+        option = AlterColumnOption(concurrency=10)
+        assert option.concurrency == 10
+
+    @pytest.mark.parametrize("attr", ["concurrency"])
+    def test_readonly_attributes(self, attr):
+        param = AlterColumnOption()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# OptimizeOption Test Case
+# ----------------------------
+class TestOptimizeOption:
+    def test_default(self):
+        option = OptimizeOption()
+        assert option is not None
+        assert option.concurrency == 0
+
+    def test_custom(self):
+        option = OptimizeOption(concurrency=10)
+        assert option.concurrency == 10
+
+    @pytest.mark.parametrize("attr", ["concurrency"])
+    def test_readonly_attributes(self, attr):
+        param = OptimizeOption()
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            setattr(param, attr, getattr(param, attr))
+
+
+# ----------------------------
+# HnswQueryParam Test Case
+# ----------------------------
+class TestHnswQueryParam:
+    def test_default(self):
+        param = HnswQueryParam()
+        assert param is not None
+        assert param.ef == 300
+        assert param.is_using_refiner == False
+        assert param.radius == 0
+        assert param.is_linear == False
+
+    def test_custom(self):
+        param = HnswQueryParam(ef=10, is_using_refiner=True, radius=30, is_linear=True)
+        assert param.ef == 10
+        assert param.is_using_refiner == True
+        assert param.radius == 30
+        assert param.is_linear == True
+
+    def test_readonly_attributes(self):
+        param = HnswQueryParam()
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+            with pytest.raises(AttributeError, match=match_pattern):
+                param.ef = 10
+                param.is_using_refiner = True
+                param.radius = 30
+                param.is_linear = True
+
+
+# # ----------------------------
+# # IVFQueryParam Test Case
+# # ----------------------------
+# class TestIVFQueryParam:
+#     def test_default(self):
+#         param = IVFQueryParam()
+#         assert param is not None
+#         assert param.nprobe == 10
+#         assert param.is_using_refiner == False
+#         assert param.radius == 0
+#         assert param.is_linear == False
+#         assert param.scale_factor == 10
+#
+#     def test_custom(self):
+#         param = IVFQueryParam(
+#             nprobe=20,
+#             is_using_refiner=True,
+#             radius=30,
+#             is_linear=True,
+#             scale_factor=40
+#         )
+#         assert param.nprobe == 20
+#         assert param.is_using_refiner == True
+#         assert param.radius == 30
+#         assert param.is_linear == True
+#         assert param.scale_factor == 40
+
+
+class TestVectorQuery:
+    def test_init_with_valid_id(self):
+        vq = VectorQuery(field_name="embedding", id="doc123")
+        assert vq.field_name == "embedding"
+        assert vq.id == "doc123"
+        assert vq.vector is None
+        assert vq.param is None
+
+    def test_init_with_valid_vector(self):
+        vec = [0.1, 0.2, 0.3]
+        param = HnswQueryParam(ef=300)
+        vq = VectorQuery(field_name="embedding", vector=vec, param=param)
+        assert vq.field_name == "embedding"
+        assert vq.vector == vec
+        assert vq.param == param
+
+    def test_init_both_id_and_vector_raises_error(self):
+        with pytest.raises(ValueError):
+            VectorQuery(field_name="embedding", id="doc123", vector=[0.1])._validate()
+
+    def test_init_without_field_name_raises_error(self):
+        with pytest.raises(ValueError):
+            VectorQuery(field_name=None)._validate()
+
+    def test_has_id_returns_true_when_id_set(self):
+        vq = VectorQuery(field_name="embedding", id="doc123")
+        assert vq.has_id()
+
+    def test_has_id_returns_false_when_no_id(self):
+        vq = VectorQuery(field_name="embedding", vector=[0.1])
+        assert not vq.has_id()
+
+    def test_has_vector_returns_true_with_non_empty_vector(self):
+        vq = VectorQuery(field_name="embedding", vector=[0.1])
+        assert vq.has_vector()
+
+    def test_validate_fails_on_both_id_and_vector(self):
+        vq = VectorQuery(field_name="test", id="doc123", vector=[0.1])
+        with pytest.raises(ValueError):
+            vq._validate()

python/tests/test_query_executor.py

@@ -0,0 +1,297 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import Dict, Union
+from unittest.mock import MagicMock
+
+import numpy as np
+import math
+from _zvec.param import _VectorQuery
+
+import pytest
+from zvec.executor.query_executor import (
+    MultiVectorQueryExecutor,
+    NoVectorQueryExecutor,
+    QueryContext,
+    QueryExecutor,
+    QueryExecutorFactory,
+    SingleVectorQueryExecutor,
+    VectorQuery,
+)
+from zvec import RrfReRanker, HnswQueryParam, CollectionSchema, VectorSchema, DataType
+
+
+# ----------------------------
+# Mock Vector Schema
+# ----------------------------
+class MockVectorSchema(VectorSchema):
+    def __init__(self, name="test_vector"):
+        self._name = name
+
+    @property
+    def name(self):
+        return self._name
+
+    def _get_object(self):
+        return MagicMock()
+
+
+# ----------------------------
+# Mock Collection Schema
+# ----------------------------
+class MockCollectionSchema(CollectionSchema):
+    def __init__(self, vectors=Union[VectorSchema, Dict[str, VectorSchema]]):
+        self._vectors = (
+            [vectors] if not isinstance(vectors, Dict) else list(vectors.values())
+        )
+
+    @property
+    def vectors(self):
+        return self._vectors
+
+
+# ----------------------------
+# VectorQuery Test Case
+# ----------------------------
+class TestVectorQuery:
+    def test_init(self):
+        query = VectorQuery(field_name="test_field")
+        assert query.field_name == "test_field"
+        assert query.id is None
+        assert query.vector is None
+        assert query.param is None
+
+        param = HnswQueryParam()
+        query = VectorQuery(
+            field_name="test_field", id="test_id", vector=[1, 2, 3], param=param
+        )
+        assert query.field_name == "test_field"
+        assert query.id == "test_id"
+        assert query.vector == [1, 2, 3]
+        assert query.param == param
+
+    def test_has_id(self):
+        query = VectorQuery(field_name="test_field")
+        assert not query.has_id()
+
+        query = VectorQuery(field_name="test_field", id="test_id")
+        assert query.has_id()
+
+    def test_has_vector(self):
+        query = VectorQuery(field_name="test_field")
+        assert not query.has_vector()
+
+        query = VectorQuery(field_name="test_field", vector=[])
+        assert not query.has_vector()
+
+        query = VectorQuery(field_name="test_field", vector=[1, 2, 3])
+        assert query.has_vector()
+
+    def test_validate_dense_fp16_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP16)
+        vec = np.array([1.1, 2.1, 3.1], dtype=np.float16)
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        assert np.array_equal(vec, ret)
+
+    def test_validate_dense_fp32_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP32)
+        vec = np.array([1.1, 2.1, 3.1], dtype=np.float32)
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        assert np.array_equal(vec, ret)
+
+    def test_validate_dense_fp64_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP64)
+        vec = np.array([1.1, 2.1, 3.1], dtype=np.float64)
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        assert np.array_equal(vec, ret)
+
+    def test_validate_dense_int8_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.VECTOR_INT8)
+        vec = np.array([1, 2, 3], dtype=np.int8)
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        assert np.array_equal(vec, ret)
+
+    def test_validate_sparse_fp32_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP32)
+        vec = {1: 1.1, 2: 2.2, 3: 3.3}
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        for k in vec.keys():
+            assert math.isclose(vec[k], ret[k], abs_tol=1e-6)
+
+    def test_validate_sparse_fp16_convert(self):
+        v = _VectorQuery()
+        schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP16)
+        vec = {1: 1.1, 2: 2.2, 3: 3.3}
+        v.set_vector(schema._get_object(), vec)
+        ret = v.get_vector(schema._get_object())
+        for k in vec.keys():
+            assert math.isclose(np.float16(vec[k]), ret[k], abs_tol=1e-6)
+
+
+class TestQueryContext:
+    def test_init(self):
+        ctx = QueryContext(topk=10)
+        assert ctx.topk == 10
+        assert ctx.queries == []
+        assert ctx.filter is None
+        assert ctx.reranker is None
+        assert ctx.output_fields is None
+        assert ctx.include_vector is False
+        assert ctx.core_vectors == []
+
+    def test_properties(self):
+        queries = [VectorQuery(field_name="test")]
+        reranker = RrfReRanker()
+        output_fields = ["field1", "field2"]
+
+        ctx = QueryContext(
+            topk=5,
+            filter="test_filter",
+            include_vector=True,
+            queries=queries,
+            output_fields=output_fields,
+            reranker=reranker,
+        )
+
+        assert ctx.topk == 5
+        assert ctx.queries == queries
+        assert ctx.filter == "test_filter"
+        assert ctx.reranker == reranker
+        assert ctx.output_fields == output_fields
+        assert ctx.include_vector is True
+
+    def test_core_vectors_setter(self):
+        ctx = QueryContext(topk=10)
+        core_vectors = [MagicMock()]
+        ctx.core_vectors = core_vectors
+        assert ctx.core_vectors == core_vectors
+
+
+class TestNoVectorQueryExecutor:
+    def test_init(self):
+        schema = MockCollectionSchema()
+        executor = NoVectorQueryExecutor(schema)
+        assert isinstance(executor, QueryExecutor)
+
+    def test_do_validate_with_queries(self):
+        schema = MockCollectionSchema()
+        executor = NoVectorQueryExecutor(schema)
+        ctx = QueryContext(topk=10, queries=[VectorQuery(field_name="test")])
+
+        with pytest.raises(
+            ValueError, match="Collection does not support query with vector or id"
+        ):
+            executor._do_validate(ctx)
+
+    def test_do_validate_without_queries(self):
+        schema = MockCollectionSchema()
+        executor = NoVectorQueryExecutor(schema)
+        ctx = QueryContext(topk=10)
+
+        executor._do_validate(ctx)
+
+    def test_do_build(self):
+        schema = MockCollectionSchema()
+        executor = NoVectorQueryExecutor(schema)
+        ctx = QueryContext(topk=5, filter="test_filter")
+
+        result = executor._do_build(ctx, MagicMock())
+        assert len(result) == 1
+        assert result[0].topk == 5
+        assert result[0].filter == "test_filter"
+
+
+class TestSingleVectorQueryExecutor:
+    def test_init(self):
+        schema = MockCollectionSchema()
+        executor = SingleVectorQueryExecutor(schema)
+        assert isinstance(executor, NoVectorQueryExecutor)
+
+    def test_do_validate_multiple_queries(self):
+        schema = MockCollectionSchema()
+        executor = SingleVectorQueryExecutor(schema)
+        queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
+        ctx = QueryContext(topk=10, queries=queries)
+
+        with pytest.raises(
+            ValueError,
+            match="Collection has only one vector field, cannot query with multiple vectors",
+        ):
+            executor._do_validate(ctx)
+
+    def test_do_build_without_queries(self):
+        schema = MockCollectionSchema()
+        executor = SingleVectorQueryExecutor(schema)
+        ctx = QueryContext(topk=5)
+
+        result = executor._do_build(ctx, MagicMock())
+        assert len(result) == 1
+        assert result[0].topk == 5
+
+
+class TestMultiVectorQueryExecutor:
+    def test_init(self):
+        schema = MockCollectionSchema()
+        executor = MultiVectorQueryExecutor(schema)
+        assert isinstance(executor, SingleVectorQueryExecutor)
+
+    def test_do_validate_multiple_queries_without_reranker(self):
+        schema = MockCollectionSchema()
+        executor = MultiVectorQueryExecutor(schema)
+        queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
+        ctx = QueryContext(topk=10, queries=queries)
+
+        with pytest.raises(
+            ValueError, match="Reranker is required for multi-vector query"
+        ):
+            executor._do_validate(ctx)
+
+    def test_do_validate_multiple_queries_with_reranker(self):
+        schema = MockCollectionSchema()
+        executor = MultiVectorQueryExecutor(schema)
+        queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
+        reranker = RrfReRanker()
+        ctx = QueryContext(topk=10, queries=queries, reranker=reranker)
+
+        executor._do_validate(ctx)
+
+
+class TestQueryExecutorFactory:
+    def test_create_no_vectors(self):
+        schema = MockCollectionSchema()
+        executor = QueryExecutorFactory.create(schema)
+        assert isinstance(executor, NoVectorQueryExecutor)
+
+    def test_create_single_vector(self):
+        schema = MockCollectionSchema(vectors=MockVectorSchema())
+        executor = QueryExecutorFactory.create(schema)
+        assert isinstance(executor, SingleVectorQueryExecutor)
+
+    def test_create_multiple_vectors(self):
+        schema = MockCollectionSchema(
+            vectors={"test1": MockVectorSchema(), "test2": MockVectorSchema()}
+        )
+        executor = QueryExecutorFactory.create(schema)
+        assert isinstance(executor, MultiVectorQueryExecutor)

python/tests/test_reranker.py

@@ -0,0 +1,184 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from unittest.mock import patch
+import pytest
+import math
+
+from zvec import RrfReRanker, WeightedReRanker, Doc, MetricType
+
+
+# ----------------------------
+# RrfRanker Test Case
+# ----------------------------
+class TestRrfReRanker:
+    def test_init(self):
+        reranker = RrfReRanker(
+            query="test", topn=5, rerank_field="content", rank_constant=100
+        )
+        assert reranker.query == "test"
+        assert reranker.topn == 5
+        assert reranker.rerank_field == "content"
+        assert reranker.rank_constant == 100
+
+    def test_rrf_score(self):
+        reranker = RrfReRanker(query="test", rank_constant=60)
+        # 根据公式 1.0 / (k + rank + 1)，其中k=60
+        assert reranker._rrf_score(0) == 1.0 / (60 + 0 + 1)
+        assert reranker._rrf_score(1) == 1.0 / (60 + 1 + 1)
+        assert reranker._rrf_score(10) == 1.0 / (60 + 10 + 1)
+
+    def test_rerank(self):
+        reranker = RrfReRanker(query="test", topn=3)
+
+        doc1 = Doc(id="1", score=0.8)
+        doc2 = Doc(id="2", score=0.7)
+        doc3 = Doc(id="3", score=0.9)
+        doc4 = Doc(id="4", score=0.6)
+
+        query_results = {"vector1": [doc1, doc2, doc3], "vector2": [doc3, doc1, doc4]}
+
+        results = reranker.rerank(query_results)
+
+        assert len(results) <= reranker.topn
+
+        for doc in results:
+            assert hasattr(doc, "score")
+
+        scores = [doc.score for doc in results]
+        assert scores == sorted(scores, reverse=True)
+
+
+# ----------------------------
+# WeightedRanker Test Case
+# ----------------------------
+class TestWeightedReRanker:
+    def test_init(self):
+        weights = {"vector1": 0.7, "vector2": 0.3}
+        reranker = WeightedReRanker(
+            query="test",
+            topn=5,
+            rerank_field="content",
+            metric=MetricType.L2,
+            weights=weights,
+        )
+        assert reranker.query == "test"
+        assert reranker.topn == 5
+        assert reranker.rerank_field == "content"
+        assert reranker.metric == MetricType.L2
+        assert reranker.weights == weights
+
+    def test_normalize_score(self):
+        reranker = WeightedReRanker(query="test")
+
+        score = reranker._normalize_score(1.0, MetricType.L2)
+        expected = 1.0 - 2 * math.atan(1.0) / math.pi
+        assert score == expected
+
+        score = reranker._normalize_score(1.0, MetricType.IP)
+        expected = 0.5 + math.atan(1.0) / math.pi
+        assert score == expected
+
+        score = reranker._normalize_score(1.0, MetricType.COSINE)
+        expected = 1.0 - 1.0 / 2.0
+        assert score == expected
+
+        with pytest.raises(ValueError, match="Unsupported metric type"):
+            reranker._normalize_score(1.0, "unsupported_metric")
+
+    def test_rerank(self):
+        weights = {"vector1": 0.7, "vector2": 0.3}
+        reranker = WeightedReRanker(
+            query="test", topn=3, weights=weights, metric=MetricType.L2
+        )
+
+        doc1 = Doc(id="1", score=0.8)
+        doc2 = Doc(id="2", score=0.7)
+        doc3 = Doc(id="3", score=0.9)
+
+        query_results = {"vector1": [doc1, doc2], "vector2": [doc2, doc3]}
+
+        results = reranker.rerank(query_results)
+
+        assert len(results) <= reranker.topn
+
+        for doc in results:
+            assert hasattr(doc, "score")
+
+        scores = [doc.score for doc in results]
+        assert scores == sorted(scores, reverse=True)
+
+
+# # ----------------------------
+# # QwenReRanker Test Case
+# # ----------------------------
+# class TestQwenReRanker:
+#     def test_init_without_query(self):
+#         with pytest.raises(ValueError):
+#             QwenReRanker()
+#
+#     def test_init_without_api_key(self):
+#         with patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""}):
+#             with pytest.raises(ValueError, match="DashScope API key is required"):
+#                 QwenReRanker(query="test")
+#
+#     @patch.dict(os.environ, {"DASHSCOPE_API_KEY": "test_key"})
+#     def test_init_with_env_api_key(self):
+#         reranker = QwenReRanker(query="test")
+#         assert reranker.query == "test"
+#         assert reranker._api_key == "test_key"
+#
+#     def test_model_property(self):
+#         reranker = QwenReRanker(query="test", api_key="test_key")
+#         assert reranker.model == "gte-rerank-v2"
+#
+#         reranker = QwenReRanker(query="test", model="custom-model", api_key="test_key")
+#         assert reranker.model == "custom-model"
+#
+#     def test_rerank_empty_results(self):
+#         reranker = QwenReRanker(query="test", api_key="test_key")
+#         results = reranker.rerank({})
+#         assert results == []
+#
+#     def test_rerank_no_documents(self):
+#         reranker = QwenReRanker(query="test", api_key="test_key")
+#         query_results = {"vector1": [Doc(id="1")]}
+#         with pytest.raises(ValueError, match="No documents to rerank"):
+#             reranker.rerank(query_results)
+#
+#     @pytest.mark.skip(reason="Qwen ReRanker is not available in CI")
+#     def test_rerank_success(self):
+#         reranker = QwenReRanker(
+#             topn=3,
+#             query="test",
+#             api_key="*",
+#             rerank_field="content",
+#         )
+#         query_results = {
+#             "vector1": [
+#                 Doc(id="1", fields={"content": "This is a test document."}),
+#                 Doc(id="2", fields={"content": "Another test document."}),
+#                 Doc(id="3", fields={"content": "Yet another test document."}),
+#                 Doc(id="4", fields={"content": "One more test document."}),
+#             ],
+#             "vector2": [
+#                 Doc(id="5", fields={"content": "This is a test document2."}),
+#                 Doc(id="6", fields={"content": "Another test document2."}),
+#                 Doc(id="7", fields={"content": "Yet another test document2."}),
+#                 Doc(id="8", fields={"content": "One more test document2."}),
+#             ],
+#         }
+#         results = reranker.rerank(query_results)
+#         assert len(results) == 3

python/tests/test_schema.py

@@ -0,0 +1,246 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import pytest
+from zvec import (
+    CollectionSchema,
+    CollectionStats,
+    FieldSchema,
+    VectorSchema,
+    HnswIndexParam,
+    InvertIndexParam,
+    DataType,
+    IndexType,
+    MetricType,
+)
+
+# ----------------------------
+# FieldSchema Test Case
+# ----------------------------
+
+
+class TestFieldSchema:
+    def test_default(self):
+        field = FieldSchema("field", data_type=DataType.FLOAT)
+        assert field.name == "field"
+        assert field.data_type == DataType.FLOAT
+        assert field.nullable is False
+        assert field.index_param is None
+
+    def test_custom(self):
+        field_1 = FieldSchema(
+            name="float",
+            data_type=DataType.FLOAT,
+            nullable=True,
+            index_param=InvertIndexParam(),
+        )
+        assert field_1.name == "float"
+        assert field_1.data_type == DataType.FLOAT
+        assert field_1.nullable is True
+        assert field_1.index_param.enable_range_optimization is False
+
+        field_2 = FieldSchema(
+            name="str",
+            data_type=DataType.STRING,
+            nullable=True,
+            index_param=InvertIndexParam(enable_range_optimization=True),
+        )
+        assert field_2.name == "str"
+        assert field_2.data_type == DataType.STRING
+        assert field_2.nullable is True
+        assert field_2.index_param.enable_range_optimization is True
+
+    def test_readonly(self):
+        field = FieldSchema(
+            name="float",
+            data_type=DataType.FLOAT,
+            nullable=True,
+            index_param=InvertIndexParam(),
+        )
+
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            field.index_param = InvertIndexParam(enable_range_optimization=True)
+
+
+# ----------------------------
+# VectorSchema Test Case
+# ----------------------------
+class TestVectorSchema:
+    def test_default(self):
+        field = VectorSchema("vector", data_type=DataType.VECTOR_FP32, dimension=128)
+        assert field.name == "vector"
+        assert field.data_type == DataType.VECTOR_FP32
+        assert field.dimension == 128
+        assert field.index_param is not None
+        assert field.index_param.type == IndexType.FLAT
+        assert field.index_param.metric_type == MetricType.IP
+
+    def test_custom(self):
+        field = VectorSchema(
+            name="vector",
+            data_type=DataType.VECTOR_INT8,
+            dimension=512,
+            index_param=HnswIndexParam(
+                metric_type=MetricType.COSINE, m=15, ef_construction=300
+            ),
+        )
+        assert field.name == "vector"
+        assert field.data_type == DataType.VECTOR_INT8
+        assert field.index_param.metric_type == MetricType.COSINE
+        assert field.index_param.m == 15
+        assert field.index_param.ef_construction == 300
+
+    def test_readonly(self):
+        field = VectorSchema(
+            name="vector",
+            dimension=128,
+            data_type=DataType.VECTOR_INT8,
+        )
+
+        import sys
+
+        if sys.version_info >= (3, 11):
+            match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
+        else:
+            match_pattern = r"can't set attribute"
+        with pytest.raises(AttributeError, match=match_pattern):
+            field.dimension = 4
+
+
+# ----------------------------
+# CollectionSchema Test Case
+# ----------------------------
+class TestCollectionSchema:
+    def test_collection_schema_with_single_field(self):
+        collection_schema = CollectionSchema(
+            name="test_collection",
+            fields=FieldSchema(
+                name="id",
+                data_type=DataType.INT64,
+                index_param=InvertIndexParam(),
+                nullable=False,
+            ),
+            vectors=VectorSchema(
+                name="vector",
+                data_type=DataType.VECTOR_INT8,
+                dimension=128,
+                index_param=HnswIndexParam(),
+            ),
+        )
+
+        assert collection_schema is not None
+        assert collection_schema.name == "test_collection"
+        assert len(collection_schema.fields) == 1
+        assert len(collection_schema.vectors) == 1
+
+        field = collection_schema.field("id")
+        assert field is not None
+        assert field.name == "id"
+        assert field.data_type == DataType.INT64
+        assert not field.nullable
+        assert field.index_param.type == IndexType.INVERT
+        assert not field.index_param.enable_range_optimization
+
+        vector = collection_schema.vector("vector")
+        assert vector is not None
+        assert vector.name == "vector"
+        assert vector.data_type == DataType.VECTOR_INT8
+        assert vector.dimension == 128
+        assert vector.index_param.type == IndexType.HNSW
+        assert vector.index_param.m == 100
+        assert vector.index_param.ef_construction == 500
+        assert vector.index_param.metric_type == MetricType.IP
+
+    def test_collection_schema_with_multi_fields(self):
+        collection_schema = CollectionSchema(
+            name="test_collection",
+            fields=[
+                FieldSchema(
+                    "id",
+                    DataType.INT64,
+                    nullable=False,
+                    index_param=InvertIndexParam(enable_range_optimization=True),
+                ),
+                FieldSchema(
+                    "name",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=InvertIndexParam(),
+                ),
+                FieldSchema(
+                    "weight",
+                    DataType.INT32,
+                    nullable=True,
+                ),
+            ],
+            vectors=[
+                VectorSchema(
+                    "dense",
+                    DataType.VECTOR_FP32,
+                    dimension=128,
+                    index_param=HnswIndexParam(),
+                ),
+                VectorSchema(
+                    "sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
+                ),
+            ],
+        )
+        assert collection_schema is not None
+        assert collection_schema.name == "test_collection"
+        assert len(collection_schema.fields) == 3
+        assert len(collection_schema.vectors) == 2
+
+        field_id = collection_schema.field("id")
+        assert field_id is not None
+        assert field_id.name == "id"
+        assert field_id.data_type == DataType.INT64
+        assert not field_id.nullable
+        assert field_id.index_param.type == IndexType.INVERT
+
+        dense = collection_schema.vector("dense")
+        assert dense is not None
+        assert dense.name == "dense"
+        assert dense.data_type == DataType.VECTOR_FP32
+        assert dense.dimension == 128
+        assert dense.index_param.type == IndexType.HNSW
+
+        sparse = collection_schema.vector("sparse")
+        assert sparse is not None
+        assert sparse.name == "sparse"
+        assert sparse.data_type == DataType.SPARSE_VECTOR_FP32
+        assert sparse.dimension == 0
+        assert sparse.index_param.type == IndexType.HNSW
+
+        assert str(collection_schema) is not None
+
+
+# ----------------------------
+# CollectionStats Test Case
+# ----------------------------
+class TestCollectionStats:
+    """
+    The constructor of CollectionStats is not provided.
+    It can only be obtained through collection.stats()
+    """
+
+    def test_collection_stats(self):
+        stats = CollectionStats()
+        assert stats is not None

python/tests/test_typing.py

@@ -0,0 +1,141 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import pytest
+from zvec import (
+    DataType,
+    IndexType,
+    MetricType,
+    QuantizeType,
+    Status,
+    StatusCode,
+)
+
+
+# ----------------------------
+# Enum Test Case
+# ----------------------------
+@pytest.mark.parametrize(
+    "member, name",
+    [
+        (DataType.FLOAT, "FLOAT"),
+        (IndexType.HNSW, "HNSW"),
+        (MetricType.COSINE, "COSINE"),
+        (QuantizeType.INT8, "INT8"),
+        (StatusCode.OK, "OK"),
+    ],
+)
+def test_enum_names(member, name):
+    assert member.name == name
+
+
+@pytest.mark.parametrize(
+    "member, value",
+    [
+        (DataType.FLOAT, 8),
+        (IndexType.HNSW, 1),
+        (MetricType.COSINE, 3),
+        (QuantizeType.INT8, 2),
+        (StatusCode.OK, 0),
+    ],
+)
+def test_enum_values(member, value):
+    assert member.value == value
+
+
+@pytest.mark.parametrize("member", ["L2", "IP", "COSINE"])
+def test_metric_type_has_member(member):
+    assert member in MetricType.__members__
+
+
+@pytest.mark.parametrize(
+    "member",
+    [
+        "STRING",
+        "BOOL",
+        "INT32",
+        "INT64",
+        "FLOAT",
+        "DOUBLE",
+        "UINT32",
+        "UINT64",
+        "VECTOR_FP16",
+        "VECTOR_FP32",
+        "VECTOR_FP64",
+        "VECTOR_INT8",
+        "SPARSE_VECTOR_FP32",
+        "SPARSE_VECTOR_FP16",
+        "ARRAY_STRING",
+        "ARRAY_INT32",
+        "ARRAY_INT64",
+        "ARRAY_FLOAT",
+        "ARRAY_DOUBLE",
+        "ARRAY_BOOL",
+        "ARRAY_UINT32",
+        "ARRAY_UINT64",
+    ],
+)
+def test_data_type_has_member(member):
+    assert member in DataType.__members__
+
+
+@pytest.mark.parametrize("member", ["HNSW", "IVF", "FLAT", "INVERT"])
+def test_index_type_has_member(member):
+    assert member in IndexType.__members__
+
+
+@pytest.mark.parametrize("member", ["FP16", "INT8", "INT4", "UNDEFINED"])
+def test_quantize_type_has_member(member):
+    assert member in QuantizeType.__members__
+
+
+@pytest.mark.parametrize(
+    "member",
+    [
+        "OK",
+        "UNKNOWN",
+        "NOT_FOUND",
+        "ALREADY_EXISTS",
+        "INVALID_ARGUMENT",
+        "PERMISSION_DENIED",
+        "FAILED_PRECONDITION",
+        "RESOURCE_EXHAUSTED",
+        "UNAVAILABLE",
+        "INTERNAL_ERROR",
+        "NOT_SUPPORTED",
+    ],
+)
+def test_status_code_has_member(member):
+    assert member in StatusCode.__members__
+
+
+# ----------------------------
+# Status Test Case
+# ----------------------------
+class TestStatus:
+    def test_status_code(self):
+        status = Status(StatusCode.OK)
+        assert status.code() == StatusCode.OK
+
+    def test_status_message(self):
+        status = Status(StatusCode.OK, "OK")
+        assert status.message() == "OK"
+
+        status = Status(StatusCode.NOT_FOUND, "Not Found")
+        assert status.message() == "Not Found"
+
+    def test_status_ok(self):
+        status = Status(StatusCode.OK)
+        assert status.ok()

python/tests/test_util.py

@@ -0,0 +1,94 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from zvec import require_module
+
+
+# ----------------------------
+# require_module func Test Case
+# ----------------------------
+def test_require_module_success():
+    module = require_module("os")
+    assert module is not None
+    assert hasattr(module, "path")
+
+
+def test_require_module_with_submodule_success():
+    module = require_module("os.path")
+    assert module is not None
+    assert hasattr(module, "join")
+
+
+def test_require_module_import_error():
+    with pytest.raises(ImportError) as exc_info:
+        require_module("nonexistent_module")
+
+    exception_msg = str(exc_info.value)
+    assert "Required package 'nonexistent_module' is not installed." in exception_msg
+
+
+def test_require_module_with_mitigation_import_error():
+    with pytest.raises(ImportError) as exc_info:
+        require_module("nonexistent_module.submodule", mitigation="custom_package")
+
+    exception_msg = str(exc_info.value)
+    assert "Required package 'custom_package' is not installed." in exception_msg
+    assert (
+        "Module 'nonexistent_module.submodule' is part of 'nonexistent_module'"
+        in exception_msg
+    )
+    assert "please pip install 'custom_package'." in exception_msg
+
+
+def test_require_module_submodule_import_error():
+    with pytest.raises(ImportError) as exc_info:
+        require_module("os.nonexistent_submodule")
+
+    exception_msg = str(exc_info.value)
+    assert (
+        "Required package 'os.nonexistent_submodule' is not installed." in exception_msg
+    )
+    assert "Module 'os.nonexistent_submodule' is part of 'os'" in exception_msg
+    assert "please pip install 'os'." in exception_msg
+
+
+@patch("importlib.import_module")
+def test_require_module_wraps_original_exception(mock_import_module):
+    original_exception = ImportError("Original error")
+    mock_import_module.side_effect = original_exception
+
+    with pytest.raises(ImportError) as exc_info:
+        require_module("some_module")
+
+    assert exc_info.value.__cause__ is original_exception
+
+
+@patch("importlib.import_module")
+def test_require_module_calls_importlib(mock_import_module):
+    mock_module = MagicMock()
+    mock_import_module.return_value = mock_module
+
+    result = require_module("test_module")
+
+    mock_import_module.assert_called_once_with("test_module")
+    assert result is mock_module
+
+
+def test_require_module_with_openai():
+    with pytest.raises(ImportError) as exc_info:
+        require_module("openai")

python/zvec/__init__.py

@@ -0,0 +1,133 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from importlib.metadata import PackageNotFoundError
+
+
+# ==============================
+# Public API — grouped by category
+# ==============================
+
+from . import model as model
+
+# —— Extensions & typing ——
+from .extension import DenseEmbeddingFunction, ReRanker, RrfReRanker, WeightedReRanker
+from .model import param as param
+from .model import schema as schema
+
+# —— Core data structures ——
+from .model.collection import Collection
+from .model.doc import Doc
+
+# —— Query & index parameters ——
+from .model.param import (
+    AddColumnOption,
+    AlterColumnOption,
+    CollectionOption,
+    FlatIndexParam,
+    HnswIndexParam,
+    HnswQueryParam,
+    IndexOption,
+    InvertIndexParam,
+    IVFIndexParam,
+    IVFQueryParam,
+    OptimizeOption,
+)
+from .model.param.vector_query import VectorQuery
+
+# —— Schema & field definitions ——
+from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
+
+# —— tools ——
+from .tool import require_module
+from .typing import (
+    DataType,
+    IndexType,
+    MetricType,
+    QuantizeType,
+    Status,
+    StatusCode,
+)
+from .typing.enum import LogLevel, LogType
+
+# —— lifecycle ——
+from .zvec import create_and_open, init, open
+
+# ==============================
+# Public interface declaration
+# ==============================
+__all__ = [
+    # Zvec functions
+    "create_and_open",
+    "init",
+    "open",
+    # Core classes
+    "Collection",
+    "Doc",
+    # Schema
+    "CollectionSchema",
+    "FieldSchema",
+    "VectorSchema",
+    "CollectionStats",
+    # Parameters
+    "VectorQuery",
+    "InvertIndexParam",
+    "HnswIndexParam",
+    "FlatIndexParam",
+    "IVFIndexParam",
+    "CollectionOption",
+    "IndexOption",
+    "OptimizeOption",
+    "AddColumnOption",
+    "AlterColumnOption",
+    "HnswQueryParam",
+    "IVFQueryParam",
+    # Extensions
+    "ReRanker",
+    "DenseEmbeddingFunction",
+    "RrfReRanker",
+    "WeightedReRanker",
+    # Typing
+    "DataType",
+    "MetricType",
+    "QuantizeType",
+    "IndexType",
+    "LogLevel",
+    "LogType",
+    "Status",
+    "StatusCode",
+    # Tools
+    "require_module",
+]
+
+# ==============================
+# Version handling
+# ==============================
+__version__: str
+
+try:
+    from importlib.metadata import version
+except ImportError:
+    from importlib_metadata import version  # Python < 3.8
+
+try:
+    __version__ = version("zvec")
+except Exception:
+    __version__ = "unknown"

python/zvec/__init__.pyi

@@ -0,0 +1,172 @@
+"""
+Zvec core module
+"""
+
+from __future__ import annotations
+
+import collections
+
+from . import typing
+from .extension import ReRanker, RrfReRanker, WeightedReRanker
+from .extension.embedding import DenseEmbeddingFunction
+from .model import param, schema
+from .model.collection import Collection
+from .model.doc import Doc
+from .model.param import (
+    AddColumnOption,
+    AlterColumnOption,
+    CollectionOption,
+    FlatIndexParam,
+    HnswIndexParam,
+    HnswQueryParam,
+    IndexOption,
+    InvertIndexParam,
+    IVFIndexParam,
+    IVFQueryParam,
+    OptimizeOption,
+)
+from .model.param.vector_query import VectorQuery
+from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
+from .tool import require_module
+from .typing import (
+    DataType,
+    IndexType,
+    MetricType,
+    QuantizeType,
+    Status,
+    StatusCode,
+)
+from .typing.enum import LogLevel, LogType
+from .zvec import create_and_open, init, open
+
+__all__: list = [
+    "AddColumnOption",
+    "AlterColumnOption",
+    "Collection",
+    "CollectionOption",
+    "CollectionSchema",
+    "CollectionStats",
+    "DataType",
+    "DenseEmbeddingFunction",
+    "DenseEmbeddingFunction",
+    "Doc",
+    "FieldSchema",
+    "FlatIndexParam",
+    "HnswIndexParam",
+    "HnswQueryParam",
+    "IVFIndexParam",
+    "IVFQueryParam",
+    "IndexOption",
+    "IndexType",
+    "InvertIndexParam",
+    "LogLevel",
+    "LogType",
+    "MetricType",
+    "OptimizeOption",
+    "QuantizeType",
+    "ReRanker",
+    "ReRanker",
+    "RrfReRanker",
+    "Status",
+    "StatusCode",
+    "VectorQuery",
+    "VectorSchema",
+    "WeightedReRanker",
+    "create_and_open",
+    "init",
+    "open",
+    "require_module",
+]
+
+class _Collection:
+    @staticmethod
+    def CreateAndOpen(
+        arg0: str, arg1: schema._CollectionSchema, arg2: param.CollectionOption
+    ) -> _Collection: ...
+    @staticmethod
+    def Open(arg0: str, arg1: param.CollectionOption) -> _Collection: ...
+    def AddColumn(
+        self,
+        arg0: str,
+        arg1: schema._FieldSchema,
+        arg2: str,
+        arg3: param.AddColumnOption,
+    ) -> None: ...
+    def AlterColumn(
+        self,
+        arg0: str,
+        arg1: str,
+        arg2: schema._FieldSchema,
+        arg3: param.AlterColumnOption,
+    ) -> None: ...
+    def CreateIndex(
+        self, arg0: str, arg1: param.IndexParam, arg2: param.IndexOption
+    ) -> None: ...
+    def Delete(self, arg0: collections.abc.Sequence[str]) -> list[typing.Status]: ...
+    def DeleteByFilter(self, arg0: str) -> None: ...
+    def Destroy(self) -> None: ...
+    def DropColumn(self, arg0: str) -> None: ...
+    def DropIndex(self, arg0: str) -> None: ...
+    def Fetch(self, arg0: collections.abc.Sequence[str]) -> dict[str, _Doc]: ...
+    def Flush(self) -> None: ...
+    def GroupByQuery(self, arg0: ...) -> list[...]: ...
+    def Insert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
+    def Optimize(self, arg0: param.OptimizeOption) -> None: ...
+    def Options(self) -> param.CollectionOption: ...
+    def Path(self) -> str: ...
+    def Query(self, arg0: param._VectorQuery) -> list[_Doc]: ...
+    def Schema(self) -> schema._CollectionSchema: ...
+    def Stats(self) -> schema.CollectionStats: ...
+    def Update(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
+    def Upsert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
+    def __getstate__(self) -> tuple: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+
+class _Doc:
+    def __getstate__(self) -> bytes: ...
+    def __init__(self) -> None: ...
+    def __setstate__(self, arg0: bytes) -> None: ...
+    def field_names(self) -> list[str]: ...
+    def get_any(self, arg0: str, arg1: typing.DataType) -> typing.Any: ...
+    def has_field(self, arg0: str) -> bool: ...
+    def pk(self) -> str: ...
+    def score(self) -> float: ...
+    def set_any(self, arg0: str, arg1: typing.DataType, arg2: typing.Any) -> bool: ...
+    def set_pk(self, arg0: str) -> None: ...
+    def set_score(self, arg0: typing.SupportsFloat) -> None: ...
+
+class _DocOp:
+    """
+    Members:
+
+      INSERT
+
+      UPDATE
+
+      DELETE
+
+      UPSERT
+    """
+
+    DELETE: typing.ClassVar[_DocOp]  # value = <_DocOp.DELETE: 3>
+    INSERT: typing.ClassVar[_DocOp]  # value = <_DocOp.INSERT: 0>
+    UPDATE: typing.ClassVar[_DocOp]  # value = <_DocOp.UPDATE: 2>
+    UPSERT: typing.ClassVar[_DocOp]  # value = <_DocOp.UPSERT: 1>
+    __members__: typing.ClassVar[
+        dict[str, _DocOp]
+    ]  # value = {'INSERT': <_DocOp.INSERT: 0>, 'UPDATE': <_DocOp.UPDATE: 2>, 'DELETE': <_DocOp.DELETE: 3>, 'UPSERT': <_DocOp.UPSERT: 1>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...

python/zvec/common/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .constants import DenseVectorType, SparseVectorType, VectorType
+
+__all__ = ["DenseVectorType", "SparseVectorType", "VectorType"]

python/zvec/common/constants.py

@@ -0,0 +1,22 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional, Union
+
+import numpy as np
+
+DenseVectorType = Union[list[float], list[int], np.ndarray]
+SparseVectorType = dict[int, float]
+VectorType = Optional[Union[DenseVectorType, SparseVectorType]]

python/zvec/executor/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .query_executor import (
+    QueryContext,
+    QueryExecutor,
+    QueryExecutorFactory,
+)
+
+__all__ = [
+    "QueryContext",
+    "QueryExecutor",
+    "QueryExecutorFactory",
+]

python/zvec/executor/query_executor.py

@@ -0,0 +1,307 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Optional, Union, final
+
+import numpy as np
+from _zvec import _Collection
+from _zvec.param import _VectorQuery
+
+from ..extension import ReRanker, RrfReRanker, WeightedReRanker
+from ..model.convert import convert_to_py_doc
+from ..model.doc import Doc
+from ..model.param.vector_query import VectorQuery
+from ..model.schema import CollectionSchema
+from ..typing import DataType
+
+__all__ = [
+    "QueryContext",
+    "QueryExecutor",
+    "QueryExecutorFactory",
+]
+
+DTYPE_MAP = {
+    DataType.VECTOR_FP16.value: np.float16,
+    DataType.VECTOR_FP32.value: np.float32,
+    DataType.VECTOR_FP64.value: np.float64,
+    DataType.VECTOR_INT8.value: np.int8,
+}
+
+
+def convert_to_numpy(vec: Union[list, np.ndarray], dtype: np.dtype) -> np.ndarray:
+    if isinstance(vec, np.ndarray):
+        if vec.dtype == dtype and vec.ndim == 1:
+            return vec
+        return np.asarray(vec, dtype=dtype).flatten()
+
+    try:
+        arr = np.asarray(vec, dtype=dtype)
+        if arr.ndim != 1:
+            arr = arr.flatten()
+        return arr
+    except (ValueError, TypeError) as e:
+        raise TypeError(
+            f"Cannot convert input to 1D numpy array with dtype={dtype}: {type(vec)}"
+        ) from e
+
+
+class QueryContext:
+    def __init__(
+        self,
+        topk: int,
+        filter: Optional[str] = None,
+        include_vector: bool = False,
+        queries: Optional[list[VectorQuery]] = None,
+        output_fields: Optional[list[str]] = None,
+        reranker: Optional[ReRanker] = None,
+    ):
+        # query param
+        self._filter = filter
+        self._queries = queries or []
+        self._topk = topk
+        self._include_vector = include_vector
+        self._output_fields = output_fields
+
+        # reranker
+        self._reranker = reranker
+
+        # core vectors
+        self._core_vectors = []
+
+    @property
+    def topk(self):
+        return self._topk
+
+    @property
+    def queries(self):
+        return self._queries
+
+    @property
+    def filter(self):
+        return self._filter
+
+    @property
+    def reranker(self):
+        return self._reranker
+
+    @property
+    def output_fields(self):
+        return self._output_fields
+
+    @property
+    def include_vector(self):
+        return self._include_vector
+
+    @property
+    def core_vectors(self):
+        return self._core_vectors
+
+    @core_vectors.setter
+    def core_vectors(self, core_vectors: list[_VectorQuery]):
+        self._core_vectors = core_vectors
+
+
+class QueryExecutor(ABC):
+    def __init__(self, schema: CollectionSchema):
+        self._schema = schema
+        self._concurrency = max(1, int(os.getenv("ZVEC_QUERY_CONCURRENCY", "1")))
+
+    @abstractmethod
+    def _do_validate(self, ctx: QueryContext) -> None:
+        pass
+
+    @abstractmethod
+    def _do_build(
+        self, ctx: QueryContext, collection: _Collection
+    ) -> list[_VectorQuery]:
+        pass
+
+    def _do_build_query_wo_vector(self, ctx: QueryContext) -> _VectorQuery:
+        core_vector = _VectorQuery()
+        core_vector.topk = ctx.topk
+        core_vector.include_vector = ctx.include_vector
+        if ctx.filter:
+            core_vector.filter = ctx.filter
+        if ctx.output_fields:
+            core_vector.output_fields = ctx.output_fields
+        return core_vector
+
+    def _do_build_query_with_vector(
+        self, ctx: QueryContext, query: VectorQuery, collection: _Collection
+    ) -> _VectorQuery:
+        core_vector = self._do_build_query_wo_vector(ctx)
+        core_vector.field_name = query.field_name
+        if query.param:
+            core_vector.query_params = query.param
+
+        vector_schema = (
+            self._schema.vector(query.field_name) if query else self._schema.vectors[0]
+        )
+
+        if vector_schema is None:
+            raise ValueError("No vector field found")
+
+        # set output_fields
+        core_vector.output_fields = ctx.output_fields
+
+        # set vector
+        if query.has_vector():
+            vec_data = query.vector
+        else:
+            fetched = collection.Fetch([query.id])
+            doc = next(iter(fetched.values()))
+            if not doc:
+                return core_vector
+            vec_data = doc.get_any(vector_schema.name, vector_schema.data_type)
+
+        target_dtype = DTYPE_MAP.get(vector_schema.data_type.value)
+        core_vector.set_vector(
+            vector_schema._get_object(),
+            convert_to_numpy(vec_data, target_dtype) if target_dtype else vec_data,
+        )
+        return core_vector
+
+    def _do_execute(
+        self, vectors: list[_VectorQuery], collection: _Collection
+    ) -> dict[str, list[Doc]]:
+        query_cnt = len(vectors)
+        if query_cnt == 0:
+            raise ValueError("No query to execute")
+
+        if len(vectors) == 1 or self._concurrency == 1:
+            results = {}
+            for query in vectors:
+                docs = collection.Query(query)
+                results[query.field_name] = [
+                    convert_to_py_doc(doc, self._schema) for doc in docs
+                ]
+            return results
+
+        results = {}
+        with ThreadPoolExecutor(max_workers=self._concurrency) as executor:
+            future_to_query = {
+                executor.submit(collection.Query, query): query.field_name
+                for query in vectors
+            }
+
+            for future in as_completed(future_to_query):
+                field_name = future_to_query[future]
+                try:
+                    docs = future.result()
+                    results[field_name] = [
+                        convert_to_py_doc(doc, self._schema) for doc in docs
+                    ]
+                except Exception as e:
+                    raise e
+        return results
+
+    def _do_merge_rerank_results(
+        self, ctx: QueryContext, docs_map: dict[str, list[Doc]]
+    ) -> list[Doc]:
+        query_result_cnt = len(docs_map) if docs_map else 0
+        if query_result_cnt == 0:
+            raise ValueError("Query results is none and dost not to rerank")
+        if query_result_cnt == 1:
+            if not ctx.reranker or isinstance(
+                ctx.reranker, (RrfReRanker, WeightedReRanker)
+            ):
+                return next(iter(docs_map.values()))
+            return ctx.reranker.rerank(docs_map)
+        return ctx.reranker.rerank(docs_map)
+
+    @final
+    def execute(self, ctx: QueryContext, collection: _Collection) -> list[Doc]:
+        # 1. validate query
+        self._do_validate(ctx)
+        # 2. build query vector
+        query_vectors = self._do_build(ctx, collection)
+        if not query_vectors:
+            raise ValueError("No query to execute")
+        # 3. execute query
+        docs = self._do_execute(query_vectors, collection)
+        # 4. merge and rerank result
+        return self._do_merge_rerank_results(ctx, docs)
+
+
+class NoVectorQueryExecutor(QueryExecutor):
+    def __init__(self, schema: CollectionSchema):
+        super().__init__(schema)
+
+    def _do_validate(self, ctx: QueryContext) -> None:
+        if len(ctx.queries) > 0:
+            raise ValueError("Collection does not support query with vector or id")
+
+    def _do_build(
+        self, ctx: QueryContext, _collection: _Collection
+    ) -> list[_VectorQuery]:
+        return [self._do_build_query_wo_vector(ctx)]
+
+
+class SingleVectorQueryExecutor(NoVectorQueryExecutor):
+    def __init__(self, schema: CollectionSchema) -> None:
+        super().__init__(schema)
+
+    def _do_validate(self, ctx: QueryContext) -> None:
+        if len(ctx.queries) > 1:
+            raise ValueError(
+                "Collection has only one vector field, cannot query with multiple vectors"
+            )
+        for query in ctx.queries:
+            query._validate()
+
+    def _do_build(
+        self, ctx: QueryContext, collection: _Collection
+    ) -> list[_VectorQuery]:
+        if len(ctx.queries) == 0:
+            return [self._do_build_query_wo_vector(ctx)]
+        vectors = []
+        for query in ctx.queries:
+            vectors.append(self._do_build_query_with_vector(ctx, query, collection))
+        return vectors
+
+
+class MultiVectorQueryExecutor(SingleVectorQueryExecutor):
+    def __init__(self, schema: CollectionSchema) -> None:
+        super().__init__(schema)
+
+    def _do_validate(self, ctx: QueryContext) -> None:
+        if len(ctx.queries) > 1 and ctx.reranker is None:
+            raise ValueError("Reranker is required for multi-vector query")
+        seen_fields = set()
+        for query in ctx.queries:
+            query._validate()
+            field = query.field_name
+            if field in seen_fields:
+                raise ValueError(f"Query field name '{field}' appears more than once")
+            seen_fields.add(field)
+
+    def _do_execute(
+        self, vectors: list[_VectorQuery], collection: _Collection
+    ) -> dict[str, list[Doc]]:
+        return super()._do_execute(vectors, collection)
+
+
+class QueryExecutorFactory:
+    @staticmethod
+    def create(schema: CollectionSchema) -> QueryExecutor:
+        vectors = schema.vectors
+        if len(vectors) == 0:
+            return NoVectorQueryExecutor(schema)
+        if len(vectors) == 1:
+            return SingleVectorQueryExecutor(schema)
+        return MultiVectorQueryExecutor(schema)

python/zvec/extension/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .embedding import DenseEmbeddingFunction, QwenEmbeddingFunction
+from .rerank import QwenReRanker, ReRanker, RrfReRanker, WeightedReRanker
+
+__all__ = [
+    "DenseEmbeddingFunction",
+    "QwenEmbeddingFunction",
+    "QwenReRanker",
+    "ReRanker",
+    "RrfReRanker",
+    "WeightedReRanker",
+]

python/zvec/extension/embedding.py

@@ -0,0 +1,188 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from http import HTTPStatus
+from typing import Optional, Union
+
+from ..tool import require_module
+from ..typing import DataType
+
+
+class DenseEmbeddingFunction(ABC):
+    """Abstract base class for dense vector embedding functions.
+
+    Dense embedding functions map text to fixed-length real-valued vectors.
+    Subclasses must implement the ``embed()`` method.
+
+    Args:
+        dimension (int): Dimensionality of the output embedding vector.
+        data_type (DataType, optional): Numeric type of the embedding.
+            Defaults to ``DataType.VECTOR_FP32``.
+
+    Note:
+        This class is callable: ``embedding_func("text")`` is equivalent to
+        ``embedding_func.embed("text")``.
+    """
+
+    def __init__(self, dimension: int, data_type: DataType = DataType.VECTOR_FP32):
+        self._dimension = dimension
+        self._data_type = data_type
+
+    @property
+    def dimension(self) -> int:
+        """int: The expected dimensionality of the embedding vector."""
+        return self._dimension
+
+    @property
+    def data_type(self) -> DataType:
+        """DataType: The numeric data type of the embedding (e.g., VECTOR_FP32)."""
+        return self._data_type
+
+    @abstractmethod
+    def embed(self, text: str) -> list[Union[int, float]]:
+        """Generate a dense embedding vector for the input text.
+
+        Args:
+            text (str): Input text to embed.
+
+        Returns:
+            list[Union[int, float]]: A list of numbers representing the embedding.
+                Length must equal ``self.dimension``.
+        """
+        raise NotImplementedError
+
+    def __call__(self, text: str) -> list[Union[int, float]]:
+        return self.embed(text)
+
+
+class SparseEmbeddingFunction(ABC):
+    """Abstract base class for sparse vector embedding functions.
+
+    Sparse embedding functions map text to a dictionary of {index: weight},
+    where only non-zero dimensions are stored.
+
+    Note:
+        Subclasses must implement the ``embed()`` method.
+    """
+
+    @abstractmethod
+    def embed(self, text: str) -> dict[int, float]:
+        """Generate a sparse embedding for the input text.
+        Args:
+            text (str): Input text to embed.
+
+        Returns:
+            dict[int, float]: Mapping from dimension index to non-zero weight.
+        """
+        raise NotImplementedError
+
+
+class QwenEmbeddingFunction(DenseEmbeddingFunction):
+    """Dense embedding function using Qwen (DashScope) Text Embedding API.
+
+    This implementation uses the DashScope service to generate embeddings
+    via Qwen's text embedding models (e.g., ``text-embedding-v4``).
+
+    Args:
+        dimension (int): Desired embedding dimension (e.g., 1024).
+        model (str, optional): DashScope embedding model name.
+            Defaults to ``"text-embedding-v4"``.
+        api_key (Optional[str], optional): DashScope API key. If not provided,
+            reads from ``DASHSCOPE_API_KEY`` environment variable.
+
+    Raises:
+        ValueError: If API key is missing or input text is invalid.
+
+    Note:
+        Requires the ``dashscope`` Python package.
+        Embedding results are cached using ``functools.lru_cache`` (maxsize=10).
+    """
+
+    def __init__(
+        self,
+        dimension: int,
+        model: str = "text-embedding-v4",
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(dimension, DataType.VECTOR_FP32)
+        self._model = model
+        self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
+        if not self._api_key:
+            raise ValueError("DashScope API key is required")
+
+    @property
+    def model(self) -> str:
+        """str: The DashScope embedding model name in use."""
+        return self._model
+
+    def _connection(self):
+        dashscope = require_module("dashscope")
+        dashscope.api_key = self._api_key
+        return dashscope
+
+    @lru_cache(maxsize=10)
+    def embed(self, text: str) -> list[Union[int, float]]:
+        """
+        Generate embedding for a given text using Qwen (via DashScope).
+
+        Args:
+            text (str): Input text to embed. Must be non-empty and valid string.
+
+        Returns:
+            list[Union[int, float]]: The dense embedding vector.
+
+        Raises:
+            ValueError: If input is invalid or API response is malformed.
+            RuntimeError: If network or internal error occurs during API call.
+        """
+        if not isinstance(text, str):
+            raise TypeError(f"Expected 'text' to be str, got {type(text).__name__}")
+
+        text = text.strip()
+        if not text:
+            raise ValueError("Input text cannot be empty or whitespace only")
+
+        resp = self._connection().TextEmbedding.call(
+            model=self.model, input=text, dimension=self.dimension, output_type="dense"
+        )
+
+        if resp.status_code != HTTPStatus.OK:
+            error_msg = getattr(resp, "message", "Unknown error")
+            error_detail = f"Status={resp.status_code}, Message={error_msg}"
+            raise ValueError(f"QwenEmbedding failed: {error_detail}")
+
+        output = getattr(resp, "output", None)
+        if not isinstance(output, dict):
+            raise ValueError("Invalid response: missing or malformed 'output' field")
+
+        embeddings = output.get("embeddings")
+        if not isinstance(embeddings, list):
+            raise ValueError(
+                "Invalid response: 'embeddings' field is missing or not a list"
+            )
+
+        if len(embeddings) != 1:
+            raise ValueError(
+                f"Expected 1 embedding, got {len(embeddings)}. Response: {resp}"
+            )
+
+        first_emb = embeddings[0]
+        if not isinstance(first_emb, dict):
+            raise ValueError("Invalid response: embedding item is not a dictionary")
+
+        return list(first_emb.get("embedding"))

python/zvec/extension/rerank.py

@@ -0,0 +1,343 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import heapq
+import math
+import os
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from http import HTTPStatus
+from typing import Optional
+
+from ..model.doc import Doc
+from ..tool import require_module
+from ..typing import MetricType
+
+
+class ReRanker(ABC):
+    """Abstract base class for re-ranking search results.
+
+    Re-rankers refine the output of one or more vector queries by applying
+    a secondary scoring strategy. They are used in the ``query()`` method of
+    ``Collection`` via the ``reranker`` parameter.
+
+    Args:
+        query (Optional[str], optional): Query text used for re-ranking.
+            Required for LLM-based re-rankers. Defaults to None.
+        topn (int, optional): Number of top documents to return after re-ranking.
+            Defaults to 10.
+        rerank_field (Optional[str], optional): Field name used as input for
+            re-ranking (e.g., document title or body). Defaults to None.
+
+    Note:
+        Subclasses must implement the ``rerank()`` method.
+    """
+
+    def __init__(
+        self,
+        query: Optional[str] = None,
+        topn: int = 10,
+        rerank_field: Optional[str] = None,
+    ):
+        self._query = query
+        self._topn = topn
+        self._rerank_field = rerank_field
+
+    @property
+    def topn(self) -> int:
+        """int: Number of top documents to return after re-ranking."""
+        return self._topn
+
+    @property
+    def query(self) -> str:
+        """str: Query text used for re-ranking."""
+        return self._query
+
+    @property
+    def rerank_field(self) -> Optional[str]:
+        """Optional[str]: Field name used as re-ranking input."""
+        return self._rerank_field
+
+    @abstractmethod
+    def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
+        """Re-rank documents from one or more vector queries.
+
+        Args:
+            query_results (dict[str, list[Doc]]): Mapping from vector field name
+                to list of retrieved documents (sorted by relevance).
+
+        Returns:
+            list[Doc]: Re-ranked list of documents (length ≤ ``topn``),
+                with updated ``score`` fields.
+        """
+        raise NotImplementedError
+
+
+class RrfReRanker(ReRanker):
+    """Re-ranker using Reciprocal Rank Fusion (RRF).
+
+    RRF combines results from multiple queries without requiring relevance scores.
+    It assigns higher weight to documents that appear early in multiple result lists.
+
+    The RRF score for a document at rank ``r`` is: ``1 / (k + r + 1)``,
+    where ``k`` is the rank constant.
+
+    Args:
+        query (Optional[str], optional): Ignored by RRF. Defaults to None.
+        topn (int, optional): Number of top documents to return. Defaults to 10.
+        rerank_field (Optional[str], optional): Ignored by RRF. Defaults to None.
+        rank_constant (int, optional): Smoothing constant ``k`` in RRF formula.
+            Larger values reduce the impact of early ranks. Defaults to 60.
+    """
+
+    def __init__(
+        self,
+        query: Optional[str] = None,
+        topn: int = 10,
+        rerank_field: Optional[str] = None,
+        rank_constant: int = 60,
+    ):
+        super().__init__(query, topn, rerank_field)
+        self._rank_constant = rank_constant
+
+    @property
+    def rank_constant(self) -> int:
+        return self._rank_constant
+
+    def _rrf_score(self, rank: int):
+        return 1.0 / (self._rank_constant + rank + 1)
+
+    def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
+        """Apply Reciprocal Rank Fusion to combine multiple query results.
+
+        Args:
+            query_results (dict[str, list[Doc]]): Results from one or more vector queries.
+
+        Returns:
+            list[Doc]: Re-ranked documents with RRF scores in the ``score`` field.
+        """
+        rrf_scores: dict[str, float] = defaultdict(float)
+        id_to_doc: dict[str, Doc] = {}
+
+        for _, query_result in query_results.items():
+            for rank, doc in enumerate(query_result):
+                doc_id = doc.id
+                rrf_score = self._rrf_score(rank)
+                rrf_scores[doc_id] += rrf_score
+                if doc_id not in id_to_doc:
+                    id_to_doc[doc_id] = doc
+
+        top_docs = heapq.nlargest(self.topn, rrf_scores.items(), key=lambda x: x[1])
+        results = []
+        for doc_id, rrf_score in top_docs:
+            doc = id_to_doc[doc_id]
+            new_doc = doc._replace(score=rrf_score)
+            results.append(new_doc)
+        return results
+
+
+class WeightedReRanker(ReRanker):
+    """Re-ranker that combines scores from multiple vector fields using weights.
+
+    Each vector field's relevance score is normalized based on its metric type,
+    then scaled by a user-provided weight. Final scores are summed across fields.
+
+    Args:
+        query (Optional[str], optional): Ignored. Defaults to None.
+        topn (int, optional): Number of top documents to return. Defaults to 10.
+        rerank_field (Optional[str], optional): Ignored. Defaults to None.
+        metric (MetricType, optional): Distance metric used for score normalization.
+            Defaults to ``MetricType.L2``.
+        weights (Optional[dict[str, float]], optional): Weight per vector field.
+            Fields not listed use weight 1.0. Defaults to None.
+
+    Note:
+        Supported metrics: L2, IP, COSINE. Scores are normalized to [0, 1].
+    """
+
+    def __init__(
+        self,
+        query: Optional[str] = None,
+        topn: int = 10,
+        rerank_field: Optional[str] = None,
+        metric: MetricType = MetricType.L2,
+        weights: Optional[dict[str, float]] = None,
+    ):
+        super().__init__(query, topn, rerank_field)
+        self._weights = weights
+        self._metric = metric
+
+    @property
+    def weights(self) -> dict[str, float]:
+        """dict[str, float]: Weight mapping for vector fields."""
+        return self._weights
+
+    @property
+    def metric(self) -> MetricType:
+        """MetricType: Distance metric used for score normalization."""
+        return self._metric
+
+    def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
+        """Combine scores from multiple vector fields using weighted sum.
+
+        Args:
+            query_results (dict[str, list[Doc]]): Results per vector field.
+
+        Returns:
+            list[Doc]: Re-ranked documents with combined scores in ``score`` field.
+        """
+        weighted_scores: dict[str, float] = defaultdict(float)
+        id_to_doc: dict[str, Doc] = {}
+
+        for vector_name, query_result in query_results.items():
+            for _, doc in enumerate(query_result):
+                doc_id = doc.id
+                weighted_score = self._normalize_score(
+                    doc.score, self.metric
+                ) * self.weights.get(vector_name, 1.0)
+                weighted_scores[doc_id] += weighted_score
+                if doc_id not in id_to_doc:
+                    id_to_doc[doc_id] = doc
+
+        top_docs = heapq.nlargest(
+            self.topn, weighted_scores.items(), key=lambda x: x[1]
+        )
+        results = []
+        for doc_id, weighted_score in top_docs:
+            doc = id_to_doc[doc_id]
+            new_doc = doc._replace(score=weighted_score)
+            results.append(new_doc)
+        return results
+
+    def _normalize_score(self, score: float, metric: MetricType) -> float:
+        if metric == MetricType.L2:
+            return 1.0 - 2 * math.atan(score) / math.pi
+        if metric == MetricType.IP:
+            return 0.5 + math.atan(score) / math.pi
+        if metric == MetricType.COSINE:
+            return 1.0 - score / 2.0
+        raise ValueError("Unsupported metric type")
+
+
+class QwenReRanker(ReRanker):
+    """Re-ranker using Qwen (DashScope) LLM-based re-ranking API.
+
+    This re-ranker sends documents to the DashScope TextReRank service for
+    cross-encoder style re-ranking based on semantic relevance to the query.
+
+    Args:
+        query (str): Query text for semantic re-ranking. **Required**.
+        topn (int, optional): Number of top documents to return. Defaults to 10.
+        rerank_field (str): Field name containing document text for re-ranking.
+            **Required**.
+        model (str, optional): DashScope re-ranking model name.
+            Defaults to ``"gte-rerank-v2"``.
+        api_key (Optional[str], optional): DashScope API key. If not provided,
+            reads from ``DASHSCOPE_API_KEY`` environment variable.
+
+    Raises:
+        ValueError: If ``query`` is missing, ``rerank_field`` is missing,
+            or API key is not provided.
+
+    Note:
+        Requires the ``dashscope`` Python package.
+        Documents without content in ``rerank_field`` are skipped.
+    """
+
+    def __init__(
+        self,
+        query: Optional[str] = None,
+        topn: int = 10,
+        rerank_field: Optional[str] = None,
+        model: str = "gte-rerank-v2",
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(query, topn, rerank_field)
+        if not query:
+            raise ValueError("Query is required for reranking")
+        self._model = model
+        self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
+        if not self._api_key:
+            raise ValueError("DashScope API key is required")
+
+    @property
+    def model(self) -> str:
+        """str: DashScope re-ranking model name."""
+        return self._model
+
+    def _connection(self):
+        dashscope = require_module("dashscope")
+        dashscope.api_key = self._api_key
+        return dashscope
+
+    def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
+        """Re-rank documents using Qwen's TextReRank API.
+
+        Args:
+            query_results (dict[str, list[Doc]]): Results from vector search.
+
+        Returns:
+            list[Doc]: Re-ranked documents with relevance scores from Qwen.
+
+        Raises:
+            ValueError: If API call fails or no valid documents are found.
+        """
+        if not query_results:
+            return []
+
+        id_to_doc: dict[str, Doc] = {}
+        doc_ids = []
+        contents = []
+
+        for _, query_result in query_results.items():
+            for doc in query_result:
+                doc_id = doc.id
+                if doc_id in id_to_doc:
+                    continue
+
+                field_value = doc.field(self.rerank_field)
+                rank_content = str(field_value).strip() if field_value else ""
+                if not rank_content:
+                    continue
+
+                id_to_doc[doc_id] = doc
+                doc_ids.append(doc_id)
+                contents.append(rank_content)
+
+        if not contents:
+            raise ValueError("No documents to rerank")
+
+        resp = self._connection().TextReRank.call(
+            model=self.model,
+            query=self.query,
+            documents=list(contents),
+            top_n=self.topn,
+            return_documents=False,
+        )
+
+        if resp.status_code != HTTPStatus.OK:
+            raise ValueError(
+                f"QwenReranker failed with status {resp.status_code}: {resp.message}"
+            )
+
+        results = []
+        for item in resp.output.results:
+            idx = item.index
+            doc_id = doc_ids[idx]
+            doc = id_to_doc[doc_id]
+            new_doc = doc._replace(score=item.relevance_score)
+            results.append(new_doc)
+
+        return results

python/zvec/model/__init__.py

@@ -0,0 +1,22 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .collection import Collection
+from .doc import Doc
+from .param.vector_query import VectorQuery
+from .schema.collection_schema import CollectionSchema
+from .schema.field_schema import FieldSchema
+
+__all__ = ["Collection", "CollectionSchema", "Doc", "FieldSchema", "VectorQuery"]

python/zvec/model/collection.py

@@ -0,0 +1,381 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from _zvec import _Collection
+
+from ..executor import QueryContext, QueryExecutorFactory
+from ..extension import ReRanker
+from ..typing import Status
+from .convert import convert_to_cpp_doc, convert_to_py_doc
+from .doc import Doc
+from .param import (
+    AddColumnOption,
+    AlterColumnOption,
+    CollectionOption,
+    FlatIndexParam,
+    HnswIndexParam,
+    IndexOption,
+    InvertIndexParam,
+    IVFIndexParam,
+    OptimizeOption,
+)
+from .param.vector_query import VectorQuery
+from .schema import CollectionSchema, CollectionStats, FieldSchema
+
+__all__ = ["Collection"]
+
+_VECTOR_INDEX_TYPES = (HnswIndexParam, IVFIndexParam, FlatIndexParam)
+
+
+class Collection:
+    """Represents an opened collection in Zvec.
+
+    A `Collection` provides methods for data definition (DDL), data manipulation (DML),
+    and querying (DQL). It is obtained via `create_and_open()` or `open()`.
+
+    This class is not meant to be instantiated directly; use factory functions instead.
+    """
+
+    def __init__(self, obj: _Collection):
+        self._obj = obj
+        self._schema = None
+        self._querier = None
+
+    @classmethod
+    def _from_core(cls, core_collection: _Collection) -> Collection:
+        if not core_collection:
+            raise ValueError("Collection is None")
+        inst = cls.__new__(cls)
+        inst._obj = core_collection
+        schema = CollectionSchema._from_core(core_collection.Schema())
+        inst._schema = schema
+        inst._querier = QueryExecutorFactory.create(schema)
+        return inst
+
+    @property
+    def path(self) -> str:
+        """str: The filesystem path of the collection."""
+        return self._obj.Path()
+
+    @property
+    def option(self) -> CollectionOption:
+        """CollectionOption: The options used to open the collection."""
+        return self._obj.Options()
+
+    @property
+    def schema(self) -> CollectionSchema:
+        """CollectionSchema: The schema defining the structure of the collection."""
+        return self._schema
+
+    @property
+    def stats(self) -> CollectionStats:
+        """CollectionStats: Runtime statistics about the collection (e.g., doc count, size)."""
+        return self._obj.Stats()
+
+    # ========== Collection DDL Methods ==========
+    def destroy(self) -> None:
+        """Permanently delete the collection from disk.
+
+        Warning:
+            This operation is irreversible. All data will be lost.
+        """
+        self._obj.Destroy()
+
+    def flush(self) -> None:
+        """Force all pending writes to disk.
+
+        Ensures durability of recent inserts/updates.
+        """
+        self._obj.Flush()
+
+    # ========== Index DDL Methods ==========
+    def create_index(
+        self,
+        field_name: str,
+        index_param: Union[
+            HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam
+        ],
+        option: IndexOption = IndexOption(),
+    ) -> None:
+        """Create an index on a field.
+
+        Vector index types (HNSW, IVF, FLAT) can only be applied to vector fields.
+        Inverted index (`InvertIndexParam`) is for scalar fields.
+
+        Args:
+            field_name (str): Name of the field to index.
+            index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam]):
+                Index configuration.
+            option (Optional[IndexOption], optional): Index creation options.
+                Defaults to ``IndexOption()``.
+
+        Raises:
+            ValueError: If a vector index is applied to a non-vector field.
+        """
+        if index_param in _VECTOR_INDEX_TYPES and not self.schema.vector(field_name):
+            supported_types = ", ".join(cls.__name__ for cls in _VECTOR_INDEX_TYPES)
+            raise ValueError(
+                f"Cannot apply vector index to non-vector field '{field_name}'. "
+                f"The field must be of vector type to use index types like {supported_types}."
+            )
+        self._obj.CreateIndex(field_name, index_param, option)
+        self._schema = CollectionSchema._from_core(self._obj.Schema())
+
+    def drop_index(self, field_name: str) -> None:
+        """Remove the index from a field.
+
+        Args:
+            field_name (str): Name of the indexed field.
+        """
+        self._obj.DropIndex(field_name)
+        self._schema = CollectionSchema._from_core(self._obj.Schema())
+
+    def optimize(self, option: OptimizeOption = OptimizeOption()) -> None:
+        """Optimize the collection (e.g., merge segments, rebuild index).
+
+        Args:
+            option (Optional[OptimizeOption], optional): Optimization options.
+                Defaults to ``OptimizeOption()``.
+        """
+        self._obj.Optimize(option)
+
+    # ========== COLUMN DDL Methods ==========
+    def add_column(
+        self,
+        field_schema: FieldSchema,
+        expression: str = "",
+        option: AddColumnOption = AddColumnOption(),
+    ) -> None:
+        """Add a new column to the collection.
+
+        The column is populated using the provided expression (e.g., SQL-like formula).
+
+        Args:
+            field_schema (FieldSchema): Schema definition for the new column.
+            expression (str): Expression to compute values for existing documents.
+            option (Optional[AddColumnOption], optional): Options for the operation.
+                Defaults to ``AddColumnOption()``.
+        """
+        self._obj.AddColumn(
+            field_schema.name, field_schema._get_object(), expression, option
+        )
+        self._schema = CollectionSchema._from_core(self._obj.Schema())
+
+    def drop_column(self, field_name: str) -> None:
+        """Remove a column from the collection.
+
+        Args:
+            field_name (str): Name of the column to drop.
+        """
+        self._obj.DropColumn(field_name)
+        self._schema = CollectionSchema._from_core(self._obj.Schema())
+
+    def alter_column(
+        self,
+        old_name: str,
+        new_name: Optional[str] = None,
+        field_schema: Optional[FieldSchema] = None,
+        option: AlterColumnOption = AlterColumnOption(),
+    ) -> None:
+        """Rename a column, update its schema.
+
+        This method supports three atomic operations:
+          1. Rename only (when `field_schema` is None).
+          2. Modify schema only (when `new_name` is None or empty string).
+
+        Args:
+            old_name (str): The current name of the column to be altered.
+            new_name (Optional[str]): The new name for the column.
+                - If provided and non-empty, the column will be renamed.
+                - If `None` or empty string, no rename occurs.
+            field_schema (Optional[FieldSchema]): The new schema definition.
+                - If provided, the column's type, dimension, or other properties will be updated.
+                - If `None`, only renaming (if requested) is performed.
+            option (AlterColumnOption, optional): Options controlling the alteration behavior.
+                Defaults to ``AlterColumnOption()``.
+
+        **Limitation**: This operation **only supports scalar numeric columns**. such as:
+        - `DOUBLE`, `FLOAT`,
+        - `INT32`, `INT64`, `UINT32`, `UINT64`
+
+        Note:
+            - Schema modification may trigger data migration or index rebuild.
+
+        Examples:
+            >>> # Rename column only
+            >>> results = collection.alter_column(old_name="id", new_name="doc_id")
+
+            >>> # Modify schema only
+            >>> new_schema = FieldSchema(name="doc_id", dtype=DataType.INT64)
+            >>> collection.alter_column("id", field_schema=new_schema)
+        """
+        self._obj.AlterColumn(
+            old_name,
+            new_name or "",
+            field_schema._get_object() if field_schema else None,
+            option,
+        )
+        self._schema = CollectionSchema._from_core(self._obj.Schema())
+
+    # ========== Collection DDL Methods ==========
+    def insert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
+        """Insert new documents into the collection.
+
+        Documents must have unique IDs and conform to the schema.
+
+        Args:
+            docs (Union[Doc, list[Doc]]): One or more documents to insert.
+
+        Returns:
+            Union[Status, list[Status]]: If a single Doc was given, returns its Status;
+            if a list was given, returns a list of Status objects.
+        """
+        is_single = isinstance(docs, Doc)
+        doc_list = [docs] if is_single else docs
+        results = self._obj.Insert(
+            [convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
+        )
+        return results[0] if is_single else results
+
+    def upsert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
+        """Insert new documents or update existing ones by ID.
+
+        Args:
+            docs (Union[Doc, list[Doc]]): Documents to upsert.
+
+        Returns:
+            Union[Status, list[Status]]: If a single Doc was given, returns its Status;
+            if a list was given, returns a list of Status objects.
+        """
+        is_single = isinstance(docs, Doc)
+        doc_list = [docs] if is_single else docs
+        results = self._obj.Upsert(
+            [convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
+        )
+        return results[0] if is_single else results
+
+    def update(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
+        """Update existing documents by ID.
+
+        Only specified fields are updated; others remain unchanged.
+
+        Args:
+            docs (Union[Doc, list[Doc]]): Documents containing updated fields.
+
+        Returns:
+            Union[Status, list[Status]]: If a single Doc was given, returns its Status;
+            if a list was given, returns a list of Status objects.
+        """
+        is_single = isinstance(docs, Doc)
+        doc_list = [docs] if is_single else docs
+        results = self._obj.Update(
+            [convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
+        )
+        return results[0] if is_single else results
+
+    def delete(self, ids: Union[str, list[str]]) -> Union[Status, list[Status]]:
+        """Delete documents by ID.
+
+        Args:
+            ids (Union[str, list[str]]): One or more document IDs to delete.
+
+        Returns:
+            Union[Status, list[Status]]: If a single id was given, returns its Status;
+            if a list was given, returns a list of Status objects.
+        """
+        is_single = isinstance(ids, str)
+        id_list = [ids] if isinstance(ids, str) else ids
+        results = self._obj.Delete(id_list)
+        return results[0] if is_single else results
+
+    def delete_by_filter(self, filter: str) -> None:
+        """Delete documents matching a filter expression.
+
+        Args:
+            filter (str): Boolean expression (e.g., ``"age > 30"``).
+        """
+        self._obj.DeleteByFilter(filter)
+
+    # ========== Collection DQL-fetch Methods ==========
+    def fetch(self, ids: Union[str, list[str]]) -> dict[str, Doc]:
+        """Retrieve documents by ID.
+
+        Args:
+            ids (Union[str, list[str]]): Document IDs to fetch.
+
+        Returns:
+            dict[str, Doc]: Mapping from ID to document. Missing IDs are omitted.
+        """
+        ids = [ids] if isinstance(ids, str) else ids
+        docs = self._obj.Fetch(ids)
+        return {
+            doc_id: py_doc
+            for doc_id, core_doc in docs.items()
+            if (py_doc := convert_to_py_doc(core_doc, self.schema)) is not None
+        }
+
+    # ========== Collection DQL-Query Methods ==========
+
+    def query(
+        self,
+        vectors: Optional[Union[VectorQuery, list[VectorQuery]]] = None,
+        *,
+        topk: int = 10,
+        filter: Optional[str] = None,
+        include_vector: bool = False,
+        output_fields: Optional[list[str]] = None,
+        reranker: Optional[ReRanker] = None,
+    ) -> list[Doc]:
+        """Perform vector similarity search with optional filtering and re-ranking.
+
+        At least one `VectorQuery` must be provided.
+
+        Args:
+            vectors (Optional[Union[VectorQuery, list[VectorQuery]]], optional):
+                One or more vector queries. Defaults to None.
+            topk (int, optional): Number of nearest neighbors to return.
+                Defaults to 10.
+            filter (Optional[str], optional): Boolean expression to pre-filter candidates.
+                Defaults to None.
+            include_vector (bool, optional): Whether to include vector data in results.
+                Defaults to False.
+            output_fields (Optional[list[str]], optional): Scalar fields to include.
+                If None, all fields are returned. Defaults to None.
+            reranker (Optional[ReRanker], optional): Re-ranker to refine results.
+                Defaults to None.
+
+        Returns:
+            list[Doc]: Top-k matching documents, sorted by relevance score.
+
+        Examples:
+            >>> from zvec import VectorQuery
+            >>> results = collection.query(
+            ...     vectors=VectorQuery("embedding", vector=[0.1, 0.2]),
+            ...     topk=5,
+            ...     filter="category == 'tech'",
+            ...     output_fields=["title", "url"]
+            ... )
+        """
+        ctx = QueryContext(
+            topk=topk,
+            filter=filter,
+            queries=[vectors] if isinstance(vectors, VectorQuery) else vectors,
+            include_vector=include_vector,
+            output_fields=output_fields,
+            reranker=reranker,
+        )
+        return self._querier.execute(ctx, self._obj)

python/zvec/model/convert.py

@@ -0,0 +1,54 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from _zvec import _Doc
+
+from .doc import Doc
+from .schema import CollectionSchema
+
+
+def convert_to_cpp_doc(doc: Doc, collection_schema: CollectionSchema) -> _Doc:
+    if not doc or not collection_schema:
+        return None
+
+    _doc = _Doc()
+
+    # set pk
+    _doc.set_pk(doc.id)
+
+    # set scalar fields
+    for k, v in doc.fields.items():
+        field_schema = collection_schema.field(k)
+        if not field_schema:
+            raise ValueError(
+                f"schema validate failed: {k} not found in collection schema"
+            )
+        _doc.set_any(k, field_schema._get_object(), v)
+
+    # set vector fields
+    for k, v in doc.vectors.items():
+        vector_schema = collection_schema.vector(k)
+        if not vector_schema:
+            raise ValueError(
+                f"schema validate failed: {k} not found in collection schema"
+            )
+        _doc.set_any(k, vector_schema._get_object(), v)
+    return _doc
+
+
+def convert_to_py_doc(doc: _Doc, collection_schema: CollectionSchema) -> Doc:
+    if not doc or not collection_schema:
+        return None
+
+    data_tuple = doc.get_all(collection_schema._get_object())
+    return Doc._from_tuple(data_tuple)

python/zvec/model/doc.py

@@ -0,0 +1,173 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+from ..common import VectorType
+
+__all__ = [
+    "Doc",
+]
+
+
+class Doc:
+    """Represents a retrieved document with optional metadata, fields, and vectors.
+
+    This immutable data class encapsulates the result of a search or retrieval
+    operation. It includes the document ID, relevance score (if applicable),
+    scalar fields, and vector embeddings.
+
+    During initialization, any `numpy.ndarray` in `vectors` is automatically
+    converted to a plain Python list for JSON serialization and immutability.
+
+    Attributes:
+        id (str): Unique identifier of the document.
+        score (Optional[float], optional): Relevance score from search.
+            Defaults to None.
+        vectors (Optional[dict[str, VectorType]], optional): Named vector
+            embeddings associated with the document. Values are converted to
+            lists if originally `np.ndarray`. Defaults to None.
+        fields (Optional[dict[str, Any]], optional): Scalar metadata fields
+            (e.g., title, timestamp). Defaults to None.
+
+    Examples:
+        >>> import numpy as np
+        >>> import zvec
+        >>> doc = zvec.Doc(
+        ...     id="doc1",
+        ...     score=0.95,
+        ...     vectors={"emb": np.array([0.1, 0.2, 0.3])},
+        ...     fields={"title": "Hello World"}
+        ... )
+        >>> print(doc.vector("emb"))
+        [0.1, 0.2, 0.3]
+        >>> print(doc.has_field("title"))
+        True
+    """
+
+    __slots__ = ("id", "score", "vectors", "fields")
+
+    def __init__(
+        self,
+        id: str,
+        score: Optional[float] = None,
+        vectors: Optional[dict[str, VectorType]] = None,
+        fields: Optional[dict[str, Any]] = None,
+    ):
+        self.id = id
+        self.score = score
+        self.vectors = vectors or {}
+        self.fields = fields or {}
+
+    def has_field(self, name: str) -> bool:
+        """Check if the document contains a scalar field with the given name.
+
+        Args:
+            name (str): Name of the field to check.
+
+        Returns:
+            bool: True if the field exists, False otherwise.
+        """
+        return name in self.fields
+
+    def has_vector(self, name: str) -> bool:
+        """Check if the document contains a vector with the given name.
+
+        Args:
+            name (str): Name of the vector to check.
+
+        Returns:
+            bool: True if the vector exists, False otherwise.
+        """
+        return name in self.vectors
+
+    def vector(self, name: str):
+        """Get a vector by name.
+
+        Args:
+            name (str): Name of the vector.
+
+        Returns:
+            Any: The vector (as a list) if it exists, otherwise None.
+        """
+        return self.vectors and self.vectors.get(name)
+
+    def field(self, name: str):
+        """Get a scalar field by name.
+
+        Args:
+            name (str): Name of the field.
+
+        Returns:
+            Any: The field value if it exists, otherwise None.
+        """
+        return self.fields and self.fields.get(name)
+
+    def vector_names(self) -> list[str]:
+        """Get the list of all vector names in this document.
+
+        Returns:
+            list[str]: A list of vector field names. Empty if no vectors.
+        """
+        return [] if not self.vectors else list(self.vectors.keys())
+
+    def field_names(self) -> list[str]:
+        """Get the list of all scalar field names in this document.
+
+        Returns:
+            list[str]: A list of field names. Empty if no fields.
+        """
+        return [] if not self.fields else list(self.fields.keys())
+
+    def __repr__(self) -> str:
+        try:
+            schema = {
+                "id": self.id,
+                "score": self.score,
+                "fields": self.fields,
+                "vectors": self.vectors,
+            }
+            return json.dumps(schema, indent=2, ensure_ascii=False)
+        except Exception as e:
+            return f"<Doc error during repr: {e}>"
+
+    def _replace(self, **changes):
+        new_tuple = (
+            changes.get("id", self.id),
+            changes.get("score", self.score),
+            changes.get("fields", self.fields.copy() if self.fields else None),
+            changes.get("vectors", self.vectors.copy() if self.vectors else None),
+        )
+        return type(self)._from_tuple(new_tuple)
+
+    @classmethod
+    def _from_tuple(
+        cls, data_tuple: tuple[str, float, dict[str, Any], dict[str, VectorType]]
+    ):
+        obj = object.__new__(cls)
+        obj.id = data_tuple[0]
+        obj.score = data_tuple[1]
+        obj.fields = data_tuple[2] or {}
+
+        vectors = data_tuple[3]
+        if vectors is not None:
+            obj.vectors = {
+                name: (vec.tolist() if hasattr(vec, "tolist") else vec)
+                for name, vec in vectors.items()
+            }
+        else:
+            obj.vectors = {}
+        return obj

python/zvec/model/param/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from _zvec.param import (
+    AddColumnOption,
+    AlterColumnOption,
+    CollectionOption,
+    FlatIndexParam,
+    HnswIndexParam,
+    HnswQueryParam,
+    IndexOption,
+    InvertIndexParam,
+    IVFIndexParam,
+    IVFQueryParam,
+    OptimizeOption,
+)
+
+__all__ = [
+    "AddColumnOption",
+    "AlterColumnOption",
+    "CollectionOption",
+    "FlatIndexParam",
+    "HnswIndexParam",
+    "HnswQueryParam",
+    "IVFIndexParam",
+    "IVFQueryParam",
+    "IndexOption",
+    "InvertIndexParam",
+    "OptimizeOption",
+]

python/zvec/model/param/__init__.pyi

@@ -0,0 +1,692 @@
+"""
+This module contains the params of Zvec
+"""
+
+from __future__ import annotations
+
+import collections
+import typing
+
+import _zvec.typing
+
+__all__: list[str] = [
+    "AddColumnOption",
+    "AlterColumnOption",
+    "CollectionOption",
+    "FlatIndexParam",
+    "HnswIndexParam",
+    "HnswQueryParam",
+    "IVFIndexParam",
+    "IVFQueryParam",
+    "IndexOption",
+    "IndexParam",
+    "InvertIndexParam",
+    "OptimizeOption",
+    "QueryParam",
+    "SegmentOption",
+    "VectorIndexParam",
+]
+
+class AddColumnOption:
+    """
+
+    Options for adding a new column to a collection.
+
+    Attributes:
+        concurrency (int): Number of threads to use when backfilling data
+            for the new column. If 0, auto-detect is used. Default is 0.
+
+    Examples:
+        >>> opt = AddColumnOption(concurrency=1)
+        >>> print(opt.concurrency)
+        1
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
+        """
+        Constructs an AddColumnOption instance.
+
+        Args:
+            concurrency (int, optional): Number of threads for data backfill.
+                0 means auto-detect. Defaults to 0.
+        """
+
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def concurrency(self) -> int:
+        """
+        int: Number of threads used when adding a column (0 = auto).
+        """
+
+class AlterColumnOption:
+    """
+
+    Options for altering an existing column (e.g., changing index settings).
+
+    Attributes:
+        concurrency (int): Number of threads to use during the alteration process.
+            If 0, the system will choose an optimal value automatically.
+            Default is 0.
+
+    Examples:
+        >>> opt = AlterColumnOption(concurrency=1)
+        >>> print(opt.concurrency)
+        1
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
+        """
+        Constructs an AlterColumnOption instance.
+
+        Args:
+            concurrency (int, optional): Number of threads for column alteration.
+                0 means auto-detect. Defaults to 0.
+        """
+
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def concurrency(self) -> int:
+        """
+        int: Number of threads used when altering a column (0 = auto).
+        """
+
+class CollectionOption:
+    """
+
+    Options for opening or creating a collection.
+
+    Attributes:
+        read_only (bool): Whether the collection is opened in read-only mode.
+            Default is False.
+        enable_mmap (bool): Whether to use memory-mapped I/O for data files.
+            Default is True.
+
+    Examples:
+        >>> opt = CollectionOption(read_only=True, enable_mmap=False)
+        >>> print(opt.read_only)
+        True
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, read_only: bool = False, enable_mmap: bool = True) -> None:
+        """
+        Constructs a CollectionOption instance.
+
+        Args:
+            read_only (bool, optional): Open collection in read-only mode.
+                Defaults to False.
+            enable_mmap (bool, optional): Enable memory-mapped I/O.
+                Defaults to True.
+        """
+
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def enable_mmap(self) -> bool: ...
+    @property
+    def read_only(self) -> bool: ...
+
+class FlatIndexParam(VectorIndexParam):
+    """
+
+    Parameters for configuring a flat (brute-force) index.
+
+    A flat index performs exact nearest neighbor search by comparing the query
+    vector against all vectors in the collection. It is simple, accurate, and
+    suitable for small to medium datasets or as a baseline.
+
+    Attributes:
+        metric_type (MetricType): Distance metric used for similarity computation.
+            Default is ``MetricType.IP`` (inner product).
+        quantize_type (QuantizeType): Optional quantization type for vector
+            compression (e.g., FP16, INT8). Use ``QuantizeType.UNDEFINED`` to
+            disable quantization. Default is ``QuantizeType.UNDEFINED``.
+
+    Examples:
+        >>> from zvec.typing import MetricType, QuantizeType
+        >>> params = FlatIndexParam(
+        ...     metric_type=MetricType.L2,
+        ...     quantize_type=QuantizeType.FP16
+        ... )
+        >>> print(params)
+        {'metric_type': 'L2', 'quantize_type': 'FP16'}
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(
+        self,
+        metric_type: _zvec.typing.MetricType = ...,
+        quantize_type: _zvec.typing.QuantizeType = ...,
+    ) -> None:
+        """
+        Constructs a FlatIndexParam instance.
+
+        Args:
+            metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
+            quantize_type (QuantizeType, optional): Vector quantization type.
+                Defaults to QuantizeType.UNDEFINED (no quantization).
+        """
+
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+class HnswIndexParam(VectorIndexParam):
+    """
+
+    Parameters for configuring an HNSW (Hierarchical Navigable Small World) index.
+
+    HNSW is a graph-based approximate nearest neighbor search index. This class
+    encapsulates its construction hyperparameters.
+
+    Attributes:
+        metric_type (MetricType): Distance metric used for similarity computation.
+            Default is ``MetricType.IP`` (inner product).
+        m (int): Number of bi-directional links created for every new element
+            during construction. Higher values improve accuracy but increase
+            memory usage and construction time. Default is 100.
+        ef_construction (int): Size of the dynamic candidate list for nearest
+            neighbors during index construction. Larger values yield better
+            graph quality at the cost of slower build time. Default is 500.
+        quantize_type (QuantizeType): Optional quantization type for vector
+            compression (e.g., FP16, INT8). Default is `QuantizeType.UNDEFINED` to
+            disable quantization.
+
+    Examples:
+        >>> from zvec.typing import MetricType, QuantizeType
+        >>> params = HnswIndexParam(
+        ...     metric_type=MetricType.COSINE,
+        ...     m=16,
+        ...     ef_construction=200,
+        ...     quantize_type=QuantizeType.INT8
+        ... )
+        >>> print(params)
+        {'metric_type': 'IP', 'm': 16, 'ef_construction': 200, 'quantize_type': 'INT8'}
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(
+        self,
+        metric_type: _zvec.typing.MetricType = ...,
+        m: typing.SupportsInt = 100,
+        ef_construction: typing.SupportsInt = 500,
+        quantize_type: _zvec.typing.QuantizeType = ...,
+    ) -> None: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+    @property
+    def ef_construction(self) -> int:
+        """
+        int: Candidate list size during index construction.
+        """
+
+    @property
+    def m(self) -> int:
+        """
+        int: Maximum number of neighbors per node in upper layers.
+        """
+
+class HnswQueryParam(QueryParam):
+    """
+
+    Query parameters for HNSW (Hierarchical Navigable Small World) index.
+
+    Controls the trade-off between search speed and accuracy via the `ef` parameter.
+
+    Attributes:
+        type (IndexType): Always ``IndexType.HNSW``.
+        ef (int): Size of the dynamic candidate list during search.
+            Larger values improve recall but slow down search.
+            Default is 300.
+        radius (float): Search radius for range queries. Default is 0.0.
+        is_linear (bool): Force linear search. Default is False.
+        is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
+
+    Examples:
+        >>> params = HnswQueryParam(ef=300)
+        >>> print(params.ef)
+        300
+        >>> print(params.to_dict() if hasattr(params, 'to_dict') else params)
+        {"type":"HNSW", "ef":300}
+    """
+    def __getstate__(self) -> tuple: ...
+    def __init__(
+        self,
+        ef: typing.SupportsInt = 300,
+        radius: typing.SupportsFloat = 0.0,
+        is_linear: bool = False,
+        is_using_refiner: bool = False,
+    ) -> None:
+        """
+        Constructs an HnswQueryParam instance.
+
+        Args:
+            ef (int, optional): Search-time candidate list size.
+                Higher values improve accuracy. Defaults to 100.
+            radius (float, optional): Search radius for range queries. Default is 0.0.
+            is_linear (bool, optional): Force linear search. Default is False.
+            is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
+        """
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def ef(self) -> int:
+        """
+        int: Size of the dynamic candidate list during HNSW search.
+        """
+
+class IVFIndexParam(VectorIndexParam):
+    """
+
+    Parameters for configuring an IVF (Inverted File Index) index.
+
+    IVF partitions the vector space into clusters (inverted lists). At query time,
+    only a subset of clusters is searched, providing a trade-off between speed
+    and accuracy.
+
+    Attributes:
+        metric_type (MetricType): Distance metric used for similarity computation.
+            Default is ``MetricType.IP`` (inner product).
+        n_list (int): Number of clusters (inverted lists) to partition the dataset into.
+            If set to 0, the system will auto-select a reasonable value based on data size.
+            Default is 0 (auto).
+        n_iters (int): Number of iterations for k-means clustering during index training.
+            Higher values yield more stable centroids. Default is 10.
+        use_soar (bool): Whether to enable SOAR (Scalable Optimized Adaptive Routing)
+            for improved IVF search performance. Default is False.
+        quantize_type (QuantizeType): Optional quantization type for vector
+            compression (e.g., FP16, INT8). Default is ``QuantizeType.UNDEFINED``.
+
+    Examples:
+        >>> from zvec.typing import MetricType, QuantizeType
+        >>> params = IVFIndexParam(
+        ...     metric_type=MetricType.COSINE,
+        ...     n_list=100,
+        ...     n_iters=15,
+        ...     use_soar=True,
+        ...     quantize_type=QuantizeType.INT8
+        ... )
+        >>> print(params.n_list)
+        100
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(
+        self,
+        metric_type: _zvec.typing.MetricType = ...,
+        n_list: typing.SupportsInt = 0,
+        n_iters: typing.SupportsInt = 10,
+        use_soar: bool = False,
+        quantize_type: _zvec.typing.QuantizeType = ...,
+    ) -> None:
+        """
+        Constructs an IVFIndexParam instance.
+
+        Args:
+            metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
+            n_list (int, optional): Number of inverted lists (clusters). Set to 0 for auto.
+                Defaults to 0.
+            n_iters (int, optional): Number of k-means iterations during training.
+                Defaults to 10.
+            use_soar (bool, optional): Enable SOAR optimization. Defaults to False.
+            quantize_type (QuantizeType, optional): Vector quantization type.
+                Defaults to QuantizeType.UNDEFINED.
+        """
+
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+    @property
+    def n_iters(self) -> int:
+        """
+        int: Number of k-means iterations during training.
+        """
+
+    @property
+    def n_list(self) -> int:
+        """
+        int: Number of inverted lists (0 = auto).
+        """
+
+    @property
+    def use_soar(self) -> bool:
+        """
+        bool: Whether SOAR optimization is enabled.
+        """
+
+class IVFQueryParam(QueryParam):
+    """
+
+    Query parameters for IVF (Inverted File Index) index.
+
+    Controls how many inverted lists (`nprobe`) to visit during search.
+
+    Attributes:
+        type (IndexType): Always ``IndexType.IVF``.
+        nprobe (int): Number of closest clusters (inverted lists) to search.
+            Higher values improve recall but increase latency.
+            Default is 10.
+        radius (float): Search radius for range queries. Default is 0.0.
+        is_linear (bool): Force linear search. Default is False.
+
+    Examples:
+        >>> params = IVFQueryParam(nprobe=20)
+        >>> print(params.nprobe)
+        20
+    """
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, nprobe: typing.SupportsInt = 10) -> None:
+        """
+        Constructs an IVFQueryParam instance.
+
+        Args:
+            nprobe (int, optional): Number of inverted lists to probe during search.
+                Higher values improve accuracy. Defaults to 10.
+        """
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def nprobe(self) -> int:
+        """
+        int: Number of inverted lists to search during IVF query.
+        """
+
+class IndexOption:
+    """
+
+    Options for creating an index.
+
+    Attributes:
+        concurrency (int): Number of threads to use during index creation.
+            If 0, the system will choose an optimal value automatically.
+            Default is 0.
+
+    Examples:
+        >>> opt = IndexOption(concurrency=4)
+        >>> print(opt.concurrency)
+        4
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
+        """
+        Constructs an IndexOption instance.
+
+        Args:
+            concurrency (int, optional): Number of concurrent threads.
+                0 means auto-detect. Defaults to 0.
+        """
+
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def concurrency(self) -> int:
+        """
+        int: Number of threads used for index creation (0 = auto).
+        """
+
+class IndexParam:
+    """
+
+    Base class for all index parameter configurations.
+
+    This abstract base class defines the common interface for index types.
+    It should not be instantiated directly; use derived classes instead.
+
+    Attributes:
+        type (IndexType): The type of the index (e.g., HNSW, FLAT, INVERT).
+    """
+
+    __hash__: typing.ClassVar[None] = None
+
+    def __eq__(self, arg0: typing.Any) -> bool: ...
+    def __getstate__(self) -> tuple: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def clone(self) -> IndexParam: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+    @property
+    def type(self) -> _zvec.typing.IndexType:
+        """
+        IndexType: The type of the index.
+        """
+
+class InvertIndexParam(IndexParam):
+    """
+
+    Parameters for configuring an invert index.
+
+    This class controls whether range query
+    optimization is enabled for invert index structures.
+
+    Attributes:
+        type (IndexType): Always `IndexType.INVERTED`.
+        enable_range_optimization (bool): Whether range optimization is enabled.
+        enable_extended_wildcard (bool): Whether extended wildcard (suffix and infix) search is enabled.
+
+    Examples:
+        >>> params = InvertIndexParam(enable_range_optimization=True, enable_extended_wildcard=False)
+        >>> print(params.enable_range_optimization)
+        True
+        >>> print(params.enable_extended_wildcard)
+        False
+        >>> config = params.to_dict()
+        >>> print(config)
+        {'enable_range_optimization': True, 'enable_extended_wildcard': False}
+    """
+    def __getstate__(self) -> tuple: ...
+    def __init__(
+        self,
+        enable_range_optimization: bool = False,
+        enable_extended_wildcard: bool = False,
+    ) -> None:
+        """
+        Constructs an InvertIndexParam instance.
+
+        Args:
+            enable_range_optimization (bool, optional): If True, enables range query
+                optimization for the invert index. Defaults to False.
+            enable_extended_wildcard (bool, optional): If True, enables extended wildcard
+                search including suffix and infix patterns. Defaults to False.
+        """
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+    @property
+    def enable_extended_wildcard(self) -> bool:
+        """
+        bool: Whether extended wildcard (suffix and infix) search is enabled.
+        Note: Prefix search is always enabled regardless of this setting.
+        """
+    @property
+    def enable_range_optimization(self) -> bool:
+        """
+        bool: Whether range optimization is enabled for this inverted index.
+        """
+
+class OptimizeOption:
+    """
+
+    Options for optimizing a collection (e.g., merging segments).
+
+    Attributes:
+        concurrency (int): Number of threads to use during optimization.
+            If 0, the system will choose an optimal value automatically.
+            Default is 0.
+
+    Examples:
+        >>> opt = OptimizeOption(concurrency=2)
+        >>> print(opt.concurrency)
+        2
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
+        """
+        Constructs an OptimizeOption instance.
+
+        Args:
+            concurrency (int, optional): Number of concurrent threads.
+                0 means auto-detect. Defaults to 0.
+        """
+
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def concurrency(self) -> int:
+        """
+        int: Number of threads used for optimization (0 = auto).
+        """
+
+class QueryParam:
+    """
+
+    Base class for all query parameter configurations.
+
+    This abstract base class defines common query settings such as search radius
+    and whether to force linear (brute-force) search. It should not be instantiated
+    directly; use derived classes like `HnswQueryParam` or `IVFQueryParam`.
+
+    Attributes:
+        type (IndexType): The index type this query is configured for.
+        radius (float): Search radius for range queries. Used in combination with
+            top-k to filter results. Default is 0.0 (disabled).
+        is_linear (bool): If True, forces brute-force linear search instead of
+            using the index. Useful for debugging or small datasets. Default is False.
+        is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
+    """
+    def __getstate__(self) -> tuple: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def is_linear(self) -> bool:
+        """
+        bool: Whether to bypass the index and use brute-force linear search.
+        """
+    @property
+    def is_using_refiner(self) -> bool:
+        """
+        bool: Whether to use refiner for the query.
+        """
+    @property
+    def radius(self) -> float:
+        """
+        IndexType: The type of index this query targets.
+        """
+    @property
+    def type(self) -> _zvec.typing.IndexType:
+        """
+        IndexType: The type of index this query targets.
+        """
+
+class SegmentOption:
+    """
+
+    Options for segment-level operations.
+
+    Currently, this class mirrors CollectionOption and is used internally.
+    It supports read-only mode, memory mapping, and buffer configuration.
+
+    Note:
+        This class is primarily for internal use. Most users should use
+        CollectionOption instead.
+
+    Examples:
+        >>> opt = SegmentOption()
+        >>> print(opt.enable_mmap)
+        True
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self) -> None:
+        """
+        Constructs a SegmentOption with default settings.
+        """
+
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    @property
+    def enable_mmap(self) -> bool:
+        """
+        bool: Whether memory-mapped I/O is enabled.
+        """
+
+    @property
+    def max_buffer_size(self) -> int:
+        """
+        int: Maximum buffer size in bytes (internal use).
+        """
+
+    @property
+    def read_only(self) -> bool:
+        """
+        bool: Whether the segment is read-only.
+        """
+
+class VectorIndexParam(IndexParam):
+    """
+
+    Base class for vector index parameter configurations.
+
+    Encapsulates common settings for all vector index types.
+
+    Attributes:
+        type (IndexType): The specific vector index type (e.g., HNSW, FLAT).
+        metric_type (MetricType): Distance metric used for similarity search.
+        quantize_type (QuantizeType): Optional vector quantization type.
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+    @property
+    def metric_type(self) -> _zvec.typing.MetricType:
+        """
+        MetricType: Distance metric (e.g., IP, COSINE, L2).
+        """
+
+    @property
+    def quantize_type(self) -> _zvec.typing.QuantizeType:
+        """
+        QuantizeType: Vector quantization type (e.g., FP16, INT8).
+        """
+
+class _VectorQuery:
+    field_name: str
+    filter: str
+    include_vector: bool
+    query_params: QueryParam
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self) -> None: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def set_vector(self, arg0: ..., arg1: typing.Any) -> None: ...
+    @property
+    def output_fields(self) -> list[str] | None: ...
+    @output_fields.setter
+    def output_fields(self, arg0: collections.abc.Sequence[str] | None) -> None: ...
+    @property
+    def topk(self) -> int: ...
+    @topk.setter
+    def topk(self, arg0: typing.SupportsInt) -> None: ...

python/zvec/model/param/vector_query.py

@@ -0,0 +1,80 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from ...common import VectorType
+from . import HnswQueryParam, IVFQueryParam
+
+__all__ = ["VectorQuery"]
+
+
+@dataclass(frozen=True)
+class VectorQuery:
+    """Represents a vector search query for a specific field in a collection.
+
+    A `VectorQuery` can be constructed using either a document ID (to look up
+    its vector) or an explicit vector. It may optionally include index-specific
+    query parameters to control search behavior (e.g., `ef` for HNSW, `nprobe` for IVF).
+
+    Exactly one of `id` or `vector` should be provided. If both are given,
+    behavior is implementation-defined (typically `id` takes precedence).
+
+    Attributes:
+        field_name (str): Name of the vector field to query.
+        id (Optional[str], optional): Document ID to fetch vector from. Default is None.
+        vector (VectorType, optional): Explicit query vector. Default is None.
+        param (Optional[Union[HnswQueryParam, IVFQueryParam]], optional):
+            Index-specific query parameters. Default is None.
+
+    Examples:
+        >>> import zvec
+        >>> # Query by ID
+        >>> q1 = zvec.VectorQuery(field_name="embedding", id="doc123")
+        >>> # Query by vector
+        >>> q2 = zvec.VectorQuery(
+        ...     field_name="embedding",
+        ...     vector=[0.1, 0.2, 0.3],
+        ...     param=HnswQueryParam(ef=300)
+        ... )
+    """
+
+    field_name: str
+    id: Optional[str] = None
+    vector: VectorType = None
+    param: Optional[Union[HnswQueryParam, IVFQueryParam]] = None
+
+    def has_id(self) -> bool:
+        """Check if the query is based on a document ID.
+
+        Returns:
+            bool: True if `id` is set, False otherwise.
+        """
+        return self.id is not None
+
+    def has_vector(self) -> bool:
+        """Check if the query contains an explicit vector.
+
+        Returns:
+            bool: True if `vector` is non-empty, False otherwise.
+        """
+        return self.vector is not None and len(self.vector) > 0
+
+    def _validate(self) -> None:
+        if self.field_name is None:
+            raise ValueError("Field name cannot be empty")
+        if self.id and self.vector:
+            raise ValueError("Cannot provide both id and vector")

python/zvec/model/schema/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from _zvec.schema import CollectionStats
+
+from .collection_schema import CollectionSchema
+from .field_schema import FieldSchema, VectorSchema
+
+__all__ = ["CollectionSchema", "CollectionStats", "FieldSchema", "VectorSchema"]

python/zvec/model/schema/__init__.pyi

@@ -0,0 +1,109 @@
+"""
+This module contains the schema of Zvec
+"""
+
+from __future__ import annotations
+
+import collections.abc
+import typing
+
+import _zvec.param
+import _zvec.typing
+
+from .collection_schema import CollectionSchema
+from .field_schema import FieldSchema, VectorSchema
+
+__all__: list[str] = [
+    "CollectionSchema",
+    "CollectionStats",
+    "FieldSchema",
+    "VectorSchema",
+]
+
+class CollectionStats:
+    def __init__(self) -> None: ...
+    def __repr__(self) -> str: ...
+    @property
+    def doc_count(self) -> int: ...
+    @property
+    def index_completeness(self) -> dict[str, float]: ...
+
+class _CollectionSchema:
+    __hash__: typing.ClassVar[None] = None
+
+    def __eq__(self, arg0: _CollectionSchema) -> bool: ...
+    def __init__(
+        self, name: str, fields: collections.abc.Sequence[_FieldSchema]
+    ) -> None:
+        """
+        Construct with name and list of fields
+        """
+
+    def __ne__(self, arg0: _CollectionSchema) -> bool: ...
+    def fields(self) -> list[_FieldSchema]:
+        """
+        Return list of all field schemas.
+        """
+
+    def forward_fields(self) -> list[_FieldSchema]:
+        """
+        Return list of forward-indexed fields.
+        """
+
+    def get_field(self, field_name: str) -> _FieldSchema:
+        """
+        Get field by name (const pointer), returns None if not found.
+        """
+
+    def get_forward_field(self, field_name: str) -> _FieldSchema:
+        """
+        Get forward field (used for filtering).
+        """
+
+    def get_vector_field(self, field_name: str) -> _FieldSchema:
+        """
+        Get vector field by name.
+        """
+
+    def has_field(self, field_name: str) -> bool:
+        """
+        Check if a field exists.
+        """
+
+    def vector_fields(self) -> list[_FieldSchema]:
+        """
+        Return list of vector fields.
+        """
+
+    @property
+    def name(self) -> str: ...
+
+class _FieldSchema:
+    __hash__: typing.ClassVar[None] = None
+
+    def __eq__(self, arg0: _FieldSchema) -> bool: ...
+    def __init__(
+        self,
+        name: str,
+        data_type: _zvec.typing.DataType,
+        nullable: bool = False,
+        dimension: typing.SupportsInt = 0,
+        index_param: _zvec.param.IndexParam = None,
+    ) -> None: ...
+    def __ne__(self, arg0: _FieldSchema) -> bool: ...
+    @property
+    def data_type(self) -> _zvec.typing.DataType: ...
+    @property
+    def dimension(self) -> int: ...
+    @property
+    def index_param(self) -> typing.Any: ...
+    @property
+    def index_type(self) -> _zvec.typing.IndexType: ...
+    @property
+    def is_dense_vector(self) -> bool: ...
+    @property
+    def is_sparse_vector(self) -> bool: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def nullable(self) -> bool: ...

python/zvec/model/schema/collection_schema.py

@@ -0,0 +1,215 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import json
+from typing import Optional, Union
+
+from _zvec.schema import _CollectionSchema, _FieldSchema
+
+from .field_schema import FieldSchema, VectorSchema
+
+__all__ = [
+    "CollectionSchema",
+]
+
+
+class CollectionSchema:
+    """Defines the structure of a collection in Zvec.
+
+    A collection schema specifies the name of the collection and its fields,
+    including both scalar fields (e.g., int, string) and vector fields.
+    Field names must be unique across both scalar and vector fields.
+
+    Args:
+        name (str): Name of the collection.
+        fields (Optional[Union[FieldSchema, list[FieldSchema]]], optional):
+            One or more scalar field definitions. Defaults to None.
+        vectors (Optional[Union[VectorSchema, list[VectorSchema]]], optional):
+            One or more vector field definitions. Defaults to None.
+
+    Raises:
+        TypeError: If `fields` or `vectors` are of unsupported types.
+        ValueError: If any field or vector name is duplicated.
+
+    Examples:
+        >>> from zvec import FieldSchema, VectorSchema, DataType, IndexType
+        >>> id_field = FieldSchema("id", DataType.INT64, is_primary=True)
+        >>> emb_field = VectorSchema("embedding", dim=128, data_type=DataType.VECTOR_FP32)
+        >>> schema = CollectionSchema(
+        ...     name="my_collection",
+        ...     fields=id_field,
+        ...     vectors=emb_field
+        ... )
+        >>> print(schema.name)
+        my_collection
+    """
+
+    def __init__(
+        self,
+        name: str,
+        fields: Optional[Union[FieldSchema, list[FieldSchema]]] = None,
+        vectors: Optional[Union[VectorSchema, list[VectorSchema]]] = None,
+    ):
+        if name is None or not isinstance(name, str):
+            raise ValueError(
+                f"schema validate failed: collection name must be str, got {type(name).__name__}"
+            )
+
+        # handle fields
+        _fields_name: list[str] = []
+        _fields_list: list[_FieldSchema] = []
+
+        self._check_fields(fields, _fields_name, _fields_list)
+        self._check_vectors(vectors, _fields_name, _fields_list)
+
+        # init
+        self._cpp_obj = _CollectionSchema(
+            name=name,
+            fields=_fields_list,
+        )
+
+    def _check_fields(
+        self,
+        fields: Optional[Union[FieldSchema, list[FieldSchema]]],
+        _fields_name: list[str],
+        _fields_list: list[_FieldSchema],
+    ) -> None:
+        field_items = []
+
+        if isinstance(fields, FieldSchema):
+            field_items = [fields]
+        elif isinstance(fields, list):
+            field_items = fields
+        elif fields is None:
+            field_items = []
+        else:
+            raise TypeError(
+                f"schema validate failed: invalid 'fields' type, expected FieldSchema or list[FieldSchema], "
+                f"got {type(fields).__name__}"
+            )
+
+        for idx, field in enumerate(field_items):
+            if not isinstance(field, FieldSchema):
+                raise TypeError(
+                    f"schema validate failed: invalid field type in 'fields' list, expected FieldSchema, "
+                    f"got {type(field).__name__} at index {idx}"
+                )
+
+            if field.name in _fields_name:
+                raise ValueError(
+                    f"schema validate failed: duplicate field name '{field.name}': field names must be unique"
+                )
+            _fields_name.append(field.name)
+            _fields_list.append(field._get_object())
+
+    def _check_vectors(
+        self,
+        vectors: Optional[Union[VectorSchema, list[VectorSchema]]],
+        _fields_name: list[str],
+        _fields_list: list[_FieldSchema],
+    ) -> None:
+        # handle vector
+        if isinstance(vectors, VectorSchema):
+            vectors_items = [vectors]
+        elif isinstance(vectors, list):
+            vectors_items = vectors
+        elif vectors is None:
+            vectors_items = []
+        else:
+            raise TypeError(
+                f"schema validate failed: invalid 'vectors' type, expected VectorSchema or list[VectorSchema], "
+                f"got {type(vectors).__name__}"
+            )
+
+        for idx, vector in enumerate(vectors_items):
+            if not isinstance(vector, VectorSchema):
+                raise TypeError(
+                    f"schema validate failed: invalid vector type in 'vectors' list, expected VectorSchema, "
+                    f"got {type(vector).__name__} at index {idx}"
+                )
+
+            if vector.name in _fields_name:
+                raise ValueError(
+                    f"schema validate failed: duplicate vector name '{vector.name}', vector names must be unique "
+                    f"(conflicts with existing field or vector)"
+                )
+            _fields_name.append(vector.name)
+            _fields_list.append(vector._get_object())
+
+    @classmethod
+    def _from_core(cls, core_collection_schema: _CollectionSchema):
+        inst = cls.__new__(cls)
+        if not core_collection_schema:
+            raise ValueError("schema validate failed: schema is null")
+        inst._cpp_obj = core_collection_schema
+        return inst
+
+    @property
+    def name(self) -> str:
+        """str: The name of the collection."""
+        return self._cpp_obj.name
+
+    def field(self, name: str) -> Optional[FieldSchema]:
+        """Retrieve a scalar field by name.
+
+        Args:
+            name (str): Name of the field.
+
+        Returns:
+            Optional[FieldSchema]: The field if found, otherwise None.
+        """
+        _field = self._cpp_obj.get_forward_field(name)
+        return FieldSchema._from_core(_field) if _field else None
+
+    def vector(self, name: str) -> Optional[VectorSchema]:
+        """Retrieve a vector field by name.
+
+        Args:
+            name (str): Name of the vector field.
+
+        Returns:
+            Optional[VectorSchema]: The vector field if found, otherwise None.
+        """
+        _field = self._cpp_obj.get_vector_field(name)
+        return VectorSchema._from_core(_field) if _field else None
+
+    @property
+    def fields(self) -> list[FieldSchema]:
+        """list[FieldSchema]: All scalar (non-vector) fields in the schema."""
+        _fields = self._cpp_obj.forward_fields()
+        return [FieldSchema._from_core(_field) for _field in _fields]
+
+    @property
+    def vectors(self) -> list[VectorSchema]:
+        """list[VectorSchema]: All vector fields in the schema."""
+        _vectors = self._cpp_obj.vector_fields()
+        return [VectorSchema._from_core(_vector) for _vector in _vectors]
+
+    def _get_object(self) -> _CollectionSchema:
+        return self._cpp_obj
+
+    def __repr__(self) -> str:
+        try:
+            schema = {
+                "name": self.name,
+                "fields": {field.name: field.__dict__() for field in self.fields},
+                "vectors": {vector.name: vector.__dict__() for vector in self.vectors},
+            }
+            return json.dumps(schema, indent=2, ensure_ascii=False)
+        except Exception as e:
+            return f"<CollectionSchema error during repr: {e}>"
+
+    def __str__(self) -> str:
+        return self.__repr__()

python/zvec/model/schema/field_schema.py

@@ -0,0 +1,300 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import json
+from typing import Any, Optional, Union
+
+from _zvec.schema import _FieldSchema
+
+from zvec.model.param import (
+    FlatIndexParam,
+    HnswIndexParam,
+    InvertIndexParam,
+    IVFIndexParam,
+)
+from zvec.typing import DataType
+
+__all__ = [
+    "FieldSchema",
+    "VectorSchema",
+]
+
+SUPPORT_VECTOR_DATA_TYPE = [
+    DataType.VECTOR_FP16,
+    DataType.VECTOR_FP32,
+    DataType.VECTOR_FP64,
+    DataType.VECTOR_INT8,
+    DataType.SPARSE_VECTOR_FP16,
+    DataType.SPARSE_VECTOR_FP32,
+]
+
+SUPPORT_SCALAR_DATA_TYPE = [
+    DataType.INT32,
+    DataType.INT64,
+    DataType.UINT32,
+    DataType.UINT64,
+    DataType.FLOAT,
+    DataType.DOUBLE,
+    DataType.STRING,
+    DataType.BOOL,
+    DataType.ARRAY_INT32,
+    DataType.ARRAY_INT64,
+    DataType.ARRAY_UINT32,
+    DataType.ARRAY_UINT64,
+    DataType.ARRAY_FLOAT,
+    DataType.ARRAY_DOUBLE,
+    DataType.ARRAY_STRING,
+    DataType.ARRAY_BOOL,
+]
+
+
+class FieldSchema:
+    """Represents a scalar (non-vector) field in a collection schema.
+
+    A `FieldSchema` defines the name, data type, nullability, and optional
+    inverted index configuration for a regular field (e.g., ID, timestamp, category).
+
+    Args:
+        name (str): Name of the field. Must be unique within the collection.
+        data_type (DataType): Data type of the field (e.g., INT64, STRING).
+        nullable (bool, optional): Whether the field can contain null values.
+            Defaults to False.
+        index_param (Optional[InvertIndexParam], optional): Inverted index
+            parameters for this field. Only applicable to fields that support
+            indexing (e.g., scalar fields used in filtering). Defaults to None.
+
+    Examples:
+        >>> from zvec.typing import DataType
+        >>> from zvec.model.param import InvertIndexParam
+        >>> id_field = FieldSchema(
+        ...     name="id",
+        ...     data_type=DataType.INT64,
+        ...     nullable=False,
+        ...     index_param=InvertIndexParam(enable_range_optimization=True)
+        ... )
+    """
+
+    def __init__(
+        self,
+        name: str,
+        data_type: DataType,
+        nullable: bool = False,
+        index_param: Optional[InvertIndexParam] = None,
+    ):
+        if name is None or not isinstance(name, str):
+            raise ValueError(
+                f"schema validate failed: field name must be str, got {type(name).__name__}"
+            )
+
+        if data_type not in SUPPORT_SCALAR_DATA_TYPE:
+            raise ValueError(
+                f"schema validate failed: scalar_field's data_type must be one of "
+                f"{', '.join(str(dt) for dt in SUPPORT_SCALAR_DATA_TYPE)}, "
+                f"but field[{name}]'s data_type is {data_type}"
+            )
+
+        self._cpp_obj = _FieldSchema(
+            name=name,
+            data_type=data_type,
+            dimension=0,
+            nullable=nullable,
+            index_param=index_param,
+        )
+
+    @classmethod
+    def _from_core(cls, core_field_schema: _FieldSchema):
+        if core_field_schema is None:
+            raise ValueError("schema validate failed: field schema is None")
+        inst = cls.__new__(cls)
+        inst._cpp_obj = core_field_schema
+        return inst
+
+    def _get_object(self) -> _FieldSchema:
+        return self._cpp_obj
+
+    @property
+    def name(self) -> str:
+        """str: The name of the field."""
+        return self._cpp_obj.name
+
+    @property
+    def data_type(self) -> DataType:
+        """DataType: The data type of the field (e.g., INT64, STRING)."""
+        return self._cpp_obj.data_type
+
+    @property
+    def nullable(self) -> bool:
+        """bool: Whether the field allows null values."""
+        return self._cpp_obj.nullable
+
+    @property
+    def index_param(self) -> Optional[InvertIndexParam]:
+        """Optional[InvertIndexParam]: Inverted index configuration, if any."""
+        return self._cpp_obj.index_param
+
+    def __dict__(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "data_type": (
+                self.data_type.name
+                if hasattr(self.data_type, "name")
+                else str(self.data_type)
+            ),
+            "nullable": self.nullable,
+            "index_param": (
+                self.index_param.to_dict() if self.index_param is not None else None
+            ),
+        }
+
+    def __repr__(self) -> str:
+        try:
+            schema = self.__dict__()
+            return json.dumps(schema, indent=2, ensure_ascii=False)
+        except Exception as e:
+            return f"<FieldSchema error during repr: {e}>"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, FieldSchema):
+            return False
+        return self._cpp_obj == other._cpp_obj
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.data_type, self.nullable))
+
+
+class VectorSchema:
+    """Represents a vector field in a collection schema.
+
+    A `VectorSchema` defines the name, data type, dimensionality, and index
+    configuration for a vector field used in similarity search.
+
+    Args:
+        name (str): Name of the vector field. Must be unique within the collection.
+        data_type (DataType): Vector data type (e.g., VECTOR_FP32, VECTOR_INT8).
+        dimension (int, optional): Dimensionality of the vector. Must be > 0 for dense vectors;
+         may be `None` for sparse vectors.
+        index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam], optional):
+            Index configuration for this vector field. Defaults to
+            ``HnswIndexParam()``.
+
+    Examples:
+        >>> from zvec.typing import DataType
+        >>> from zvec.model.param import HnswIndexParam
+        >>> emb_field = VectorSchema(
+        ...     name="embedding",
+        ...     data_type=DataType.VECTOR_FP32,
+        ...     dimension=128,
+        ...     index_param=HnswIndexParam(ef_construction=200, m=16)
+        ... )
+    """
+
+    def __init__(
+        self,
+        name: str,
+        data_type: DataType,
+        dimension: Optional[int] = 0,
+        index_param: Optional[
+            Union[HnswIndexParam, FlatIndexParam, IVFIndexParam]
+        ] = None,
+    ):
+        if name is None or not isinstance(name, str):
+            raise ValueError(
+                f"schema validate failed: field name must be str, got {type(name).__name__}"
+            )
+
+        if not isinstance(dimension, int) or dimension < 0:
+            raise ValueError("schema validate failed: vector's dimension must be >= 0")
+
+        if data_type not in SUPPORT_VECTOR_DATA_TYPE:
+            raise ValueError(
+                f"schema validate failed: vector's data_type must be one of "
+                f"{', '.join(str(dt) for dt in SUPPORT_VECTOR_DATA_TYPE)}, "
+                f"but field[{name}]'s data_type is {data_type}"
+            )
+
+        if index_param is None:
+            index_param = FlatIndexParam()
+
+        self._cpp_obj = _FieldSchema(
+            name=name,
+            data_type=data_type,
+            dimension=dimension,
+            nullable=False,
+            index_param=index_param,
+        )
+
+    @classmethod
+    def _from_core(cls, core_field_schema: _FieldSchema):
+        inst = cls.__new__(cls)
+        inst._cpp_obj = core_field_schema
+        return inst
+
+    def _get_object(self) -> _FieldSchema:
+        return self._cpp_obj
+
+    @property
+    def name(self) -> str:
+        """str: The name of the vector field."""
+        return self._cpp_obj.name
+
+    @property
+    def data_type(self) -> DataType:
+        """DataType: The vector data type (e.g., VECTOR_FP32)."""
+        return self._cpp_obj.data_type
+
+    @property
+    def dimension(self) -> int:
+        """int: The dimensionality of the vector."""
+        return self._cpp_obj.dimension
+
+    @property
+    def index_param(self) -> Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]:
+        """Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]: Index configuration for the vector."""
+        return self._cpp_obj.index_param
+
+    def __dict__(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "data_type": (
+                self.data_type.name
+                if hasattr(self.data_type, "name")
+                else str(self.data_type)
+            ),
+            "dimension": self.dimension,
+            "index_param": (
+                self.index_param.to_dict() if self.index_param is not None else None
+            ),
+        }
+
+    def __repr__(self) -> str:
+        try:
+            schema = self.__dict__()
+            return json.dumps(schema, indent=2, ensure_ascii=False)
+        except Exception as e:
+            return f"<FieldSchema error during repr: {e}>"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, VectorSchema):
+            return False
+        return self._cpp_obj == other._cpp_obj
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.data_type, self.dimension))

python/zvec/tool/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .util import require_module
+
+__all__ = ["require_module"]

python/zvec/tool/util.py

@@ -0,0 +1,63 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import importlib
+from typing import Any, Optional
+
+
+def require_module(module: str, mitigation: Optional[str] = None) -> Any:
+    """Import a Python module and raise a user-friendly error if it is not available.
+
+    This utility helps provide actionable error messages when optional dependencies
+    are missing. It attempts to import the given module and, on failure, suggests
+    a `pip install` command based on either the module name or an optional
+    mitigation package name.
+
+    Args:
+        module (str): The full module name to import (e.g., ``"numpy"``, ``"pandas.io.parquet"``).
+        mitigation (Optional[str], optional): The package name to suggest for installation
+            if the import fails. If not provided, the top-level package of `module`
+            will be used (e.g., ``"pandas"`` for ``"pandas.io.parquet"``).
+
+    Returns:
+        Any: The imported module object.
+
+    Raises:
+        ImportError: If the module cannot be imported, with a clear installation hint.
+
+    Examples:
+        >>> import zvec
+        >>> np = zvec.require_module("numpy")
+        >>> pq = zvec.require_module("pyarrow.parquet", mitigation="pyarrow")
+
+    Note:
+        This function is intended for lazy-loading optional dependencies
+        with helpful error messages, not for core dependencies.
+    """
+    try:
+        return importlib.import_module(module)
+    except ImportError as e:
+        package = mitigation or module
+        msg = f"Required package '{package}' is not installed. "
+        if "." in module:
+            top_level = module.split(".")[0]
+            msg += f"Module '{module}' is part of '{top_level}', "
+            if mitigation:
+                msg += f"please pip install '{mitigation}'."
+            else:
+                msg += f"please pip install '{top_level}'."
+        else:
+            msg += f"Please pip install '{package}."
+        raise ImportError(msg) from e

python/zvec/typing/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from _zvec.typing import (
+    DataType,
+    IndexType,
+    MetricType,
+    QuantizeType,
+    Status,
+    StatusCode,
+)
+
+__all__ = [
+    "DataType",
+    "IndexType",
+    "MetricType",
+    "QuantizeType",
+    "Status",
+    "StatusCode",
+]

python/zvec/typing/__init__.pyi

@@ -0,0 +1,404 @@
+"""
+This module contains the basic data types of Zvec
+"""
+
+from __future__ import annotations
+
+import typing
+
+__all__: list[str] = [
+    "DataType",
+    "IndexType",
+    "MetricType",
+    "QuantizeType",
+    "Status",
+    "StatusCode",
+]
+
+class DataType:
+    """
+
+    Enumeration of supported data types in Zvec.
+
+    Includes scalar types, dense/sparse vector types, and array types.
+
+    Examples:
+        >>> import zvec
+        >>> print(zvec.DataType.FLOAT)
+        DataType.FLOAT
+        >>> print(zvec.DataType.VECTOR_FP32)
+        DataType.VECTOR_FP32
+
+
+    Members:
+
+      STRING
+
+      BOOL
+
+      INT32
+
+      INT64
+
+      FLOAT
+
+      DOUBLE
+
+      UINT32
+
+      UINT64
+
+      VECTOR_FP16
+
+      VECTOR_FP32
+
+      VECTOR_FP64
+
+      VECTOR_INT8
+
+      SPARSE_VECTOR_FP32
+
+      SPARSE_VECTOR_FP16
+
+      ARRAY_STRING
+
+      ARRAY_INT32
+
+      ARRAY_INT64
+
+      ARRAY_FLOAT
+
+      ARRAY_DOUBLE
+
+      ARRAY_BOOL
+
+      ARRAY_UINT32
+
+      ARRAY_UINT64
+    """
+
+    ARRAY_BOOL: typing.ClassVar[DataType]  # value = <DataType.ARRAY_BOOL: 42>
+    ARRAY_DOUBLE: typing.ClassVar[DataType]  # value = <DataType.ARRAY_DOUBLE: 48>
+    ARRAY_FLOAT: typing.ClassVar[DataType]  # value = <DataType.ARRAY_FLOAT: 47>
+    ARRAY_INT32: typing.ClassVar[DataType]  # value = <DataType.ARRAY_INT32: 43>
+    ARRAY_INT64: typing.ClassVar[DataType]  # value = <DataType.ARRAY_INT64: 44>
+    ARRAY_STRING: typing.ClassVar[DataType]  # value = <DataType.ARRAY_STRING: 41>
+    ARRAY_UINT32: typing.ClassVar[DataType]  # value = <DataType.ARRAY_UINT32: 45>
+    ARRAY_UINT64: typing.ClassVar[DataType]  # value = <DataType.ARRAY_UINT64: 46>
+    BOOL: typing.ClassVar[DataType]  # value = <DataType.BOOL: 3>
+    DOUBLE: typing.ClassVar[DataType]  # value = <DataType.DOUBLE: 9>
+    FLOAT: typing.ClassVar[DataType]  # value = <DataType.FLOAT: 8>
+    INT32: typing.ClassVar[DataType]  # value = <DataType.INT32: 4>
+    INT64: typing.ClassVar[DataType]  # value = <DataType.INT64: 5>
+    SPARSE_VECTOR_FP16: typing.ClassVar[
+        DataType
+    ]  # value = <DataType.SPARSE_VECTOR_FP16: 30>
+    SPARSE_VECTOR_FP32: typing.ClassVar[
+        DataType
+    ]  # value = <DataType.SPARSE_VECTOR_FP32: 31>
+    STRING: typing.ClassVar[DataType]  # value = <DataType.STRING: 2>
+    UINT32: typing.ClassVar[DataType]  # value = <DataType.UINT32: 6>
+    UINT64: typing.ClassVar[DataType]  # value = <DataType.UINT64: 7>
+    VECTOR_FP16: typing.ClassVar[DataType]  # value = <DataType.VECTOR_FP16: 22>
+    VECTOR_FP32: typing.ClassVar[DataType]  # value = <DataType.VECTOR_FP32: 23>
+    VECTOR_FP64: typing.ClassVar[DataType]  # value = <DataType.VECTOR_FP64: 24>
+    VECTOR_INT8: typing.ClassVar[DataType]  # value = <DataType.VECTOR_INT8: 26>
+    __members__: typing.ClassVar[
+        dict[str, DataType]
+    ]  # value = {'STRING': <DataType.STRING: 2>, 'BOOL': <DataType.BOOL: 3>, 'INT32': <DataType.INT32: 4>, 'INT64': <DataType.INT64: 5>, 'FLOAT': <DataType.FLOAT: 8>, 'DOUBLE': <DataType.DOUBLE: 9>, 'UINT32': <DataType.UINT32: 6>, 'UINT64': <DataType.UINT64: 7>, 'VECTOR_FP16': <DataType.VECTOR_FP16: 22>, 'VECTOR_FP32': <DataType.VECTOR_FP32: 23>, 'VECTOR_FP64': <DataType.VECTOR_FP64: 24>, 'VECTOR_INT8': <DataType.VECTOR_INT8: 26>, 'SPARSE_VECTOR_FP32': <DataType.SPARSE_VECTOR_FP32: 31>, 'SPARSE_VECTOR_FP16': <DataType.SPARSE_VECTOR_FP16: 30>, 'ARRAY_STRING': <DataType.ARRAY_STRING: 41>, 'ARRAY_INT32': <DataType.ARRAY_INT32: 43>, 'ARRAY_INT64': <DataType.ARRAY_INT64: 44>, 'ARRAY_FLOAT': <DataType.ARRAY_FLOAT: 47>, 'ARRAY_DOUBLE': <DataType.ARRAY_DOUBLE: 48>, 'ARRAY_BOOL': <DataType.ARRAY_BOOL: 42>, 'ARRAY_UINT32': <DataType.ARRAY_UINT32: 45>, 'ARRAY_UINT64': <DataType.ARRAY_UINT64: 46>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class IndexType:
+    """
+
+    Enumeration of supported index types in Zvec.
+
+    Examples:
+        >>> import zvec
+        >>> print(zvec.IndexType.HNSW)
+        IndexType.HNSW
+
+
+    Members:
+
+      UNDEFINED
+
+      HNSW
+
+      IVF
+
+      FLAT
+
+      INVERT
+    """
+
+    FLAT: typing.ClassVar[IndexType]  # value = <IndexType.FLAT: 4>
+    HNSW: typing.ClassVar[IndexType]  # value = <IndexType.HNSW: 1>
+    INVERT: typing.ClassVar[IndexType]  # value = <IndexType.INVERT: 10>
+    IVF: typing.ClassVar[IndexType]  # value = <IndexType.IVF: 3>
+    UNDEFINED: typing.ClassVar[IndexType]  # value = <IndexType.UNDEFINED: 0>
+    __members__: typing.ClassVar[
+        dict[str, IndexType]
+    ]  # value = {'UNDEFINED': <IndexType.UNDEFINED: 0>, 'HNSW': <IndexType.HNSW: 1>, 'IVF': <IndexType.IVF: 3>, 'FLAT': <IndexType.FLAT: 4>, 'INVERT': <IndexType.INVERT: 10>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class MetricType:
+    """
+
+    Enumeration of supported distance/similarity metrics.
+
+    - COSINE: Cosine similarity.
+    - IP: Inner product (dot product).
+    - L2: Euclidean distance (L2 norm).
+
+    Examples:
+        >>> import zvec
+        >>> print(zvec.MetricType.COSINE)
+        MetricType.COSINE
+
+
+    Members:
+
+      COSINE
+
+      IP
+
+      L2
+    """
+
+    COSINE: typing.ClassVar[MetricType]  # value = <MetricType.COSINE: 3>
+    IP: typing.ClassVar[MetricType]  # value = <MetricType.IP: 2>
+    L2: typing.ClassVar[MetricType]  # value = <MetricType.L2: 1>
+    __members__: typing.ClassVar[
+        dict[str, MetricType]
+    ]  # value = {'COSINE': <MetricType.COSINE: 3>, 'IP': <MetricType.IP: 2>, 'L2': <MetricType.L2: 1>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class QuantizeType:
+    """
+
+    Enumeration of supported quantization types for vector compression.
+
+    Examples:
+        >>> import zvec
+        >>> print(zvec.QuantizeType.INT8)
+        QuantizeType.INT8
+
+
+    Members:
+
+      UNDEFINED
+
+      FP16
+
+      INT8
+
+      INT4
+    """
+
+    FP16: typing.ClassVar[QuantizeType]  # value = <QuantizeType.FP16: 1>
+    INT4: typing.ClassVar[QuantizeType]  # value = <QuantizeType.INT4: 3>
+    INT8: typing.ClassVar[QuantizeType]  # value = <QuantizeType.INT8: 2>
+    UNDEFINED: typing.ClassVar[QuantizeType]  # value = <QuantizeType.UNDEFINED: 0>
+    __members__: typing.ClassVar[
+        dict[str, QuantizeType]
+    ]  # value = {'UNDEFINED': <QuantizeType.UNDEFINED: 0>, 'FP16': <QuantizeType.FP16: 1>, 'INT8': <QuantizeType.INT8: 2>, 'INT4': <QuantizeType.INT4: 3>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class Status:
+    """
+
+    Represents the outcome of a Zvec operation.
+
+    A `Status` object is either OK (success) or carries an error code and message.
+
+    Examples:
+        >>> from zvec.typing import Status, StatusCode
+        >>> s = Status()
+        >>> print(s.ok())
+        True
+        >>> s = Status(StatusCode.INVALID_ARGUMENT, "Field not found")
+        >>> print(s.code() == StatusCode.INVALID_ARGUMENT)
+        True
+        >>> print(s.message())
+        Field not found
+    """
+
+    __hash__: typing.ClassVar[None] = None
+
+    @staticmethod
+    def AlreadyExists(message: str) -> Status: ...
+    @staticmethod
+    def InternalError(message: str) -> Status: ...
+    @staticmethod
+    def InvalidArgument(message: str) -> Status: ...
+    @staticmethod
+    def NotFound(message: str) -> Status: ...
+    @staticmethod
+    def OK() -> Status:
+        """
+        Create an OK status.
+        """
+
+    @staticmethod
+    def PermissionDenied(message: str) -> Status: ...
+    def __eq__(self, arg0: Status) -> bool: ...
+    @typing.overload
+    def __init__(self) -> None: ...
+    @typing.overload
+    def __init__(self, code: StatusCode, message: str = "") -> None:
+        """
+        Construct a status with the given code and optional message.
+
+        Args:
+            code (StatusCode): The status code.
+            message (str, optional): Error message. Defaults to empty string.
+        """
+
+    def __ne__(self, arg0: Status) -> bool: ...
+    def __repr__(self) -> str: ...
+    def code(self) -> StatusCode:
+        """
+        StatusCode: Returns the status code.
+        """
+
+    def message(self) -> str:
+        """
+        str: Returns the error message (may be empty).
+        """
+
+    def ok(self) -> bool:
+        """
+        bool: Returns True if the status is OK.
+        """
+
+class StatusCode:
+    """
+
+    Enumeration of possible status codes for Zvec operations.
+
+    Used by the `Status` class to indicate success or failure reason.
+
+
+    Members:
+
+      OK
+
+      NOT_FOUND
+
+      ALREADY_EXISTS
+
+      INVALID_ARGUMENT
+
+      PERMISSION_DENIED
+
+      FAILED_PRECONDITION
+
+      RESOURCE_EXHAUSTED
+
+      UNAVAILABLE
+
+      INTERNAL_ERROR
+
+      NOT_SUPPORTED
+
+      UNKNOWN
+    """
+
+    ALREADY_EXISTS: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.ALREADY_EXISTS: 2>
+    FAILED_PRECONDITION: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.FAILED_PRECONDITION: 5>
+    INTERNAL_ERROR: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.INTERNAL_ERROR: 8>
+    INVALID_ARGUMENT: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.INVALID_ARGUMENT: 3>
+    NOT_FOUND: typing.ClassVar[StatusCode]  # value = <StatusCode.NOT_FOUND: 1>
+    NOT_SUPPORTED: typing.ClassVar[StatusCode]  # value = <StatusCode.NOT_SUPPORTED: 9>
+    OK: typing.ClassVar[StatusCode]  # value = <StatusCode.OK: 0>
+    PERMISSION_DENIED: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.PERMISSION_DENIED: 4>
+    RESOURCE_EXHAUSTED: typing.ClassVar[
+        StatusCode
+    ]  # value = <StatusCode.RESOURCE_EXHAUSTED: 6>
+    UNAVAILABLE: typing.ClassVar[StatusCode]  # value = <StatusCode.UNAVAILABLE: 7>
+    UNKNOWN: typing.ClassVar[StatusCode]  # value = <StatusCode.UNKNOWN: 10>
+    __members__: typing.ClassVar[
+        dict[str, StatusCode]
+    ]  # value = {'OK': <StatusCode.OK: 0>, 'NOT_FOUND': <StatusCode.NOT_FOUND: 1>, 'ALREADY_EXISTS': <StatusCode.ALREADY_EXISTS: 2>, 'INVALID_ARGUMENT': <StatusCode.INVALID_ARGUMENT: 3>, 'PERMISSION_DENIED': <StatusCode.PERMISSION_DENIED: 4>, 'FAILED_PRECONDITION': <StatusCode.FAILED_PRECONDITION: 5>, 'RESOURCE_EXHAUSTED': <StatusCode.RESOURCE_EXHAUSTED: 6>, 'UNAVAILABLE': <StatusCode.UNAVAILABLE: 7>, 'INTERNAL_ERROR': <StatusCode.INTERNAL_ERROR: 8>, 'NOT_SUPPORTED': <StatusCode.NOT_SUPPORTED: 9>, 'UNKNOWN': <StatusCode.UNKNOWN: 10>}
+
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: typing.SupportsInt) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: typing.SupportsInt) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...

python/zvec/typing/enum.py

@@ -0,0 +1,62 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from enum import IntEnum
+
+__all__ = ["LogLevel", "LogType"]
+
+
+class LogLevel(IntEnum):
+    """Enumeration of logging severity levels, ordered from lowest to highest priority.
+
+    Used to control verbosity and filtering of log messages. Higher numeric values
+    indicate more severe conditions.
+
+    Note:
+        ``WARNING`` is an alias for ``WARN`` to match Python's built-in :mod:`logging`
+        module convention.
+
+    Attributes:
+        DEBUG (int): Detailed information, typically of interest only when diagnosing problems.
+        INFO (int): Confirmation that things are working as expected.
+        WARN (int): An indication that something unexpected happened, or indicative of
+            potential future problems. (Alias: ``WARNING``)
+        WARNING (int): Same as ``WARN``.
+        ERROR (int): Due to a more serious problem, the software has not been able
+            to perform some function.
+        FATAL (int): A serious error, indicating that the program itself may be unable
+            to continue running.
+    """
+
+    DEBUG = 0
+    INFO = 1
+    WARN = 2
+    WARNING = 2
+    ERROR = 3
+    FATAL = 4
+
+
+class LogType(IntEnum):
+    """Enumeration of supported log output destinations.
+
+    Specifies where log messages should be written.
+
+    Attributes:
+        CONSOLE (int): Output logs to standard output/error (e.g., terminal or IDE console).
+        FILE (int): Write logs to a persistent file on disk.
+    """
+
+    CONSOLE = 0
+    FILE = 1

python/zvec/zvec.py

@@ -0,0 +1,226 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional
+
+from _zvec import Initialize, _Collection
+
+from .model import Collection
+from .model.param import CollectionOption
+from .model.schema import CollectionSchema
+
+__all__ = ["create_and_open", "init", "open"]
+
+from .typing.enum import LogLevel, LogType
+
+
+def init(
+    *,
+    log_type: Optional[LogType] = LogType.CONSOLE,
+    log_level: Optional[LogLevel] = LogLevel.WARN,
+    log_dir: Optional[str] = "./logs",
+    log_basename: Optional[str] = "zvec.log",
+    log_file_size: Optional[int] = 2048,
+    log_overdue_days: Optional[int] = 7,
+    query_threads: Optional[int] = None,
+    optimize_threads: Optional[int] = None,
+    invert_to_forward_scan_ratio: Optional[float] = None,
+    brute_force_by_keys_ratio: Optional[float] = None,
+    memory_limit_mb: Optional[int] = None,
+) -> None:
+    """Initialize Zvec with configuration options.
+
+    This function must be called before any other operation.
+    It can only be called once — subsequent calls raise a ``RuntimeError``.
+
+    Parameters set to ``None`` are **omitted** from the configuration and
+    fall back to Zvec's internal defaults, which may be derived from the runtime
+    environment (e.g., cgroup CPU/memory limits). Explicitly provided values
+    always override defaults.
+
+    Args:
+        log_type (Optional[LogType], optional): Logger destination.
+            - ``LogType.CONSOLE`` (default if omitted or set to this)
+            - ``LogType.FILE``
+            - If ``None``, uses internal default (currently ``CONSOLE``).
+        log_level (Optional[LogLevel], optional): Minimum log severity.
+            Default: ``LogLevel.WARN``.
+            Accepted values: ``DEBUG``, ``INFO``, ``WARN``, ``ERROR``, ``FATAL``.
+            If ``None``, uses internal default (``WARN``).
+        log_dir (Optional[str], optional):
+            Directory for log files (only used when ``log_type=FILE``).
+            Parent directories are **not** created automatically.
+            Default: ``"./logs"``.
+            If ``None``, internal default is used.
+        log_basename (Optional[str], optional):
+            Base name for rotated log files (e.g., ``zvec.log.1``, ``zvec.log.2``).
+            Default: ``"zvec.log"``.
+        log_file_size (Optional[int], optional):
+            Max size per log file in **MB** before rotation.
+            Default: ``2048`` MB (2 GB).
+        log_overdue_days (Optional[int], optional):
+            Days to retain rotated log files before deletion.
+            Default: ``7`` days.
+        query_threads (Optional[int], optional):
+            Number of threads for query execution.
+            If ``None`` (default), inferred from available CPU cores (via cgroup).
+            Must be ≥ 1 if provided.
+        optimize_threads (Optional[int], optional):
+            Threads for background tasks (e.g., compaction, indexing).
+            If ``None``, defaults to same as ``query_threads`` or CPU count.
+        invert_to_forward_scan_ratio (Optional[float], optional):
+            Threshold to switch from inverted index to full forward scan.
+            Range: [0.0, 1.0]. Higher → more aggressive index skipping.
+            Default: ``0.9`` (if omitted).
+        brute_force_by_keys_ratio (Optional[float], optional):
+            Threshold to use brute-force key lookup over index.
+            Lower → prefer index; higher → prefer brute-force.
+            Range: [0.0, 1.0]. Default: ``0.1``.
+        memory_limit_mb (Optional[int], optional):
+            Soft memory cap in MB. Zvec may throttle or fail operations
+            approaching this limit.
+            If ``None``, inferred from cgroup memory limit * 0.8 (e.g., in Docker).
+            Must be > 0 if provided.
+
+    Raises:
+        RuntimeError: If Zvec is already initialized.
+        ValueError: On invalid values (e.g., negative thread count, log level out of range).
+        TypeError: If a value has incorrect type (e.g., string for ``query_threads``).
+
+    Note:
+        - All ``None`` arguments are **excluded** from the configuration payload,
+          allowing the core library to apply environment-aware defaults.
+        - This design ensures container-friendliness: in Kubernetes/Docker,
+          omitting ``memory_limit_mb`` and thread counts lets Zvec auto-adapt.
+
+    Examples:
+        Initialize with defaults (log to console, auto-detect resources):
+        >>> import zvec
+        >>> zvec.init()
+
+        Customize logging to file with rotation:
+        >>> zvec.init(
+        ...     log_type=LogType.FILE,
+        ...     log_dir="/var/log/zvec",
+        ...     log_file_size=1024,
+        ...     log_overdue_days=30
+        ... )
+
+        Limit resources explicitly:
+        >>> zvec.init(
+        ...     memory_limit_mb=2048,
+        ...     query_threads=4,
+        ...     optimize_threads=2
+        ... )
+
+        Fine-tune query heuristics:
+        >>> zvec.init(
+        ...     invert_to_forward_scan_ratio=0.95,
+        ...     brute_force_by_keys_ratio=0.05
+        ... )
+    """
+    # Build config dict, skipping None values
+    config_dict = {}
+    if log_type is not None:
+        if not isinstance(log_type, LogType):
+            raise TypeError("log_type must be LogType")
+        config_dict["log_type"] = log_type.name
+    if log_level is not None:
+        if not isinstance(log_level, LogLevel):
+            raise TypeError("log_level must be LogLevel")
+        config_dict["log_level"] = log_level.name
+    if log_dir is not None:
+        config_dict["log_dir"] = log_dir
+    if log_basename is not None:
+        config_dict["log_basename"] = log_basename
+    if log_file_size is not None:
+        config_dict["log_file_size"] = log_file_size
+    if log_overdue_days is not None:
+        config_dict["log_overdue_days"] = log_overdue_days
+    if query_threads is not None:
+        config_dict["query_threads"] = query_threads
+    if optimize_threads is not None:
+        config_dict["optimize_threads"] = optimize_threads
+    if invert_to_forward_scan_ratio is not None:
+        config_dict["invert_to_forward_scan_ratio"] = invert_to_forward_scan_ratio
+    if brute_force_by_keys_ratio is not None:
+        config_dict["brute_force_by_keys_ratio"] = brute_force_by_keys_ratio
+    if memory_limit_mb is not None:
+        config_dict["memory_limit_mb"] = memory_limit_mb
+
+    Initialize(config_dict)
+
+
+def create_and_open(
+    path: str,
+    schema: CollectionSchema,
+    option: Optional[CollectionOption] = None,
+) -> Collection:
+    """Create a new collection and open it for use.
+
+    If a collection already exists at the given path, it may raise an error
+    depending on the underlying implementation.
+
+    Args:
+        path (str): Path or name of the collection to create.
+        schema (CollectionSchema): Schema defining the structure of the collection.
+        option (CollectionOption): Configuration options
+            for opening the collection. Defaults to a default-constructed
+            ``CollectionOption()`` if not provided.
+
+    Returns:
+        Collection: An opened collection instance ready for operations.
+
+    Examples:
+        >>> import zvec
+        >>> schema = zvec.CollectionSchema(
+        ...     name="my_collection",
+        ...     fields=[zvec.FieldSchema("id", zvec.DataType.INT64, nullable=True)]
+        ... )
+        >>> coll = create_and_open("./my_collection", schema)
+    """
+    if not isinstance(path, str):
+        raise TypeError("path must be a string")
+    if not isinstance(schema, CollectionSchema):
+        raise TypeError("schema must be a CollectionSchema")
+
+    option = option or CollectionOption()
+    if not isinstance(option, CollectionOption):
+        raise TypeError("option must be a CollectionOption")
+
+    _collection = _Collection.CreateAndOpen(path, schema._get_object(), option)
+    return Collection._from_core(_collection)
+
+
+def open(path: str, option: CollectionOption = CollectionOption()) -> Collection:
+    """Open an existing collection from disk.
+
+    The collection must have been previously created with ``create_and_open``.
+
+    Args:
+        path (str): Path or name of the existing collection.
+        option (CollectionOption): Configuration options
+            for opening the collection. Defaults to a default-constructed
+            ``CollectionOption()`` if not provided.
+
+    Returns:
+        Collection: An opened collection instance.
+
+    Examples:
+        >>> import zvec
+        >>> coll = zvec.open("./my_collection")
+    """
+    _collection = _Collection.Open(path, option)
+    return Collection._from_core(_collection)