zvec/src/db/proto/zvec.proto

syntax = "proto3";

package zvec.proto;

option cc_enable_arenas = true;

// The Go package name, refers to
// https://developers.google.com/protocol-buffers/docs/reference/go-generated#package
option go_package = "proxima/zvec/proto";

/*! Types of Data
 */
enum DataType {
  DT_UNDEFINED = 0;

  DT_BINARY = 1;
  DT_STRING = 2;
  DT_BOOL = 3;
  DT_INT32 = 4;
  DT_INT64 = 5;
  DT_UINT32 = 6;
  DT_UINT64 = 7;
  DT_FLOAT = 8;
  DT_DOUBLE = 9;

  DT_VECTOR_BINARY32 = 20;
  DT_VECTOR_BINARY64 = 21;
  DT_VECTOR_FP16 = 22;
  DT_VECTOR_FP32 = 23;
  DT_VECTOR_FP64 = 24;
  DT_VECTOR_INT4 = 25;
  DT_VECTOR_INT8 = 26;
  DT_VECTOR_INT16 = 27;

  DT_SPARSE_VECTOR_FP16 = 30;
  DT_SPARSE_VECTOR_FP32 = 31;

  // ARRAY
  DT_ARRAY_BINARY = 40;
  DT_ARRAY_STRING = 41;
  DT_ARRAY_BOOL = 42;
  DT_ARRAY_INT32 = 43;
  DT_ARRAY_INT64 = 44;
  DT_ARRAY_UINT32 = 45;
  DT_ARRAY_UINT64 = 46;
  DT_ARRAY_FLOAT = 47;
  DT_ARRAY_DOUBLE = 48;
};

enum IndexType {
  // Undefined
  IT_UNDEFINED = 0;
  // Proxima HNSW Index
  IT_HNSW = 1;
  // Proxima IVF Index
  IT_IVF = 2;
  // Proxima FLAT Index
  IT_FLAT = 3;
  // Proxima HNSW RABITQ Index
  IT_HNSW_RABITQ = 4;
  // Proxima Vamana (DiskANN) Index
  IT_VAMANA = 5;
  // Invert Index
  IT_INVERT = 10;
};

enum QuantizeType {
  QT_UNDEFINED = 0;
  QT_FP16 = 1;
  QT_INT8 = 2;
  QT_INT4 = 3;
  QT_RABITQ = 4;
};

enum MetricType {
  MT_UNDEFINED = 0;
  MT_L2 = 1;
  MT_IP = 2;
  MT_COSINE = 3;
};

message InvertIndexParams {
  bool enable_range_optimization = 1;
};

message BaseIndexParams {
  MetricType metric_type = 1;
  QuantizeType quantize_type = 2;
};

message HnswIndexParams {
  BaseIndexParams base = 1;
  int32 m = 2;
  int32 ef_construction = 3;
  // When enabled, the HNSW streamer allocates a single contiguous memory
  // arena for all graph nodes, which improves cache locality / search
  // throughput at the cost of peak memory usage. Defaults to false.
  bool use_contiguous_memory = 4;
}

message HnswRabitqIndexParams {
  BaseIndexParams base = 1;
  int32 m = 2;
  int32 ef_construction = 3;
  int32 total_bits = 4;
  int32 num_clusters = 5;
  int32 sample_count = 6;
}

message FlatIndexParams {
  BaseIndexParams base = 1;
}

message IVFIndexParams {
  BaseIndexParams base = 1;
  int32 n_list = 2;
  int32 n_iters = 3;
  bool use_soar = 4;
}

message VamanaIndexParams {
  BaseIndexParams base = 1;
  int32 max_degree = 2;
  int32 search_list_size = 3;
  float alpha = 4;
  bool saturate_graph = 5;
  // When enabled, the Vamana streamer allocates a single contiguous memory
  // arena for all graph nodes, which improves cache locality / search
  // throughput at the cost of peak memory usage. Defaults to false.
  bool use_contiguous_memory = 6;
  bool use_id_map = 7;
}

message IndexParams {
  oneof params {
    InvertIndexParams invert = 1;
    HnswIndexParams hnsw = 2;
    FlatIndexParams flat = 3;
    IVFIndexParams ivf = 4;
    HnswRabitqIndexParams hnsw_rabitq = 5;
    VamanaIndexParams vamana = 6;
  };
};

message FieldSchema {
  string name = 1;
  DataType data_type = 2;
  uint32 dimension = 3;
  bool nullable = 4;
  IndexParams index_params = 5;
};

message CollectionSchema {
  string name = 1;
  repeated FieldSchema fields = 2;
  uint64 max_doc_count_per_segment = 3;
};

enum BlockType {
  BT_UNDEFINED = 0;
  BT_SCALAR = 1;
  BT_SCALAR_INDEX = 2;
  BT_VECTOR_INDEX = 3;
  BT_VECTOR_INDEX_QUANTIZE = 4;
};

message BlockMeta {
  uint32 block_id = 1;
  BlockType block_type = 2;  // for getting filename prefix
  uint64 min_doc_id = 3;
  uint64 max_doc_id = 4;
  uint64 doc_count = 5;
  repeated string columns = 6;  // columns contained in this block
};

// message AlterColumnMeta {
//   string old_column_name = 1;
//   FieldSchema new_schema = 2;
// };

message SegmentMeta {
  uint32 segment_id = 1;
  // scalar data, vector data and vector index
  repeated BlockMeta persisted_blocks = 2;

  BlockMeta writing_forward_block = 3;

  // if indexed, index_params can be retrieved from schema
  // if not indexed, index_params is default index_params(flat)
  repeated string indexed_vector_fields = 4;
  // repeated AlterColumnMeta alter_columns = 10;
};

message Manifest {
  uint32 version = 1;

  CollectionSchema schema = 2;

  bool enable_mmap = 3;

  repeated SegmentMeta persisted_segment_metas = 4;

  SegmentMeta writing_segment_meta = 5;

  uint32 id_map_path_suffix = 6;
  uint32 delete_snapshot_path_suffix = 7;

  uint32 next_segment_id = 8;
};