feat(index): expose is_dirty status through core Index interface (#488)

Adds is_dirty() at each layer: IndexMapping → IndexStorage (virtual) → MMapFileStorage / BufferStorage → Index::IsDirty().
This commit is contained in:
Jalin Wang
2026-06-17 11:20:40 +08:00
committed by GitHub
parent 841b1ed3f9
commit b1af0e4ebf
7 changed files with 148 additions and 1 deletions
+7
View File
@@ -381,6 +381,13 @@ int Index::Flush() {
return 0;
}
bool Index::IsDirty() const {
if (!storage_) {
return false;
}
return storage_->is_dirty();
}
int Index::Fetch(const uint32_t doc_id, VectorDataBuffer *vector_data_buffer) {
if (!is_open_) {
LOG_ERROR("Index is not open");
+5
View File
@@ -823,6 +823,7 @@ class BufferStorage : public IndexStorage {
size_t capacity = static_cast<size_t>(meta->padding_size + meta->data_size);
memcpy(segment->data(), IndexVersion::Details(), data_size);
segment->set_dirty();
set_as_dirty();
meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0);
meta->data_size = data_size;
meta->padding_size = capacity - data_size;
@@ -860,6 +861,10 @@ class BufferStorage : public IndexStorage {
return ret;
}
bool is_dirty(void) const override {
return index_dirty_.load(std::memory_order_relaxed);
}
//! Mark the index as dirty. HOT PATH: store(true) unconditionally --
//! a load-then-store guard could let a stale cached `true` skip the
//! store after flush_index() CAS'd dirty=false on another core, losing
+6 -1
View File
@@ -109,8 +109,8 @@ class MMapFileStorage : public IndexStorage {
if (data_tail > meta->data_size) {
meta->data_size = data_tail;
meta->padding_size = capacity_ - data_tail;
owner_->set_as_dirty();
}
owner_->set_as_dirty();
memmove((uint8_t *)segment_->data() + offset, data, len);
segment_->set_dirty();
return len;
@@ -254,6 +254,7 @@ class MMapFileStorage : public IndexStorage {
size_t capacity = static_cast<size_t>(meta->padding_size + meta->data_size);
memcpy(segment->data(), IndexVersion::Details(), data_size);
segment->set_dirty();
set_as_dirty();
meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0);
meta->data_size = data_size;
meta->padding_size = capacity - data_size;
@@ -285,6 +286,10 @@ class MMapFileStorage : public IndexStorage {
return mapping_.huge_page();
}
bool is_dirty(void) const override {
return index_dirty_ || mapping_.is_header_dirty();
}
//! Set the index file as dirty
void set_as_dirty(void) {
index_dirty_ = true;
@@ -181,6 +181,11 @@ class IndexMapping {
return huge_page_;
}
//! Test if any data needs to be flushed to disk
bool is_header_dirty() const {
return header_dirty_;
}
protected:
//! Initialize index file mapping
int init_index_mapping(size_t len);
@@ -386,6 +386,11 @@ class IndexStorage : public IndexModule {
return MemoryBlock::MBT_MMAP;
}
//! Test if the storage has unflushed data
virtual bool is_dirty(void) const {
return false;
}
//! Retrieve file ptr if has
virtual std::shared_ptr<ailego::File> file(void) const {
return nullptr;
+2
View File
@@ -140,6 +140,8 @@ class Index {
return is_trained_;
}
bool IsDirty() const;
uint32_t GetDocCount() const {
if (streamer_ == nullptr) {
return -1;