feat(index): expose is_dirty status through core Index interface (#488)
Adds is_dirty() at each layer: IndexMapping → IndexStorage (virtual) → MMapFileStorage / BufferStorage → Index::IsDirty().
This commit is contained in:
@@ -381,6 +381,13 @@ int Index::Flush() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Index::IsDirty() const {
|
||||
if (!storage_) {
|
||||
return false;
|
||||
}
|
||||
return storage_->is_dirty();
|
||||
}
|
||||
|
||||
int Index::Fetch(const uint32_t doc_id, VectorDataBuffer *vector_data_buffer) {
|
||||
if (!is_open_) {
|
||||
LOG_ERROR("Index is not open");
|
||||
|
||||
@@ -823,6 +823,7 @@ class BufferStorage : public IndexStorage {
|
||||
size_t capacity = static_cast<size_t>(meta->padding_size + meta->data_size);
|
||||
memcpy(segment->data(), IndexVersion::Details(), data_size);
|
||||
segment->set_dirty();
|
||||
set_as_dirty();
|
||||
meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0);
|
||||
meta->data_size = data_size;
|
||||
meta->padding_size = capacity - data_size;
|
||||
@@ -860,6 +861,10 @@ class BufferStorage : public IndexStorage {
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool is_dirty(void) const override {
|
||||
return index_dirty_.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
//! Mark the index as dirty. HOT PATH: store(true) unconditionally --
|
||||
//! a load-then-store guard could let a stale cached `true` skip the
|
||||
//! store after flush_index() CAS'd dirty=false on another core, losing
|
||||
|
||||
@@ -109,8 +109,8 @@ class MMapFileStorage : public IndexStorage {
|
||||
if (data_tail > meta->data_size) {
|
||||
meta->data_size = data_tail;
|
||||
meta->padding_size = capacity_ - data_tail;
|
||||
owner_->set_as_dirty();
|
||||
}
|
||||
owner_->set_as_dirty();
|
||||
memmove((uint8_t *)segment_->data() + offset, data, len);
|
||||
segment_->set_dirty();
|
||||
return len;
|
||||
@@ -254,6 +254,7 @@ class MMapFileStorage : public IndexStorage {
|
||||
size_t capacity = static_cast<size_t>(meta->padding_size + meta->data_size);
|
||||
memcpy(segment->data(), IndexVersion::Details(), data_size);
|
||||
segment->set_dirty();
|
||||
set_as_dirty();
|
||||
meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0);
|
||||
meta->data_size = data_size;
|
||||
meta->padding_size = capacity - data_size;
|
||||
@@ -285,6 +286,10 @@ class MMapFileStorage : public IndexStorage {
|
||||
return mapping_.huge_page();
|
||||
}
|
||||
|
||||
bool is_dirty(void) const override {
|
||||
return index_dirty_ || mapping_.is_header_dirty();
|
||||
}
|
||||
|
||||
//! Set the index file as dirty
|
||||
void set_as_dirty(void) {
|
||||
index_dirty_ = true;
|
||||
|
||||
@@ -181,6 +181,11 @@ class IndexMapping {
|
||||
return huge_page_;
|
||||
}
|
||||
|
||||
//! Test if any data needs to be flushed to disk
|
||||
bool is_header_dirty() const {
|
||||
return header_dirty_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Initialize index file mapping
|
||||
int init_index_mapping(size_t len);
|
||||
|
||||
@@ -386,6 +386,11 @@ class IndexStorage : public IndexModule {
|
||||
return MemoryBlock::MBT_MMAP;
|
||||
}
|
||||
|
||||
//! Test if the storage has unflushed data
|
||||
virtual bool is_dirty(void) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve file ptr if has
|
||||
virtual std::shared_ptr<ailego::File> file(void) const {
|
||||
return nullptr;
|
||||
|
||||
@@ -140,6 +140,8 @@ class Index {
|
||||
return is_trained_;
|
||||
}
|
||||
|
||||
bool IsDirty() const;
|
||||
|
||||
uint32_t GetDocCount() const {
|
||||
if (streamer_ == nullptr) {
|
||||
return -1;
|
||||
|
||||
Reference in New Issue
Block a user