From b1af0e4ebf72f178138e4d7cb0d4cbcd0d22bc4d Mon Sep 17 00:00:00 2001 From: Jalin Wang Date: Wed, 17 Jun 2026 11:20:40 +0800 Subject: [PATCH] feat(index): expose is_dirty status through core Index interface (#488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds is_dirty() at each layer: IndexMapping → IndexStorage (virtual) → MMapFileStorage / BufferStorage → Index::IsDirty(). --- src/core/interface/index.cc | 7 ++ src/core/utility/buffer_storage.cc | 5 + src/core/utility/mmap_file_storage.cc | 7 +- .../zvec/core/framework/index_mapping.h | 5 + .../zvec/core/framework/index_storage.h | 5 + src/include/zvec/core/interface/index.h | 2 + tests/core/interface/index_interface_test.cc | 118 ++++++++++++++++++ 7 files changed, 148 insertions(+), 1 deletion(-) diff --git a/src/core/interface/index.cc b/src/core/interface/index.cc index b969c92..be6f5ff 100644 --- a/src/core/interface/index.cc +++ b/src/core/interface/index.cc @@ -381,6 +381,13 @@ int Index::Flush() { return 0; } +bool Index::IsDirty() const { + if (!storage_) { + return false; + } + return storage_->is_dirty(); +} + int Index::Fetch(const uint32_t doc_id, VectorDataBuffer *vector_data_buffer) { if (!is_open_) { LOG_ERROR("Index is not open"); diff --git a/src/core/utility/buffer_storage.cc b/src/core/utility/buffer_storage.cc index c7cba5e..3db6c58 100644 --- a/src/core/utility/buffer_storage.cc +++ b/src/core/utility/buffer_storage.cc @@ -823,6 +823,7 @@ class BufferStorage : public IndexStorage { size_t capacity = static_cast(meta->padding_size + meta->data_size); memcpy(segment->data(), IndexVersion::Details(), data_size); segment->set_dirty(); + set_as_dirty(); meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0); meta->data_size = data_size; meta->padding_size = capacity - data_size; @@ -860,6 +861,10 @@ class BufferStorage : public IndexStorage { return ret; } + bool is_dirty(void) const override { + return index_dirty_.load(std::memory_order_relaxed); + } + //! Mark the index as dirty. HOT PATH: store(true) unconditionally -- //! a load-then-store guard could let a stale cached `true` skip the //! store after flush_index() CAS'd dirty=false on another core, losing diff --git a/src/core/utility/mmap_file_storage.cc b/src/core/utility/mmap_file_storage.cc index 506ad21..c172e49 100644 --- a/src/core/utility/mmap_file_storage.cc +++ b/src/core/utility/mmap_file_storage.cc @@ -109,8 +109,8 @@ class MMapFileStorage : public IndexStorage { if (data_tail > meta->data_size) { meta->data_size = data_tail; meta->padding_size = capacity_ - data_tail; - owner_->set_as_dirty(); } + owner_->set_as_dirty(); memmove((uint8_t *)segment_->data() + offset, data, len); segment_->set_dirty(); return len; @@ -254,6 +254,7 @@ class MMapFileStorage : public IndexStorage { size_t capacity = static_cast(meta->padding_size + meta->data_size); memcpy(segment->data(), IndexVersion::Details(), data_size); segment->set_dirty(); + set_as_dirty(); meta->data_crc = ailego::Crc32c::Hash(segment->data(), data_size, 0); meta->data_size = data_size; meta->padding_size = capacity - data_size; @@ -285,6 +286,10 @@ class MMapFileStorage : public IndexStorage { return mapping_.huge_page(); } + bool is_dirty(void) const override { + return index_dirty_ || mapping_.is_header_dirty(); + } + //! Set the index file as dirty void set_as_dirty(void) { index_dirty_ = true; diff --git a/src/include/zvec/core/framework/index_mapping.h b/src/include/zvec/core/framework/index_mapping.h index d2b9809..0013ff9 100644 --- a/src/include/zvec/core/framework/index_mapping.h +++ b/src/include/zvec/core/framework/index_mapping.h @@ -181,6 +181,11 @@ class IndexMapping { return huge_page_; } + //! Test if any data needs to be flushed to disk + bool is_header_dirty() const { + return header_dirty_; + } + protected: //! Initialize index file mapping int init_index_mapping(size_t len); diff --git a/src/include/zvec/core/framework/index_storage.h b/src/include/zvec/core/framework/index_storage.h index 9a7ad36..049c799 100644 --- a/src/include/zvec/core/framework/index_storage.h +++ b/src/include/zvec/core/framework/index_storage.h @@ -386,6 +386,11 @@ class IndexStorage : public IndexModule { return MemoryBlock::MBT_MMAP; } + //! Test if the storage has unflushed data + virtual bool is_dirty(void) const { + return false; + } + //! Retrieve file ptr if has virtual std::shared_ptr file(void) const { return nullptr; diff --git a/src/include/zvec/core/interface/index.h b/src/include/zvec/core/interface/index.h index f3f0fb2..921221c 100644 --- a/src/include/zvec/core/interface/index.h +++ b/src/include/zvec/core/interface/index.h @@ -140,6 +140,8 @@ class Index { return is_trained_; } + bool IsDirty() const; + uint32_t GetDocCount() const { if (streamer_ == nullptr) { return -1; diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index 44d9061..f1e35af 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -1856,6 +1856,124 @@ TEST(IndexInterface, ContiguousMemoryEndToEnd) { .build()); } +TEST(IndexInterface, IsDirty) { + constexpr uint32_t kDimension = 16; + const std::string index_name{"test_is_dirty.index"}; + + auto test = [&](const BaseIndexParam::Pointer ¶m) { + zvec::test_util::RemoveTestFiles(index_name); + + // Before open: not dirty (no storage) + { + auto index = IndexFactory::CreateAndInitIndex(*param); + ASSERT_NE(nullptr, index); + ASSERT_FALSE(index->IsDirty()); + } + + // Create the index file: dirty from initial metadata writes + { + auto index = IndexFactory::CreateAndInitIndex(*param); + index->Open(index_name, {StorageOptions::StorageType::kMMAP, true}); + ASSERT_TRUE(index->IsDirty()); + ASSERT_EQ(0, index->Flush()); + ASSERT_FALSE(index->IsDirty()); + index->Close(); + } + + // Reopen existing file: should be clean + auto index = IndexFactory::CreateAndInitIndex(*param); + index->Open(index_name, {StorageOptions::StorageType::kMMAP, false}); + ASSERT_FALSE(index->IsDirty()); + + // Add a vector: should become dirty + std::vector vec(kDimension, 1.0f); + VectorData vd; + vd.vector = DenseVector{vec.data()}; + ASSERT_EQ(0, index->Add(vd, 1)); + ASSERT_TRUE(index->IsDirty()); + + // Flush: should become clean + ASSERT_EQ(0, index->Flush()); + ASSERT_FALSE(index->IsDirty()); + + // Add another vector: dirty again + ASSERT_EQ(0, index->Add(vd, 2)); + ASSERT_TRUE(index->IsDirty()); + + // Close flushes implicitly, verify no crash + index->Close(); + zvec::test_util::RemoveTestFiles(index_name); + }; + + test(FlatIndexParamBuilder() + .WithMetricType(MetricType::kInnerProduct) + .WithDataType(DataType::DT_FP32) + .WithDimension(kDimension) + .WithIsSparse(false) + .Build()); + + test(HNSWIndexParamBuilder() + .WithMetricType(MetricType::kInnerProduct) + .WithDataType(DataType::DT_FP32) + .WithDimension(kDimension) + .WithIsSparse(false) + .WithEFConstruction(100) + .Build()); +} + +TEST(IndexInterface, IsDirtyBufferPool) { + constexpr uint32_t kDimension = 16; + const std::string index_name{"test_is_dirty_bp.index"}; + + zvec::test_util::RemoveTestFiles(index_name); + + // First create and populate the index with MMAP storage + { + auto param = FlatIndexParamBuilder() + .WithMetricType(MetricType::kInnerProduct) + .WithDataType(DataType::DT_FP32) + .WithDimension(kDimension) + .WithIsSparse(false) + .Build(); + auto index = IndexFactory::CreateAndInitIndex(*param); + ASSERT_NE(nullptr, index); + index->Open(index_name, {StorageOptions::StorageType::kMMAP, true}); + std::vector vec(kDimension, 1.0f); + VectorData vd; + vd.vector = DenseVector{vec.data()}; + ASSERT_EQ(0, index->Add(vd, 1)); + index->Close(); + } + + // Reopen with BufferPool storage in writable mode + { + auto param = FlatIndexParamBuilder() + .WithMetricType(MetricType::kInnerProduct) + .WithDataType(DataType::DT_FP32) + .WithDimension(kDimension) + .WithIsSparse(false) + .Build(); + auto index = IndexFactory::CreateAndInitIndex(*param); + ASSERT_NE(nullptr, index); + index->Open(index_name, {StorageOptions::StorageType::kBufferPool, true}); + + ASSERT_FALSE(index->IsDirty()); + + std::vector vec(kDimension, 2.0f); + VectorData vd; + vd.vector = DenseVector{vec.data()}; + ASSERT_EQ(0, index->Add(vd, 2)); + ASSERT_TRUE(index->IsDirty()); + + ASSERT_EQ(0, index->Flush()); + ASSERT_FALSE(index->IsDirty()); + + index->Close(); + } + + zvec::test_util::RemoveTestFiles(index_name); +} + #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop #endif \ No newline at end of file