From 8f276e07b14b6bdc64a4a32a1bcf6257609d2c66 Mon Sep 17 00:00:00 2001 From: ZeFeng Yin Date: Mon, 27 Jul 2026 16:57:52 +0800 Subject: [PATCH] fix(diskann): make Linux AIO fallback safe for partial operations (#615) --- .../algorithm/diskann/diskann_file_reader.cc | 155 ++++++++--- .../diskann/diskann_file_reader_aio_test.cc | 261 ++++++++++++++++++ 2 files changed, 380 insertions(+), 36 deletions(-) create mode 100644 tests/core/algorithm/diskann/diskann_file_reader_aio_test.cc diff --git a/src/core/algorithm/diskann/diskann_file_reader.cc b/src/core/algorithm/diskann/diskann_file_reader.cc index 85be8c3..cde0755 100644 --- a/src/core/algorithm/diskann/diskann_file_reader.cc +++ b/src/core/algorithm/diskann/diskann_file_reader.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "diskann_file_reader.h" +#include #include #include #include @@ -69,10 +70,13 @@ int setup_io_ctx(IOContext &ctx) { int destroy_io_ctx(IOContext &ctx) { #if (defined(__linux) || defined(__linux__)) - if (ailego::IOBackend::Instance().is_pread()) { + if (ailego::IOBackend::Instance().is_pread() || ctx == nullptr) { return 0; } int ret = LibAioLoader::Instance().io_destroy(ctx); + if (ret == 0) { + ctx = nullptr; + } return ret; #else @@ -98,12 +102,40 @@ static int execute_io_pread(int fd, std::vector &read_reqs) { return 0; } -int execute_io(IOContext ctx, int fd, std::vector &read_reqs, - uint64_t n_retries = 0) { #if (defined(__linux) || defined(__linux__)) - if (ailego::IOBackend::Instance().is_pread()) { - return execute_io_pread(fd, read_reqs); +// io_getevents() should only fail permanently for an invalid context or +// invalid arguments. If that happens after submission, io_destroy() is the +// only safe way to quiesce the context before synchronous I/O touches the same +// destination buffers. Recreate the context so later reads can still use AIO. +static bool reset_aio_context(IOContext &ctx) { + auto &loader = LibAioLoader::Instance(); + int ret; + do { + ret = loader.io_destroy(ctx); + } while (ret == -EINTR); + + if (ret != 0) { + LOG_ERROR("io_destroy failed while draining AIO; returned: %d, %s", ret, + ::strerror(-ret)); + return false; } + + ctx = nullptr; + IOContext replacement = nullptr; + ret = loader.io_setup(MAX_EVENTS, &replacement); + if (ret != 0) { + LOG_ERROR( + "io_setup failed while recreating an AIO context; returned: %d, %s. " + "this context will use pread", + ret, ::strerror(-ret)); + return true; + } + ctx = replacement; + return true; +} + +int execute_io_libaio(IOContext &ctx, int fd, + std::vector &read_reqs, uint64_t n_retries) { uint64_t iters = DiskAnnUtil::div_round_up(read_reqs.size(), MAX_EVENTS); for (uint64_t iter = 0; iter < iters; iter++) { @@ -124,60 +156,111 @@ int execute_io(IOContext ctx, int fd, std::vector &read_reqs, } size_t n_tries = 0; - // Phase 1: io_submit with retry. - while (true) { - int ret = - LibAioLoader::Instance().io_submit(ctx, (int64_t)n_ops, cbs.data()); - if (ret == (int)n_ops) { - break; + size_t submitted = 0; + bool submission_ok = true; + + // Phase 1: accumulate partial submissions. A positive return value means + // that exactly that prefix is now in flight and must never be submitted + // again. + while (submitted < n_ops) { + size_t remaining = n_ops - submitted; + int ret = LibAioLoader::Instance().io_submit(ctx, (int64_t)remaining, + cbs.data() + submitted); + if (ret > 0 && static_cast(ret) <= remaining) { + submitted += static_cast(ret); + n_tries = 0; + continue; } if ((ret == -EAGAIN || ret == -EINTR) && n_tries < n_retries) { n_tries++; continue; } LOG_WARN( - "io_submit failed; returned: %d, expected=%lu. falling back to " - "pread", - ret, n_ops); - return execute_io_pread(fd, read_reqs); + "io_submit stopped after %zu/%lu requests; returned: %d. " + "falling back to pread after draining submitted AIO", + submitted, (unsigned long)n_ops, ret); + submission_ok = false; + break; } - // Phase 2: io_getevents with retry (never re-submits). - n_tries = 0; - while (true) { + // Phase 2: accumulate completions for every request that was actually + // submitted. Partial completion is normal and must not trigger fallback: + // the remaining requests can still write into the caller's buffers. + size_t completed = 0; + while (completed < submitted) { + size_t remaining = submitted - completed; int ret = LibAioLoader::Instance().io_getevents( - ctx, (int64_t)n_ops, (int64_t)n_ops, evts.data(), nullptr); - if (ret == (int)n_ops) { - break; - } - if (ret == -EINTR && n_tries < n_retries) { - n_tries++; + ctx, (int64_t)remaining, (int64_t)remaining, evts.data() + completed, + nullptr); + if (ret > 0 && static_cast(ret) <= remaining) { + completed += static_cast(ret); continue; } - LOG_WARN( - "io_getevents failed; returned: %d, expected=%lu, errno=%d, %s, " - "falling back to pread", - ret, n_ops, errno, ::strerror(-ret)); + if (ret == -EINTR) { + // Once requests are in flight, EINTR cannot safely turn into pread + // regardless of the caller's submission retry budget. + continue; + } + + LOG_ERROR( + "io_getevents failed after %zu/%zu completions; returned: %d, %s. " + "resetting the AIO context before falling back to pread", + completed, submitted, ret, + ret < 0 ? ::strerror(-ret) : "invalid completion count"); + if (!reset_aio_context(ctx)) { + // Do not run pread unless io_destroy confirmed that no request can + // still write into these buffers. + return IndexError_Runtime; + } return execute_io_pread(fd, read_reqs); } - // Phase 3: verify each completed read (res must equal requested length). + // Phase 3: verify every harvested event. Completion order is unspecified, + // so use io_event::obj instead of assuming it matches request order. bool all_ok = true; - for (uint64_t i = 0; i < n_ops; i++) { - int64_t expected_len = read_reqs[i + iter * MAX_EVENTS].len; - if ((int64_t)evts[i].res != expected_len) { - LOG_WARN("aio request %zu failed: res=%ld, expected=%ld, offset=%zu", - (size_t)i, (long)evts[i].res, (long)expected_len, - (size_t)read_reqs[i + iter * MAX_EVENTS].offset); + std::vector seen(submitted, false); + for (size_t i = 0; i < completed; i++) { + auto cb_it = std::find(cbs.begin(), cbs.begin() + submitted, evts[i].obj); + if (cb_it == cbs.begin() + submitted) { + LOG_WARN("aio completion %zu referenced an unknown request", i); + all_ok = false; + continue; + } + + size_t request_index = static_cast(cb_it - cbs.begin()); + const AlignedRead &req = read_reqs[request_index + iter * MAX_EVENTS]; + int64_t result = static_cast(evts[i].res); + int64_t result2 = static_cast(evts[i].res2); + if (seen[request_index] || result != static_cast(req.len) || + result2 != 0) { + LOG_WARN( + "aio request %zu failed: res=%ld, res2=%ld, expected=%lu, " + "offset=%lu", + request_index, (long)result, (long)result2, (unsigned long)req.len, + (unsigned long)req.offset); all_ok = false; } + seen[request_index] = true; } - if (!all_ok) { + + if (!submission_ok || !all_ok) { + // All submitted requests have been harvested at this point. It is now + // safe for synchronous reads to reuse their destination buffers. return execute_io_pread(fd, read_reqs); } } return 0; +} +#endif + +int execute_io(IOContext &ctx, int fd, std::vector &read_reqs, + uint64_t n_retries = 0) { +#if (defined(__linux) || defined(__linux__)) + if (ailego::IOBackend::Instance().is_pread() || ctx == nullptr) { + return execute_io_pread(fd, read_reqs); + } + return execute_io_libaio(ctx, fd, read_reqs, n_retries); #else return execute_io_pread(fd, read_reqs); #endif diff --git a/tests/core/algorithm/diskann/diskann_file_reader_aio_test.cc b/tests/core/algorithm/diskann/diskann_file_reader_aio_test.cc new file mode 100644 index 0000000..fdc1a39 --- /dev/null +++ b/tests/core/algorithm/diskann/diskann_file_reader_aio_test.cc @@ -0,0 +1,261 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "diskann_file_reader.h" + +#if defined(__linux) || defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zvec { +namespace core { +int execute_io_libaio(IOContext &ctx, int fd, + std::vector &read_reqs, + uint64_t n_retries = 0); +} // namespace core +} // namespace zvec + +using namespace zvec::core; + +namespace { + +constexpr size_t kBlockSize = 512; +constexpr size_t kBlockCount = 4; + +struct FakeAioState { + std::vector submit_results; + std::vector completion_results; + std::vector submit_sizes; + std::vector completion_sizes; + std::vector submitted; + size_t submit_call = 0; + size_t completion_call = 0; + size_t completed = 0; + size_t short_completion = std::numeric_limits::max(); +}; + +FakeAioState *g_fake_aio = nullptr; + +int fake_io_submit(io_context_t, long nr, struct iocb *ios[]) { + if (g_fake_aio == nullptr || + g_fake_aio->submit_call >= g_fake_aio->submit_results.size()) { + return -EINVAL; + } + + g_fake_aio->submit_sizes.push_back(nr); + int ret = g_fake_aio->submit_results[g_fake_aio->submit_call++]; + if (ret <= 0) { + return ret; + } + if (ret > nr) { + return -EINVAL; + } + + for (int i = 0; i < ret; ++i) { + g_fake_aio->submitted.push_back(ios[i]); + } + return ret; +} + +int fake_io_getevents(io_context_t, long min_nr, long nr, + struct io_event *events, struct timespec *) { + if (g_fake_aio == nullptr || min_nr != nr || + g_fake_aio->completion_call >= g_fake_aio->completion_results.size()) { + return -EINVAL; + } + + g_fake_aio->completion_sizes.push_back(nr); + int ret = g_fake_aio->completion_results[g_fake_aio->completion_call++]; + if (ret <= 0) { + return ret; + } + if (ret > nr || g_fake_aio->completed + static_cast(ret) > + g_fake_aio->submitted.size()) { + return -EINVAL; + } + + for (int i = 0; i < ret; ++i) { + size_t completion_index = g_fake_aio->completed++; + struct iocb *cb = g_fake_aio->submitted[completion_index]; + std::memset(cb->u.c.buf, 0xa5, cb->u.c.nbytes); + events[i].data = cb->data; + events[i].obj = cb; + events[i].res = completion_index == g_fake_aio->short_completion + ? cb->u.c.nbytes - 1 + : cb->u.c.nbytes; + events[i].res2 = 0; + } + return ret; +} + +class FakeAioGuard { + public: + explicit FakeAioGuard(FakeAioState *state) + : loader_(LibAioLoader::Instance()), + original_submit_(loader_.io_submit), + original_getevents_(loader_.io_getevents) { + g_fake_aio = state; + loader_.io_submit = fake_io_submit; + loader_.io_getevents = fake_io_getevents; + } + + ~FakeAioGuard() { + loader_.io_submit = original_submit_; + loader_.io_getevents = original_getevents_; + g_fake_aio = nullptr; + } + + private: + LibAioLoader &loader_; + aio_submit_fn original_submit_; + aio_getevents_fn original_getevents_; +}; + +class TemporaryFile { + public: + TemporaryFile() : fd_(::mkstemp(path_)) {} + + ~TemporaryFile() { + if (fd_ >= 0) { + ::close(fd_); + } + ::unlink(path_); + } + + int fd() const { + return fd_; + } + + private: + char path_[64] = "DiskAnnLinuxAioTest.XXXXXX"; + int fd_; +}; + +void *allocate_aligned(size_t size) { + void *buffer = nullptr; + if (::posix_memalign(&buffer, kBlockSize, size) != 0) { + return nullptr; + } + std::memset(buffer, 0, size); + return buffer; +} + +std::vector make_requests(void *buffer) { + std::vector requests; + requests.reserve(kBlockCount); + for (size_t i = 0; i < kBlockCount; ++i) { + requests.emplace_back(i * kBlockSize, kBlockSize, + static_cast(buffer) + i * kBlockSize); + } + return requests; +} + +std::vector make_source() { + std::vector source(kBlockSize * kBlockCount); + for (size_t block = 0; block < kBlockCount; ++block) { + std::fill(source.begin() + block * kBlockSize, + source.begin() + (block + 1) * kBlockSize, + static_cast(block + 1)); + } + return source; +} + +} // namespace + +TEST(DiskAnnLinuxAioTest, AccumulatesPartialSubmissionsAndCompletions) { + void *output = allocate_aligned(kBlockSize * kBlockCount); + ASSERT_NE(output, nullptr); + std::vector requests = make_requests(output); + + FakeAioState state; + state.submit_results = {2, 2}; + state.completion_results = {1, 2, 1}; + FakeAioGuard guard(&state); + IOContext ctx = reinterpret_cast(static_cast(1)); + + // An invalid fd makes any accidental pread fallback fail the test. + EXPECT_EQ(execute_io_libaio(ctx, -1, requests), 0); + EXPECT_EQ(state.submit_sizes, (std::vector{4, 2})); + EXPECT_EQ(state.completion_sizes, (std::vector{4, 3, 1})); + EXPECT_EQ(state.completed, kBlockCount); + const auto *bytes = static_cast(output); + EXPECT_TRUE(std::all_of(bytes, bytes + kBlockSize * kBlockCount, + [](uint8_t value) { return value == 0xa5; })); + + std::free(output); +} + +TEST(DiskAnnLinuxAioTest, DrainsPartialSubmissionBeforePreadFallback) { + TemporaryFile file; + ASSERT_GE(file.fd(), 0); + std::vector source = make_source(); + ASSERT_EQ(::pwrite(file.fd(), source.data(), source.size(), 0), + static_cast(source.size())); + + void *output = allocate_aligned(source.size()); + ASSERT_NE(output, nullptr); + std::vector requests = make_requests(output); + + FakeAioState state; + state.submit_results = {2, -EAGAIN}; + state.completion_results = {1, 1}; + FakeAioGuard guard(&state); + IOContext ctx = reinterpret_cast(static_cast(1)); + + EXPECT_EQ(execute_io_libaio(ctx, file.fd(), requests), 0); + EXPECT_EQ(state.submit_sizes, (std::vector{4, 2})); + EXPECT_EQ(state.completion_sizes, (std::vector{2, 1})); + EXPECT_EQ(state.completed, 2u); + EXPECT_EQ(std::memcmp(output, source.data(), source.size()), 0); + + std::free(output); +} + +TEST(DiskAnnLinuxAioTest, DrainsAllCompletionsBeforePreadFallback) { + TemporaryFile file; + ASSERT_GE(file.fd(), 0); + std::vector source = make_source(); + ASSERT_EQ(::pwrite(file.fd(), source.data(), source.size(), 0), + static_cast(source.size())); + + void *output = allocate_aligned(source.size()); + ASSERT_NE(output, nullptr); + std::vector requests = make_requests(output); + + FakeAioState state; + state.submit_results = {4}; + state.completion_results = {1, 1, 2}; + state.short_completion = 0; + FakeAioGuard guard(&state); + IOContext ctx = reinterpret_cast(static_cast(1)); + + EXPECT_EQ(execute_io_libaio(ctx, file.fd(), requests), 0); + EXPECT_EQ(state.completion_sizes, (std::vector{4, 3, 2})); + EXPECT_EQ(state.completed, kBlockCount); + EXPECT_EQ(std::memcmp(output, source.data(), source.size()), 0); + + std::free(output); +} + +#endif // __linux__