xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: support UTF-8 file paths via std::filesystem (#359) 

Rewrite file/path handling to use std::filesystem and UTF-8-safe helpers.
Switch Windows file open/create paths to wide-char APIs, replace manual
separator concatenation with PathJoin, and enable RocksDB UTF-8 filenames.

Also add UTF-8 path coverage for file IO, version manager recovery, and
collection open/flush/reopen flows.
This commit is contained in:
Jalin Wang 2026-05-08 17:44:22 +08:00 committed by GitHub
parent 68a497efdb
commit 1d4ae0b1b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 937 additions and 432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/ailego/io/file.cc
View File

@ -24,6 +24,8 @@
#include <unistd.h>
#else
#include <Windows.h>
#include <cstring>
#include <string>
#endif
namespace zvec {
@ -405,15 +407,26 @@ bool File::MemoryUnlock(void *addr, size_t len) {
#else
namespace {
bool Utf8PathOk(const char *path, const std::wstring &wide) {
return path && path[0] != '\0' && !wide.empty();
}
} // namespace
//! Create a local file
bool File::create(const char *path, size_t len, bool direct) {
ailego_false_if_false(native_handle_ == File::InvalidHandle && path);
const std::wstring wpath = FileHelper::Utf8ToWide(path);
ailego_false_if_false(Utf8PathOk(path, wpath));
// Try opening or creating the file
HANDLE file_handle =
CreateFileA(path, GENERIC_WRITE | GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
::CreateFileW(wpath.c_str(), GENERIC_WRITE | GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
// Truncate the file to the specified size
@ -432,8 +445,8 @@ bool File::create(const char *path, size_t len, bool direct) {
} else {
// Close and reopen file
CloseHandle(file_handle);
file_handle = CreateFileA(
path, GENERIC_WRITE | GENERIC_READ,
file_handle = ::CreateFileW(
wpath.c_str(), GENERIC_WRITE | GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr);
ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
@ -448,15 +461,18 @@ bool File::create(const char *path, size_t len, bool direct) {
bool File::open(const char *path, bool rdonly, bool direct) {
ailego_false_if_false(native_handle_ == File::InvalidHandle && path);
const std::wstring wpath = FileHelper::Utf8ToWide(path);
ailego_false_if_false(Utf8PathOk(path, wpath));
// Try opening the file
DWORD flags = FILE_ATTRIBUTE_NORMAL;
if (direct) {
flags |= FILE_FLAG_NO_BUFFERING;
}
HANDLE file_handle =
CreateFileA(path, (rdonly ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE),
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, OPEN_EXISTING, flags, nullptr);
HANDLE file_handle = ::CreateFileW(
wpath.c_str(), (rdonly ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE),
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr,
OPEN_EXISTING, flags, nullptr);
ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
read_only_ = rdonly;

517
src/ailego/utility/file_helper.cc
View File

@ -12,34 +12,238 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <string>
#include <system_error>
#include <zvec/ailego/utility/file_helper.h>
#if defined(_WIN32) || defined(_WIN64)
#include <Windows.h>
#ifdef RemoveDirectory
#undef RemoveDirectory
#endif
#ifdef DeleteFile
#undef DeleteFile
#endif
#ifdef GetFileSize
#undef GetFileSize
#endif
#else
#if defined(__APPLE__) || defined(__MACH__)
#include <mach-o/dyld.h>
#endif
#include <sys/stat.h>
#include <dirent.h>
#include <errno.h>
#if defined(__FreeBSD__)
#include <sys/sysctl.h>
#endif
#include <fcntl.h>
#include <string.h>
#include <unistd.h>
#endif
#include <filesystem>
namespace fs = std::filesystem;
// TODO: refactor all file operations by std::filesystem;
namespace zvec {
namespace ailego {
namespace {
thread_local std::error_code g_last_fs_error;
void ClearFsError() {
g_last_fs_error.clear();
}
void SetFsError(std::error_code ec) {
g_last_fs_error = ec;
}
} // namespace
// ---------- public UTF-8 / wide helpers ----------
fs::path FileHelper::PathFromUtf8(const char *s) {
#if defined(_WIN32) || defined(_WIN64)
if (!s || !*s) {
return fs::path();
}
return fs::u8path(s);
#else
return fs::path(s ? s : "");
#endif
}
fs::path FileHelper::PathFromUtf8(const std::string &s) {
return PathFromUtf8(s.c_str());
}
std::string FileHelper::PathToUtf8(const fs::path &p) {
return p.u8string();
}
#if defined(_WIN32) || defined(_WIN64)
std::wstring FileHelper::Utf8ToWide(const std::string &src) {
if (src.empty()) {
return {};
}
int src_len = static_cast<int>(src.size());
int dst_len =
MultiByteToWideChar(CP_UTF8, 0, src.data(), src_len, nullptr, 0);
if (dst_len <= 0) {
return {};
}
std::wstring dst(static_cast<size_t>(dst_len), L'\0');
if (MultiByteToWideChar(CP_UTF8, 0, src.data(), src_len, dst.data(),
dst_len) != dst_len) {
return {};
}
return dst;
}
std::string FileHelper::WideToUtf8(const std::wstring &src) {
if (src.empty()) {
return {};
}
int src_len = static_cast<int>(src.size());
int dst_len = WideCharToMultiByte(CP_UTF8, 0, src.data(), src_len, nullptr, 0,
nullptr, nullptr);
if (dst_len <= 0) {
return {};
}
std::string dst(static_cast<size_t>(dst_len), '\0');
if (WideCharToMultiByte(CP_UTF8, 0, src.data(), src_len, dst.data(), dst_len,
nullptr, nullptr) != dst_len) {
return {};
}
return dst;
}
#endif
// ---------- internal helpers ----------
namespace {
static bool GetFileSizeImpl(const fs::path &p, size_t *psz) {
ClearFsError();
std::error_code ec;
auto sz = fs::file_size(p, ec);
if (ec) {
SetFsError(ec);
return false;
}
*psz = static_cast<size_t>(sz);
return true;
}
static bool DeleteFileImpl(const fs::path &p) {
ClearFsError();
std::error_code ec;
fs::file_status st = fs::symlink_status(p, ec);
if (ec) {
SetFsError(ec);
return false;
}
if (fs::is_directory(st) && !fs::is_symlink(st)) {
ec = std::make_error_code(std::errc::is_a_directory);
SetFsError(ec);
return false;
}
if (!fs::remove(p, ec)) {
SetFsError(ec ? ec
: std::make_error_code(std::errc::no_such_file_or_directory));
return false;
}
return true;
}
static bool RenameFileImpl(const fs::path &from, const fs::path &to) {
ClearFsError();
std::error_code ec;
fs::rename(from, to, ec);
if (ec) {
SetFsError(ec);
return false;
}
return true;
}
static bool MakePathImpl(const fs::path &p) {
ClearFsError();
std::error_code ec;
fs::create_directories(p, ec);
if (ec) {
SetFsError(ec);
return false;
}
return true;
}
static bool RemoveDirectoryImpl(const fs::path &p) {
ClearFsError();
std::error_code ec;
if (!fs::is_directory(p, ec)) {
if (ec) {
SetFsError(ec);
}
return false;
}
std::uintmax_t n = fs::remove_all(p, ec);
if (ec) {
SetFsError(ec);
return false;
}
(void)n;
return true;
}
static bool IsExistImpl(const fs::path &p) {
std::error_code ec;
return fs::exists(p, ec);
}
static bool IsRegularImpl(const fs::path &p) {
std::error_code ec;
return fs::is_regular_file(p, ec);
}
static bool IsDirectoryImpl(const fs::path &p) {
std::error_code ec;
return fs::is_directory(p, ec);
}
static bool IsSymbolicLinkImpl(const fs::path &p) {
std::error_code ec;
return fs::is_symlink(p, ec);
}
static bool IsSameImpl(const fs::path &a, const fs::path &b) {
std::error_code ec;
return fs::equivalent(a, b, ec);
}
} // namespace
bool FileHelper::GetSelfPath(std::string *path) {
#if defined(_WIN32) || defined(_WIN64)
char buf[MAX_PATH];
DWORD len = GetModuleFileNameA(NULL, buf, MAX_PATH);
std::wstring wbuf(4096, L'\0');
DWORD n =
GetModuleFileNameW(nullptr, wbuf.data(), static_cast<DWORD>(wbuf.size()));
while (n >= wbuf.size() - 1) {
if (wbuf.size() > 65536) {
return false;
}
wbuf.resize(wbuf.size() * 2);
n = GetModuleFileNameW(nullptr, wbuf.data(),
static_cast<DWORD>(wbuf.size()));
}
if (n == 0) {
return false;
}
wbuf.resize(n);
*path = WideToUtf8(wbuf);
return !path->empty();
#elif defined(__APPLE__) || defined(__MACH__)
char buf[PATH_MAX];
size_t len = 0;
@ -62,18 +266,31 @@ bool FileHelper::GetSelfPath(std::string *path) {
ssize_t len = readlink("/proc/self/exe", buf, PATH_MAX);
#endif
#if !defined(_WIN32) && !defined(_WIN64)
if (len <= 0) {
return false;
}
path->assign(buf, len);
return true;
#endif
}
bool FileHelper::GetFilePath(NativeHandle handle, std::string *path) {
#if defined(_WIN32) || defined(_WIN64)
char buf[MAX_PATH];
DWORD len =
GetFinalPathNameByHandleA(handle, buf, MAX_PATH, FILE_NAME_OPENED);
DWORD need = GetFinalPathNameByHandleW(static_cast<HANDLE>(handle), nullptr,
0, FILE_NAME_OPENED);
if (need == 0) {
return false;
}
std::wstring wbuf(static_cast<size_t>(need) + 1, L'\0');
DWORD got = GetFinalPathNameByHandleW(
static_cast<HANDLE>(handle), wbuf.data(), need + 1, FILE_NAME_OPENED);
if (got == 0 || got > need) {
return false;
}
wbuf.resize(got);
*path = WideToUtf8(wbuf);
return !path->empty();
#elif defined(__linux) || defined(__linux__)
char buf[PATH_MAX];
char src[32];
@ -87,276 +304,75 @@ bool FileHelper::GetFilePath(NativeHandle handle, std::string *path) {
}
#endif
#if !defined(_WIN32) && !defined(_WIN64)
if (len <= 0) {
return false;
}
path->assign(buf, len);
return true;
}
#if !defined(_WIN32) && !defined(_WIN64)
static inline char *JoinFilePath(const char *prefix, const char *suffix) {
size_t prefix_len = strlen(prefix);
size_t suffix_len = strlen(suffix);
char *path = (char *)malloc(prefix_len + suffix_len + 2);
if (path) {
memcpy(path, prefix, prefix_len);
memcpy(path + prefix_len + 1, suffix, suffix_len);
path[prefix_len] = '/';
path[prefix_len + suffix_len + 1] = '\0';
}
return path;
#endif
}
bool FileHelper::GetWorkingDirectory(std::string *path) {
char buf[PATH_MAX];
if (!getcwd(buf, PATH_MAX)) {
ClearFsError();
std::error_code ec;
fs::path cwd = fs::current_path(ec);
if (ec) {
SetFsError(ec);
return false;
}
path->assign(buf);
*path = PathToUtf8(cwd);
return !path->empty();
}
bool FileHelper::GetFileSize(const char *path, size_t *psz) {
struct stat buf;
if (stat(path, &buf) != 0) {
return false;
}
*psz = buf.st_size;
return true;
return GetFileSizeImpl(PathFromUtf8(path), psz);
}
bool FileHelper::DeleteFile(const char *path) {
// Delete a file by the path
return (unlink(path) == 0);
return DeleteFileImpl(PathFromUtf8(path));
}
bool FileHelper::RenameFile(const char *oldpath, const char *newpath) {
return (rename(oldpath, newpath) == 0);
return RenameFileImpl(PathFromUtf8(oldpath), PathFromUtf8(newpath));
}
bool FileHelper::MakePath(const char *path) {
char pathbuf[PATH_MAX];
char *sp, *pp;
strncpy(pathbuf, path, sizeof(pathbuf) - 1);
pathbuf[PATH_MAX - 1] = '\0';
pp = pathbuf;
while ((sp = strchr(pp, '/')) != nullptr) {
// Neither root nor double slash in path
if (sp != pp) {
*sp = '\0';
if (mkdir(pathbuf, 0755) == -1 && errno != EEXIST) {
return false;
}
*sp = '/';
}
pp = sp + 1;
}
return !(*pp != '\0' && mkdir(pathbuf, 0755) == -1 && errno != EEXIST);
}
bool FileHelper::RemoveDirectory(const char *path) {
DIR *dir = opendir(path);
if (!dir) {
return false;
}
struct dirent *dent;
while ((dent = readdir(dir)) != nullptr) {
if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
continue;
}
char *fullpath = JoinFilePath(path, dent->d_name);
if (!fullpath) {
continue;
}
if (FileHelper::IsDirectory(fullpath)) {
FileHelper::RemoveDirectory(fullpath);
} else {
FileHelper::DeleteFile(fullpath);
}
free(fullpath);
}
closedir(dir);
return (rmdir(path) == 0);
}
bool FileHelper::IsExist(const char *path) {
return (access(path, F_OK) == 0);
}
bool FileHelper::IsRegular(const char *path) {
struct stat buf;
if (stat(path, &buf) != 0) {
return false;
}
return ((buf.st_mode & S_IFREG) != 0);
}
bool FileHelper::IsDirectory(const char *path) {
struct stat buf;
if (stat(path, &buf) != 0) {
return false;
}
return ((buf.st_mode & S_IFDIR) != 0);
}
bool FileHelper::IsSymbolicLink(const char *path) {
struct stat buf;
if (stat(path, &buf) != 0) {
return false;
}
return ((buf.st_mode & S_IFLNK) != 0);
}
bool FileHelper::IsSame(const char *path1, const char *path2) {
char real_path1[PATH_MAX];
char real_path2[PATH_MAX];
if (!realpath(path1, real_path1)) {
return false;
}
if (!realpath(path2, real_path2)) {
return false;
}
return (!strcmp(real_path1, real_path2));
}
std::string FileHelper::GetLastErrorString() {
return strerror(errno);
}
#else
#undef RemoveDirectory
#undef DeleteFile
#undef GetFileSize
static inline char *JoinFilePath(const char *prefix, const char *suffix) {
size_t prefix_len = strlen(prefix);
size_t suffix_len = strlen(suffix);
char *path = (char *)malloc(prefix_len + suffix_len + 2);
if (path) {
memcpy(path, prefix, prefix_len);
memcpy(path + prefix_len + 1, suffix, suffix_len);
path[prefix_len] = '\\';
path[prefix_len + suffix_len + 1] = '\0';
}
return path;
}
bool FileHelper::GetWorkingDirectory(std::string *path) {
char buf[MAX_PATH];
DWORD len = GetCurrentDirectoryA(MAX_PATH, buf);
if (len <= 0) {
return false;
}
path->assign(buf, len);
return true;
}
bool FileHelper::GetFileSize(const char *path, size_t *psz) {
HANDLE handle =
CreateFileA(path, GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
if (handle == INVALID_HANDLE_VALUE) {
return false;
}
LARGE_INTEGER file_size;
if (!GetFileSizeEx(handle, &file_size)) {
CloseHandle(handle);
return false;
}
CloseHandle(handle);
*psz = (size_t)file_size.QuadPart;
return true;
}
bool FileHelper::DeleteFile(const char *path) {
// Delete a file by the path
return (DeleteFileA(path));
}
bool FileHelper::RenameFile(const char *oldpath, const char *newpath) {
return (MoveFileA(oldpath, newpath));
}
bool FileHelper::MakePath(const char *path) {
char pathbuf[MAX_PATH];
char *sp, *pp;
strncpy(pathbuf, path, sizeof(pathbuf) - 1);
pathbuf[MAX_PATH - 1] = '\0';
pp = pathbuf;
while ((sp = strpbrk(pp, "/\\")) != nullptr) {
// Neither root nor double slash in path
if (sp != pp) {
*sp = '\0';
// Skip Windows drive roots like "C:" — CreateDirectoryA on a bare drive
// letter returns ERROR_ACCESS_DENIED (not ERROR_ALREADY_EXISTS), which
// would cause MakePath to fail even when all parent dirs already exist.
bool is_drive_root = (sp - pathbuf == 2 && pathbuf[1] == ':');
if (!is_drive_root && !CreateDirectoryA(pathbuf, nullptr) &&
GetLastError() != ERROR_ALREADY_EXISTS) {
return false;
}
*sp = '\\';
}
pp = sp + 1;
}
return !(*pp != '\0' && !CreateDirectoryA(pathbuf, nullptr) &&
GetLastError() != ERROR_ALREADY_EXISTS);
return MakePathImpl(PathFromUtf8(path));
}
bool FileHelper::RemoveDirectory(const char *path) {
if (path == nullptr || *path == '\0') {
return false;
}
if (!FileHelper::IsDirectory(path)) {
return false;
}
std::error_code ec;
fs::remove_all(path, ec);
if (ec) {
return false;
}
return true;
return RemoveDirectoryImpl(PathFromUtf8(path));
}
bool FileHelper::IsExist(const char *path) {
DWORD attr = GetFileAttributesA(path);
return (attr != INVALID_FILE_ATTRIBUTES);
return IsExistImpl(PathFromUtf8(path));
}
bool FileHelper::IsRegular(const char *path) {
DWORD attr = GetFileAttributesA(path);
return (attr != INVALID_FILE_ATTRIBUTES &&
!(attr & FILE_ATTRIBUTE_DIRECTORY));
return IsRegularImpl(PathFromUtf8(path));
}
bool FileHelper::IsDirectory(const char *path) {
DWORD attr = GetFileAttributesA(path);
return (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY));
return IsDirectoryImpl(PathFromUtf8(path));
}
bool FileHelper::IsSymbolicLink(const char *path) {
DWORD attr = GetFileAttributesA(path);
return (attr != INVALID_FILE_ATTRIBUTES &&
(attr & FILE_ATTRIBUTE_REPARSE_POINT));
return IsSymbolicLinkImpl(PathFromUtf8(path));
}
bool FileHelper::IsSame(const char *path1, const char *path2) {
return IsSameImpl(PathFromUtf8(path1), PathFromUtf8(path2));
}
std::string FileHelper::GetLastErrorString() {
if (g_last_fs_error) {
return g_last_fs_error.message();
}
#if defined(_WIN32) || defined(_WIN64)
DWORD err = GetLastError();
if (err == 0) {
return "No error";
@ -366,34 +382,17 @@ std::string FileHelper::GetLastErrorString() {
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, err,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, sizeof(buf), nullptr);
if (len > 0) {
// Strip trailing newline that FormatMessage often appends
while (len > 0 && (buf[len - 1] == '\r' || buf[len - 1] == '\n')) {
buf[--len] = '\0';
}
return std::string(buf, len);
}
return "Unknown error " + std::to_string(err);
#else
return strerror(errno);
#endif
}
bool FileHelper::IsSame(const char *path1, const char *path2) {
char real_path1[MAX_PATH];
char real_path2[MAX_PATH];
char **part_path1 = nullptr;
char **part_path2 = nullptr;
DWORD path1_size =
GetFullPathNameA(path1, sizeof(real_path1), real_path1, part_path1);
DWORD path2_size =
GetFullPathNameA(path2, sizeof(real_path2), real_path2, part_path2);
if ((part_path1 && *part_path1 != 0) || (part_path2 && *part_path2 != 0) ||
(path1_size != path2_size)) {
return false;
}
return (!strcmp(real_path1, real_path2));
}
#endif // !_WIN32 && !_WIN64
bool FileHelper::RemovePath(const char *path) {
if (FileHelper::IsDirectory(path)) {
return FileHelper::RemoveDirectory(path);
@ -402,4 +401,4 @@ bool FileHelper::RemovePath(const char *path) {
}
} // namespace ailego
} // namespace zvec
} // namespace zvec

4
src/db/collection.cc
View File

@ -14,6 +14,7 @@
#include <atomic>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <mutex>
#include <shared_mutex>
@ -25,7 +26,6 @@
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/pattern/expected.hpp>
#include <zvec/ailego/utility/file_helper.h>
#include <zvec/ailego/utility/string_helper.h>
#include <zvec/db/collection.h>
#include <zvec/db/doc.h>
#include <zvec/db/options.h>
@ -1875,7 +1875,7 @@ Status CollectionImpl::init_writing_segment() {
}
Status CollectionImpl::acquire_file_lock(bool create) {
std::string lock_file_path = ailego::StringHelper::Concat(path_, "/", "LOCK");
std::string lock_file_path = ailego::FileHelper::PathJoin(path_, "LOCK");
if (create) {
if (!lock_file_.create(lock_file_path.c_str(), 0)) {

120
src/db/common/file_helper.cc
View File

@ -13,20 +13,12 @@
// limitations under the License.
#include "file_helper.h"
#include <errno.h>
#include <string.h>
#include <algorithm>
#include <cstdio>
#ifdef _MSC_VER
#include <cstring>
#include <filesystem>
#include <fstream>
#else
#include <sys/stat.h>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#endif
#include <ailego/pattern/defer.h>
#include <vector>
namespace zvec {
@ -37,132 +29,58 @@ const std::string FileHelper::RECOVER_SUFFIX = ".recovering";
bool FileHelper::CopyFile(const std::string &src_file_path,
const std::string &dst_file_path) {
#ifdef _MSC_VER
std::string dst_file_path_tmp = dst_file_path + ".tmp";
std::error_code ec;
std::filesystem::copy_file(src_file_path, dst_file_path_tmp,
std::filesystem::copy_options::overwrite_existing,
ec);
std::filesystem::copy_file(
ailego::FileHelper::PathFromUtf8(src_file_path),
ailego::FileHelper::PathFromUtf8(dst_file_path_tmp),
std::filesystem::copy_options::overwrite_existing, ec);
if (ec) {
return false;
}
std::filesystem::rename(dst_file_path_tmp, dst_file_path, ec);
std::filesystem::rename(ailego::FileHelper::PathFromUtf8(dst_file_path_tmp),
ailego::FileHelper::PathFromUtf8(dst_file_path), ec);
return !ec;
#else
int src_fd = open(src_file_path.c_str(), O_RDONLY, 0);
if (src_fd < 0) {
return false;
}
AILEGO_DEFER([src_fd] { close(src_fd); });
std::string dst_file_path_tmp = dst_file_path + ".tmp";
int dst_fd =
open(dst_file_path_tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (dst_fd < 0) {
return false;
}
AILEGO_DEFER([dst_fd] { close(dst_fd); });
ssize_t size;
char buf[BUFSIZ];
while ((size = read(src_fd, buf, BUFSIZ)) > 0) {
if (size != write(dst_fd, buf, size)) {
return false;
}
}
return rename(dst_file_path_tmp.c_str(), dst_file_path.c_str()) == 0;
#endif
}
bool FileHelper::CopyDirectory(const std::string &src_dir_path,
const std::string &dst_dir_path) {
#ifdef _MSC_VER
std::error_code ec;
std::filesystem::copy(src_dir_path, dst_dir_path,
std::filesystem::copy(ailego::FileHelper::PathFromUtf8(src_dir_path),
ailego::FileHelper::PathFromUtf8(dst_dir_path),
std::filesystem::copy_options::recursive |
std::filesystem::copy_options::overwrite_existing,
ec);
return !ec;
#else
DIR *dir = opendir(src_dir_path.c_str());
if (!dir) {
return false;
}
AILEGO_DEFER([dir] { closedir(dir); });
if (!ailego::FileHelper::IsExist(dst_dir_path.c_str())) {
if (!ailego::FileHelper::MakePath(dst_dir_path.c_str())) {
return false;
}
}
struct dirent *dent;
while ((dent = readdir(dir)) != nullptr) {
if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
continue;
}
std::string src_full_path =
ailego::StringHelper::Concat(src_dir_path, "/", dent->d_name);
std::string dst_full_path =
ailego::StringHelper::Concat(dst_dir_path, "/", dent->d_name);
if (ailego::FileHelper::IsDirectory(src_full_path.c_str())) {
if (!CopyDirectory(src_full_path, dst_full_path)) {
return false;
}
} else {
if (!CopyFile(src_full_path, dst_full_path)) {
return false;
}
}
}
return true;
#endif
}
void FileHelper::CleanupDirectory(const std::string &backup_dir,
size_t max_backup_count,
const char *prefix_name) {
if (max_backup_count <= 0) {
if (max_backup_count == 0) {
return;
}
#ifdef _MSC_VER
size_t prefix_len = strlen(prefix_name);
std::vector<std::string> candidates;
std::error_code ec;
for (const auto &entry :
std::filesystem::directory_iterator(backup_dir, ec)) {
std::string name = entry.path().filename().string();
for (const auto &entry : std::filesystem::directory_iterator(
ailego::FileHelper::PathFromUtf8(backup_dir), ec)) {
std::string name = entry.path().filename().u8string();
if (name.compare(0, prefix_len, prefix_name) == 0) {
candidates.emplace_back(name);
}
}
#else
DIR *dir = opendir(backup_dir.c_str());
if (!dir) {
if (ec) {
return;
}
AILEGO_DEFER([dir] { closedir(dir); });
size_t prefix_len = strlen(prefix_name);
std::vector<std::string> candidates;
struct dirent *dent;
while ((dent = readdir(dir)) != nullptr) {
if (strncmp(dent->d_name, prefix_name, prefix_len) == 0) {
candidates.emplace_back(dent->d_name);
}
}
#endif
if (candidates.size() <= max_backup_count) {
return;
}
std::sort(candidates.begin(), candidates.end());
for (size_t i = 0; i < candidates.size() - max_backup_count; ++i) {
std::string path =
ailego::StringHelper::Concat(backup_dir, "/", candidates[i].c_str());
ailego::FileHelper::RemovePath(path.c_str());
std::string path_str =
ailego::FileHelper::PathJoin(backup_dir, candidates[i]);
ailego::FileHelper::RemovePath(path_str.c_str());
}
}

75
src/db/common/file_helper.h
View File

@ -73,16 +73,17 @@ class FileHelper {
public:
static const std::string MakeWalPath(const std::string &path, uint32_t seg_id,
uint32_t block_id) {
return ailego::StringHelper::Concat(path, "/", seg_id, "/", block_id,
".wal");
return ailego::FileHelper::PathJoin(
path, seg_id, ailego::StringHelper::Concat(block_id, ".wal"));
}
static std::string MakeSegmentPath(const std::string &path, uint32_t id,
const std::string &suffix = "") {
if (suffix.empty()) {
return ailego::StringHelper::Concat(path, "/", id);
return ailego::FileHelper::PathJoin(path, id);
}
return ailego::StringHelper::Concat(path, "/", id, ".", suffix);
return ailego::FileHelper::PathJoin(
path, ailego::StringHelper::Concat(id, ".", suffix));
}
static std::string MakeTempSegmentPath(const std::string &path, uint32_t id) {
@ -94,99 +95,105 @@ class FileHelper {
uint32_t seg_id,
uint32_t block_id,
bool use_parquet = false) {
return use_parquet ? MakeForwardBlockPath(path, seg_id, block_id,
std::string("parquet"))
: MakeForwardBlockPath(path, seg_id, block_id,
std::string("ipc"));
return MakeForwardBlockPath(path, seg_id, block_id,
std::string(use_parquet ? "parquet" : "ipc"));
}
static const std::string MakeForwardBlockPath(const std::string &path,
uint32_t seg_id,
uint32_t block_id,
const std::string &suffix) {
return ailego::StringHelper::Concat(path, "/", seg_id, "/scalar.", block_id,
".", suffix);
return ailego::FileHelper::PathJoin(
path, seg_id,
ailego::StringHelper::Concat("scalar.", block_id, ".", suffix));
}
static const std::string MakeForwardBlockPath(const std::string &seg_path,
uint32_t block_id,
bool use_parquet = false) {
return use_parquet ? ailego::StringHelper::Concat(seg_path, "/scalar.",
block_id, ".parquet")
: ailego::StringHelper::Concat(seg_path, "/scalar.",
block_id, ".ipc");
return MakeForwardBlockPath(seg_path, block_id,
std::string(use_parquet ? "parquet" : "ipc"));
}
static const std::string MakeForwardBlockPath(const std::string &seg_path,
uint32_t block_id,
const std::string &suffix) {
return ailego::StringHelper::Concat(seg_path, "/scalar.", block_id, ".",
suffix);
return ailego::FileHelper::PathJoin(
seg_path,
ailego::StringHelper::Concat("scalar.", block_id, ".", suffix));
}
// e.g.: **/seg1/scalar.index.block.1.rocksdb
static const std::string MakeInvertIndexPath(const std::string &path,
uint32_t seg_id,
uint32_t block_id) {
return ailego::StringHelper::Concat(path, "/", seg_id, "/scalar.index.",
block_id, ".rocksdb");
return ailego::FileHelper::PathJoin(
path, seg_id,
ailego::StringHelper::Concat("scalar.index.", block_id, ".rocksdb"));
}
static const std::string MakeInvertIndexPath(const std::string &seg_path,
uint32_t block_id) {
return ailego::StringHelper::Concat(seg_path, "/scalar.index.", block_id,
".rocksdb");
return ailego::FileHelper::PathJoin(
seg_path,
ailego::StringHelper::Concat("scalar.index.", block_id, ".rocksdb"));
}
static const std::string MakeVectorIndexPath(const std::string &path,
const std::string &column,
uint32_t seg_id,
uint32_t block_id) {
return ailego::StringHelper::Concat(path, "/", seg_id, "/", column,
".index.", block_id, ".proxima");
return ailego::FileHelper::PathJoin(
path, seg_id,
ailego::StringHelper::Concat(column, ".index.", block_id, ".proxima"));
}
static const std::string MakeVectorIndexPath(const std::string &seg_path,
const std::string &column,
uint32_t block_id) {
return ailego::StringHelper::Concat(seg_path, "/", column, ".index.",
block_id, ".proxima");
return ailego::FileHelper::PathJoin(
seg_path,
ailego::StringHelper::Concat(column, ".index.", block_id, ".proxima"));
}
// e.g.: **/{seg_id}/{column}.index.block.{block_id}.proxima
static const std::string MakeQuantizeVectorIndexPath(
const std::string &path, const std::string &column, uint32_t seg_id,
uint32_t block_id) {
return ailego::StringHelper::Concat(path, "/", seg_id, "/", column,
".qindex.", block_id, ".proxima");
return ailego::FileHelper::PathJoin(
path, seg_id,
ailego::StringHelper::Concat(column, ".qindex.", block_id, ".proxima"));
}
static const std::string MakeQuantizeVectorIndexPath(
const std::string &seg_path, const std::string &column,
uint32_t block_id) {
return ailego::StringHelper::Concat(seg_path, "/", column, ".qindex.",
block_id, ".proxima");
return ailego::FileHelper::PathJoin(
seg_path,
ailego::StringHelper::Concat(column, ".qindex.", block_id, ".proxima"));
}
//! Make file path with ${prefix_path}/${file_name}
static std::string MakeFilePath(const std::string &prefix_path,
FileID file_id) {
return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id));
return ailego::FileHelper::PathJoin(prefix_path, GetFileName(file_id));
}
//! Make file path with ${prefix_path}/${file_name}.${number}
static std::string MakeFilePath(const std::string &prefix_path,
FileID file_id, uint32_t number) {
return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id),
".", number);
return ailego::FileHelper::PathJoin(
prefix_path,
ailego::StringHelper::Concat(GetFileName(file_id), ".", number));
}
//! Make file path with ${prefix_path}/${file_name}.${suffix_name}.${number}
static std::string MakeFilePath(const std::string &prefix_path,
FileID file_id, uint32_t number,
const std::string &suffix_name) {
return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id),
".", suffix_name, ".", number);
return ailego::FileHelper::PathJoin(
prefix_path, ailego::StringHelper::Concat(GetFileName(file_id), ".",
suffix_name, ".", number));
}
//! Create directory
@ -273,4 +280,4 @@ class FileHelper {
};
} // namespace zvec
} // namespace zvec

4
src/db/common/logger.h
View File

@ -18,6 +18,7 @@
#include <zvec/ailego/io/file.h>
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/pattern/factory.h>
#include <zvec/ailego/utility/file_helper.h>
#include <zvec/db/status.h>
#include "db/common/constants.h"
#include "error_code.h"
@ -50,7 +51,8 @@ class LogUtil {
if (logger_type == FILE_LOG_TYPE_NAME) {
params.set("proxima.file.logger.log_dir", log_dir);
params.set("proxima.file.logger.log_file", log_file);
params.set("proxima.file.logger.path", log_dir + "/" + log_file);
params.set("proxima.file.logger.path",
ailego::FileHelper::PathJoin(log_dir, log_file));
std::string program_name = ailego::File::BaseName(gflags::GetArgv0());
params.set("proxima.program.program_name", program_name);
params.set("proxima.file.logger.file_size", log_file_size);

17
src/db/index/common/version_manager.cc
View File

@ -34,7 +34,8 @@
namespace zvec {
Status Version::Load(const std::string &path, Version *version) {
std::ifstream ifs(path, std::ios::binary);
std::ifstream ifs;
ailego::FileHelper::OpenIfstream(ifs, path, std::ios::binary);
if (!ifs.is_open()) {
LOG_ERROR("Failed to open file: %s", path.c_str());
return Status::InternalError("Failed to open file");
@ -74,7 +75,8 @@ Status Version::Load(const std::string &path, Version *version) {
}
Status Version::Save(const std::string &path, const Version &version) {
std::ofstream ofs(path, std::ios::binary);
std::ofstream ofs;
ailego::FileHelper::OpenOfstream(ofs, path, std::ios::binary);
if (!ofs.is_open()) {
LOG_ERROR("Failed to open file: %s, err: %s", path.c_str(),
ailego::FileHelper::GetLastErrorString().c_str());
@ -187,12 +189,13 @@ std::string Version::to_string_formatted(int indent_level) const {
Result<VersionManager::Ptr> VersionManager::Recovery(const std::string &path) {
namespace fs = std::filesystem;
if (!fs::exists(path)) {
auto u8path = ailego::FileHelper::PathFromUtf8(path);
if (!fs::exists(u8path)) {
LOG_ERROR("VersionManager::Recovery: path %s does not exist", path.c_str());
return tl::make_unexpected(
Status::NotFound("path ", path, " does not exist"));
}
if (!fs::is_directory(path)) {
if (!fs::is_directory(u8path)) {
LOG_ERROR("VersionManager::Recovery: path %s is not a directory",
path.c_str());
return tl::make_unexpected(
@ -207,14 +210,14 @@ Result<VersionManager::Ptr> VersionManager::Recovery(const std::string &path) {
uint64_t max_id = UINT64_MAX;
std::string version_path;
for (const auto &entry : fs::directory_iterator(path)) {
for (const auto &entry : fs::directory_iterator(u8path)) {
if (entry.is_regular_file()) {
std::string filename = entry.path().filename().string();
std::string filename = entry.path().filename().u8string();
if (std::regex_match(filename, match, regex)) {
uint64_t id = std::stoull(match[1].str());
if (id > max_id || max_id == UINT64_MAX) {
max_id = id;
version_path = entry.path().string();
version_path = entry.path().u8string();
}
}
}

2
src/db/index/common/version_manager.h
View File

@ -164,7 +164,7 @@ class Version {
private:
CollectionSchema::Ptr schema_;
bool enable_mmap_;
bool enable_mmap_{false};
std::unordered_map<SegmentID, SegmentMeta::Ptr> persisted_segment_metas_map_;

3
src/db/index/storage/store_helper.h
View File

@ -40,7 +40,8 @@
namespace zvec {
inline FileFormat InferFileFormat(const std::string &file_path) {
std::string ext = std::filesystem::path(file_path).extension().string();
std::string ext =
ailego::FileHelper::PathFromUtf8(file_path).extension().u8string();
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
if (ext == ".parquet") {
return FileFormat::PARQUET;

4
src/db/sqlengine/parser/zvec_parser.cc
View File

@ -22,6 +22,7 @@
#include <typeinfo>
#include <zvec/ailego/encoding/json.h>
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/utility/file_helper.h>
#include <zvec/ailego/utility/string_helper.h>
#include "db/sqlengine/common/util.h"
#include "tree/ParseTree.h"
@ -118,7 +119,8 @@ std::string ZVecParser::to_formatted_string_tree(void *tree, void *parser) {
void ZVecParser::save_to_file(const std::string &file_name,
const std::string &formatted) {
std::ofstream outfile(file_name);
std::ofstream outfile;
ailego::FileHelper::OpenOfstream(outfile, file_name);
outfile << formatted;
outfile << std::endl;
outfile.close();

60
src/include/zvec/ailego/utility/file_helper.h
View File

@ -15,8 +15,11 @@
#pragma once
#include <cstring>
#include <filesystem>
#include <fstream>
#include <string>
#include <zvec/ailego/internal/platform.h>
#include <zvec/ailego/utility/string_helper.h>
namespace zvec {
namespace ailego {
@ -41,10 +44,11 @@ struct FileHelper {
//! Retrieve the final path for the specified file
static bool GetFilePath(NativeHandle handle, std::string *path);
//! Retrieve current working directory
//! Retrieve current working directory (UTF-8 bytes in \p *path)
static bool GetWorkingDirectory(std::string *path);
//! Get the size of a file
//! Narrow paths are UTF-8 on all platforms (on Windows, decoded as UTF-8;
//! on POSIX, native narrow encoding is typically UTF-8).
static bool GetFileSize(const char *path, size_t *psz);
//! Delete a name and possibly the file it refers to
@ -77,8 +81,8 @@ struct FileHelper {
//! Retrieve non-zero if two paths are pointing to the same file
static bool IsSame(const char *path1, const char *path2);
//! Retrieve a human-readable string for the most recent OS error
//! (GetLastError() on Windows, strerror(errno) on POSIX)
//! Human-readable error: last std::filesystem failure on this thread if any,
//! else GetLastError() on Windows or strerror(errno) on POSIX.
static std::string GetLastErrorString();
//! Retrieve the size of a file
@ -96,6 +100,54 @@ struct FileHelper {
}
return (output ? output + 1 : path);
}
//! Build std::filesystem::path from a UTF-8 string (handles Windows codepage)
static std::filesystem::path PathFromUtf8(const std::string &s);
static std::filesystem::path PathFromUtf8(const char *s);
//! Convert std::filesystem::path back to a UTF-8 std::string
static std::string PathToUtf8(const std::filesystem::path &p);
//! Concatenate path segments with the native separator.
//! Accepts strings, c-strings, and numbers (anything Alphameric accepts).
//! E.g. PathJoin(dir, seg_id, "file.ext")
static std::string PathJoin(const internal::Alphameric &a,
const internal::Alphameric &b) {
return StringHelper::Concat(a, kPathSep, b);
}
template <typename... Rest>
static std::string PathJoin(const internal::Alphameric &a,
const internal::Alphameric &b,
const Rest &...rest) {
return PathJoin(StringHelper::Concat(a, kPathSep, b), rest...);
}
private:
#if defined(_WIN32) || defined(_WIN64)
static constexpr const char *kPathSep = "\\";
#else
static constexpr const char *kPathSep = "/";
#endif
public:
#if defined(_WIN32) || defined(_WIN64)
//! UTF-8 narrow string -> wide string (Win32 API ready)
static std::wstring Utf8ToWide(const std::string &utf8);
//! Wide string -> UTF-8 narrow string
static std::string WideToUtf8(const std::wstring &ws);
#endif
//! Open a std::ifstream from a UTF-8 path
static void OpenIfstream(std::ifstream &ifs, const std::string &path,
std::ios_base::openmode mode = std::ios_base::in) {
ifs.open(PathFromUtf8(path), mode);
}
//! Open a std::ofstream from a UTF-8 path
static void OpenOfstream(std::ofstream &ofs, const std::string &path,
std::ios_base::openmode mode = std::ios_base::out) {
ofs.open(PathFromUtf8(path), mode);
}
};
} // namespace ailego

9
src/include/zvec/core/framework/index_bundle.h
View File

@ -20,6 +20,7 @@
#include <zvec/ailego/container/blob.h>
#include <zvec/ailego/io/file.h>
#include <zvec/ailego/io/mmap_file.h>
#include <zvec/ailego/utility/file_helper.h>
namespace zvec {
namespace core {
@ -210,7 +211,7 @@ class MMapFileIndexBundle : public IndexBundle {
//! Create a memory mapping file in bundle
bool create(const std::string &prefix, const std::string &key, size_t len) {
ailego::MMapFile file;
if (!file.create(prefix + '/' + key, len)) {
if (!file.create(ailego::FileHelper::PathJoin(prefix, key), len)) {
return false;
}
map_[key] = std::move(file);
@ -220,7 +221,7 @@ class MMapFileIndexBundle : public IndexBundle {
//! Create a memory mapping file in bundle
bool create(const std::string &prefix, std::string &&key, size_t len) {
ailego::MMapFile file;
if (!file.create(prefix + '/' + key, len)) {
if (!file.create(ailego::FileHelper::PathJoin(prefix, key), len)) {
return false;
}
map_[std::move(key)] = std::move(file);
@ -240,7 +241,7 @@ class MMapFileIndexBundle : public IndexBundle {
//! Open a memory mapping file in bundle
bool open(const std::string &prefix, const std::string &key, bool rdonly) {
ailego::MMapFile file;
if (!file.open(prefix + '/' + key, rdonly)) {
if (!file.open(ailego::FileHelper::PathJoin(prefix, key), rdonly)) {
return false;
}
map_[key] = std::move(file);
@ -250,7 +251,7 @@ class MMapFileIndexBundle : public IndexBundle {
//! Open a memory mapping file in bundle
bool open(const std::string &prefix, std::string &&key, bool rdonly) {
ailego::MMapFile file;
if (!file.open(prefix + '/' + key, rdonly)) {
if (!file.open(ailego::FileHelper::PathJoin(prefix, key), rdonly)) {
return false;
}
map_[std::move(key)] = std::move(file);

4
tests/ailego/io/file_test.cc
View File

@ -45,7 +45,9 @@ TEST(File, General) {
}
TEST(File, MakePath) {
EXPECT_TRUE(File::MakePath(""));
EXPECT_FALSE(File::MakePath(""));
std::cout << FileHelper::GetLastErrorString() << std::endl;
EXPECT_TRUE(File::MakePath("."));
EXPECT_TRUE(File::MakePath(".."));
EXPECT_TRUE(File::MakePath("../"));

287
tests/ailego/io/utf8_path_test.cc Normal file
View File

@ -0,0 +1,287 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if defined(_WIN32) || defined(_WIN64)
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#undef DeleteFile
#undef RemoveDirectory
#undef CreateFile
#endif
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <string>
#include <gtest/gtest.h>
#include <zvec/ailego/io/file.h>
#include <zvec/ailego/utility/file_helper.h>
#include <zvec/ailego/utility/string_helper.h>
using namespace zvec::ailego;
// UTF-8 encoded Chinese / Japanese test strings.
// These contain characters outside the Windows ANSI codepage, so any path
// handling that falls back to narrow Win32 APIs will produce the mojibake.
static const std::string kChinese =
"\xe4\xb8\xad\xe6\x96\x87\xe8\xb7\xaf\xe5\xbe\x84"; // 中文路径
static const std::string kJapanese =
"\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88"; // テスト
static const std::string kMixed = "data_测试_2025"; // data_测试_2025
class Utf8PathTest : public ::testing::Test {
protected:
void TearDown() override {
for (auto &p : cleanup_) {
FileHelper::RemovePath(p.c_str());
}
}
void ScheduleCleanup(const std::string &path) {
cleanup_.push_back(path);
}
private:
std::vector<std::string> cleanup_;
};
// ---------------------------------------------------------------------------
// 1. Directory create / query / remove with Chinese path
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, MakePath_Chinese) {
std::string dir = "utf8_test_" + kChinese;
ScheduleCleanup(dir);
ASSERT_TRUE(FileHelper::MakePath(dir.c_str()));
EXPECT_TRUE(FileHelper::IsExist(dir.c_str()));
EXPECT_TRUE(FileHelper::IsDirectory(dir.c_str()));
EXPECT_FALSE(FileHelper::IsRegular(dir.c_str()));
ASSERT_TRUE(FileHelper::RemoveDirectory(dir.c_str()));
EXPECT_FALSE(FileHelper::IsExist(dir.c_str()));
}
// ---------------------------------------------------------------------------
// 2. Nested directories with mixed UTF-8 segments
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, MakePath_Nested) {
std::string root = "utf8_test_nested_" + kChinese;
std::string nested = FileHelper::PathJoin(root, kJapanese, kMixed);
ScheduleCleanup(root);
ASSERT_TRUE(FileHelper::MakePath(nested.c_str()));
EXPECT_TRUE(FileHelper::IsDirectory(nested.c_str()));
ASSERT_TRUE(FileHelper::RemoveDirectory(root.c_str()));
EXPECT_FALSE(FileHelper::IsExist(root.c_str()));
}
// ---------------------------------------------------------------------------
// 3. File create / read / write through ailego::File with UTF-8 path
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, FileCreateAndReadWrite) {
std::string dir = "utf8_test_file_" + kChinese;
std::string file_path = FileHelper::PathJoin(dir, kJapanese + ".dat");
ScheduleCleanup(dir);
ASSERT_TRUE(FileHelper::MakePath(dir.c_str()));
const std::string payload =
"hello_utf8_\xe4\xbd\xa0\xe5\xa5\xbd"; // hello_utf8_你好
{
File f;
ASSERT_TRUE(f.create(file_path.c_str(), 0));
EXPECT_EQ(payload.size(), f.write(payload.data(), payload.size()));
EXPECT_TRUE(f.flush());
}
EXPECT_TRUE(FileHelper::IsRegular(file_path.c_str()));
size_t sz = 0;
ASSERT_TRUE(FileHelper::GetFileSize(file_path.c_str(), &sz));
EXPECT_EQ(payload.size(), sz);
{
File f;
ASSERT_TRUE(f.open(file_path.c_str(), true));
std::string buf(sz, '\0');
EXPECT_EQ(sz, f.read(buf.data(), sz));
EXPECT_EQ(payload, buf);
}
ASSERT_TRUE(FileHelper::DeleteFile(file_path.c_str()));
EXPECT_FALSE(FileHelper::IsExist(file_path.c_str()));
}
// ---------------------------------------------------------------------------
// 4. FileHelper::RenameFile across UTF-8 paths
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, RenameFile) {
std::string dir = "utf8_test_rename_" + kChinese;
std::string src = FileHelper::PathJoin(dir, "src_" + kJapanese);
std::string dst = FileHelper::PathJoin(dir, "dst_" + kMixed);
ScheduleCleanup(dir);
ASSERT_TRUE(FileHelper::MakePath(dir.c_str()));
{
File f;
ASSERT_TRUE(f.create(src.c_str(), 0));
}
EXPECT_TRUE(FileHelper::IsExist(src.c_str()));
ASSERT_TRUE(FileHelper::RenameFile(src.c_str(), dst.c_str()));
EXPECT_FALSE(FileHelper::IsExist(src.c_str()));
EXPECT_TRUE(FileHelper::IsExist(dst.c_str()));
}
// ---------------------------------------------------------------------------
// 5. OpenIfstream / OpenOfstream with UTF-8 path
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, FstreamUtf8) {
std::string dir = "utf8_test_fstream_" + kChinese;
std::string file_path = FileHelper::PathJoin(dir, kMixed + ".txt");
ScheduleCleanup(dir);
ASSERT_TRUE(FileHelper::MakePath(dir.c_str()));
const std::string content =
"line1\nline2_\xe6\xb5\x8b\xe8\xaf\x95\n"; // line2_测试
{
std::ofstream ofs;
FileHelper::OpenOfstream(ofs, file_path, std::ios::binary);
ASSERT_TRUE(ofs.is_open());
ofs.write(content.data(), content.size());
}
{
std::ifstream ifs;
FileHelper::OpenIfstream(ifs, file_path, std::ios::binary);
ASSERT_TRUE(ifs.is_open());
std::string buf((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
EXPECT_EQ(content, buf);
}
}
// ---------------------------------------------------------------------------
// 6. PathJoin produces correct separators
// ---------------------------------------------------------------------------
TEST_F(Utf8PathTest, PathJoinSeparator) {
std::string result = FileHelper::PathJoin("root", "sub", "file.txt");
#if defined(_WIN32) || defined(_WIN64)
EXPECT_EQ("root\\sub\\file.txt", result);
#else
EXPECT_EQ("root/sub/file.txt", result);
#endif
}
// ---------------------------------------------------------------------------
// 7. Verify NO mojibake on actual filesystem via "dir" (Windows) / "ls" (POSIX)
// We create a directory + file with Chinese names, then list the parent
// using the OS shell and check the expected name appears verbatim.
// ---------------------------------------------------------------------------
#if defined(_WIN32) || defined(_WIN64)
TEST_F(Utf8PathTest, NoGarbledNames_FindFirstFile) {
std::string parent = "utf8_test_dir_verify";
std::string child_dir = kChinese;
std::string child_file = kJapanese + ".dat";
std::string full_dir = FileHelper::PathJoin(parent, child_dir);
std::string full_file = FileHelper::PathJoin(parent, child_file);
ScheduleCleanup(parent);
ASSERT_TRUE(FileHelper::MakePath(full_dir.c_str()));
{
File f;
ASSERT_TRUE(f.create(full_file.c_str(), 0));
}
// Use FindFirstFileW / FindNextFileW to enumerate the parent directory.
// This is the most reliable way to verify the OS actually stored correct
// Unicode names (no mojibake), bypassing any console codepage issues.
std::wstring pattern = FileHelper::Utf8ToWide(parent) + L"\\*";
WIN32_FIND_DATAW fd;
HANDLE hFind = FindFirstFileW(pattern.c_str(), &fd);
ASSERT_NE(INVALID_HANDLE_VALUE, hFind) << "FindFirstFileW failed";
std::vector<std::wstring> entries;
do {
std::wstring name(fd.cFileName);
if (name != L"." && name != L"..") {
entries.push_back(name);
}
} while (FindNextFileW(hFind, &fd));
FindClose(hFind);
std::wstring expected_dir = FileHelper::Utf8ToWide(child_dir);
std::wstring expected_file = FileHelper::Utf8ToWide(child_file);
bool found_dir = false, found_file = false;
std::cout << "[NoGarbledNames] Entries in " << parent << ":" << std::endl;
for (const auto &e : entries) {
std::string utf8_name = FileHelper::WideToUtf8(e);
std::cout << " " << utf8_name << std::endl;
if (e == expected_dir) found_dir = true;
if (e == expected_file) found_file = true;
}
EXPECT_TRUE(found_dir) << "Directory '" << child_dir
<< "' not found (garbled Unicode?)";
EXPECT_TRUE(found_file) << "File '" << (kJapanese + ".dat")
<< "' not found (garbled Unicode?)";
}
TEST_F(Utf8PathTest, NoGarbledNames_DirCmd) {
std::string parent = "utf8_test_dir_cmd";
std::string child = kChinese;
std::string full = FileHelper::PathJoin(parent, child);
std::string tmpfile = "utf8_test_dir_cmd_output.txt";
ScheduleCleanup(parent);
ASSERT_TRUE(FileHelper::MakePath(full.c_str()));
// "cmd /u" makes built-in commands output UTF-16LE when redirected to file.
std::wstring wparent = FileHelper::Utf8ToWide(parent);
std::wstring wtmp = FileHelper::Utf8ToWide(tmpfile);
std::wstring cmd =
L"cmd /u /c dir /b \"" + wparent + L"\" >\"" + wtmp + L"\"";
_wsystem(cmd.c_str());
// Read back the file as raw bytes (UTF-16LE)
std::ifstream ifs;
FileHelper::OpenIfstream(ifs, tmpfile, std::ios::binary);
ASSERT_TRUE(ifs.is_open()) << "Could not read dir output file";
std::string raw((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
ifs.close();
FileHelper::DeleteFile(tmpfile.c_str());
// Interpret as UTF-16LE: skip BOM if present, then convert to UTF-8
const wchar_t *wdata = reinterpret_cast<const wchar_t *>(raw.data());
size_t wlen = raw.size() / sizeof(wchar_t);
if (wlen > 0 && wdata[0] == L'\xFEFF') {
wdata++;
wlen--;
}
std::wstring woutput(wdata, wlen);
std::string output = FileHelper::WideToUtf8(woutput);
std::cout << "[NoGarbledNames_DirCmd] dir output: [" << output << "]";
EXPECT_NE(std::string::npos, output.find(child))
<< "Chinese directory name not found in 'dir' output (garbled?).\n"
<< "Got: [" << output << "]";
}
#endif

4
tests/db/collection_test.cc
View File

@ -216,7 +216,7 @@ TEST_F(CollectionTest, Feature_CreateAndOpen_PathValidate) {
auto schema = TestHelper::CreateNormalSchema();
{
std::vector<std::string> valid_paths = {"abc",
std::vector<std::string> valid_paths = {"你好",
"data123",
"my_collection",
"v1.2_alpha-beta",
@ -234,6 +234,8 @@ TEST_F(CollectionTest, Feature_CreateAndOpen_PathValidate) {
auto result = Collection::CreateAndOpen(path, *schema, options);
if (!result.has_value()) {
std::cout << result.error().message() << std::endl;
std::cout << "File error:" << ailego::FileHelper::GetLastErrorString()
<< std::endl;
}
ASSERT_TRUE(result.has_value());

4
tests/db/crash_recovery/collection_optimizer.cc
View File

@ -85,8 +85,8 @@ int main(int argc, char **argv) {
// Scope 'result' so its shared_ptr is released before we call _exit().
zvec::Collection::Ptr collection;
{
auto result =
zvec::Collection::Open(config.path, zvec::CollectionOptions{false, true});
auto result = zvec::Collection::Open(config.path,
zvec::CollectionOptions{false, true});
if (!result) {
LOG_ERROR("Failed to open collection[%s]: %s", config.path.c_str(),
result.error().c_str());

9
tests/db/index/common/version_manager_test.cc
View File

@ -27,9 +27,10 @@ class VersionManagerTest : public ::testing::Test {
protected:
void SetUp() override {
// Create a temporary directory for testing
test_path_ = "./version_manager_test";
test_path_ = "./test_version_manager";
FileHelper::RemoveDirectory(test_path_);
FileHelper::CreateDirectory(test_path_);
ASSERT_TRUE(FileHelper::CreateDirectory(test_path_))
<< ailego::FileHelper::GetLastErrorString();
}
void TearDown() override {
@ -131,7 +132,7 @@ TEST_F(VersionManagerTest, VersionManagerCreateAndRecover) {
// Create VersionManager
auto create_result = VersionManager::Create(version_path, initial_version);
EXPECT_TRUE(create_result.has_value());
ASSERT_TRUE(create_result.has_value());
auto version_manager = create_result.value();
@ -149,7 +150,7 @@ TEST_F(VersionManagerTest, VersionManagerCreateAndRecover) {
// Recover VersionManager
auto recover_result = VersionManager::Recovery(version_path);
EXPECT_TRUE(recover_result.has_value());
ASSERT_TRUE(recover_result.has_value());
auto recovered_manager = recover_result.value();
auto recovered_version = recovered_manager->get_current_version();

211
tests/db/utf8_collection_test.cc Normal file
View File

@ -0,0 +1,211 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdio>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include <zvec/ailego/io/file.h>
#include <zvec/ailego/utility/file_helper.h>
#include "db/common/file_helper.h"
#include "index/utils/utils.h"
#include "zvec/db/collection.h"
#include "zvec/db/doc.h"
#include "zvec/db/options.h"
#include "zvec/db/schema.h"
#include "zvec/db/status.h"
#if defined(_WIN32) || defined(_WIN64)
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#undef DeleteFile
#undef RemoveDirectory
#undef CreateFile
#undef DELETE
#endif
using namespace zvec;
using namespace zvec::test;
// UTF-8 test path containing Chinese characters.
// If the path handling is broken, the OS will show garbled names (mojibake).
static const std::string kUtf8Dir =
"utf8_col_\xe4\xb8\xad\xe6\x96\x87\xe6\xb5\x8b\xe8\xaf\x95"; // utf8_col_中文测试
class Utf8CollectionTest : public ::testing::Test {
protected:
void SetUp() override {
ailego::FileHelper::RemovePath(kUtf8Dir.c_str());
}
void TearDown() override {
ailego::FileHelper::RemovePath(kUtf8Dir.c_str());
}
};
// ---------------------------------------------------------------------------
// 1. Create a collection in a UTF-8 path, insert docs, flush, reopen
// ---------------------------------------------------------------------------
TEST_F(Utf8CollectionTest, CreateInsertFlushReopen) {
CollectionOptions opts;
opts.read_only_ = false;
opts.enable_mmap_ = true;
auto schema = TestHelper::CreateNormalSchema();
auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts);
if (!result.has_value()) {
std::cout << result.error().message() << std::endl;
}
ASSERT_TRUE(result.has_value());
ASSERT_TRUE(ailego::FileHelper::IsExist(kUtf8Dir.c_str()));
auto col = std::move(result).value();
ASSERT_EQ(col->Path(), kUtf8Dir);
const int kDocCount = 10;
auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount);
ASSERT_TRUE(s.ok()) << s.message();
ASSERT_TRUE(col->Flush().ok());
auto stats = col->Stats().value();
ASSERT_EQ(stats.doc_count, kDocCount);
col.reset();
// Reopen and verify doc count survives
auto reopen = Collection::Open(kUtf8Dir, opts);
ASSERT_TRUE(reopen.has_value()) << reopen.error().message();
auto col2 = std::move(reopen).value();
auto stats2 = col2->Stats().value();
ASSERT_EQ(stats2.doc_count, kDocCount);
}
// ---------------------------------------------------------------------------
// 2. Destroy a collection in a UTF-8 path
// ---------------------------------------------------------------------------
TEST_F(Utf8CollectionTest, CreateAndDestroy) {
CollectionOptions opts;
opts.read_only_ = false;
opts.enable_mmap_ = true;
auto schema = TestHelper::CreateNormalSchema();
auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts);
ASSERT_TRUE(result.has_value());
auto col = std::move(result).value();
ASSERT_EQ(col->Destroy(), Status::OK());
ASSERT_FALSE(ailego::FileHelper::IsExist(kUtf8Dir.c_str()));
}
// ---------------------------------------------------------------------------
// 3. Verify the on-disk layout contains correct UTF-8 names (no mojibake)
// by listing files via the wide-char Windows API or POSIX readdir.
// ---------------------------------------------------------------------------
#if defined(_WIN32) || defined(_WIN64)
TEST_F(Utf8CollectionTest, NoDirGarble) {
CollectionOptions opts;
opts.read_only_ = false;
opts.enable_mmap_ = true;
auto schema = TestHelper::CreateNormalSchema();
auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts);
ASSERT_TRUE(result.has_value());
auto col = std::move(result).value();
const int kDocCount = 5;
auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount);
ASSERT_TRUE(s.ok()) << s.message();
ASSERT_TRUE(col->Flush().ok());
// --- Check parent directory via FindFirstFileW ---
{
WIN32_FIND_DATAW fd;
HANDLE hFind = FindFirstFileW(L".\\*", &fd);
ASSERT_NE(INVALID_HANDLE_VALUE, hFind);
std::wstring expected = ailego::FileHelper::Utf8ToWide(kUtf8Dir);
bool found = false;
do {
if (std::wstring(fd.cFileName) == expected) {
found = true;
break;
}
} while (FindNextFileW(hFind, &fd));
FindClose(hFind);
EXPECT_TRUE(found) << "Collection dir '" << kUtf8Dir
<< "' not found via FindFirstFileW (garbled Unicode?)";
}
// --- Check inside collection via "cmd /u /c dir /b" redirected to file ---
{
std::wstring wdir = ailego::FileHelper::Utf8ToWide(kUtf8Dir);
std::string tmpfile = "utf8_col_dir_output.txt";
std::wstring wtmp = ailego::FileHelper::Utf8ToWide(tmpfile);
std::wstring cmd = L"cmd /u /c dir /b \"" + wdir + L"\" >\"" + wtmp + L"\"";
_wsystem(cmd.c_str());
std::ifstream ifs;
ailego::FileHelper::OpenIfstream(ifs, tmpfile, std::ios::binary);
ASSERT_TRUE(ifs.is_open());
std::string raw((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
ifs.close();
ailego::FileHelper::DeleteFile(tmpfile.c_str());
const wchar_t *wdata = reinterpret_cast<const wchar_t *>(raw.data());
size_t wlen = raw.size() / sizeof(wchar_t);
if (wlen > 0 && wdata[0] == L'\xFEFF') {
wdata++;
wlen--;
}
std::wstring woutput(wdata, wlen);
std::string output = ailego::FileHelper::WideToUtf8(woutput);
EXPECT_FALSE(output.empty()) << "dir listing inside collection is empty";
std::cout << "[NoDirGarble] Contents of " << kUtf8Dir << ":\n"
<< output << std::endl;
}
}
#endif
// ---------------------------------------------------------------------------
// 4. MakeWalPath / MakeSegmentPath etc. produce correct UTF-8 results
// ---------------------------------------------------------------------------
TEST_F(Utf8CollectionTest, FileHelperPaths) {
std::string wal = FileHelper::MakeWalPath(kUtf8Dir, 0, 1);
std::string seg = FileHelper::MakeSegmentPath(kUtf8Dir, 0);
std::string fwd = FileHelper::MakeForwardBlockPath(kUtf8Dir, 0, 1, true);
std::string inv = FileHelper::MakeInvertIndexPath(kUtf8Dir, 0, 1);
std::string vec = FileHelper::MakeVectorIndexPath(kUtf8Dir, "vec1", 0, 1);
// All paths should start with our UTF-8 base
EXPECT_EQ(0u, wal.find(kUtf8Dir));
EXPECT_EQ(0u, seg.find(kUtf8Dir));
EXPECT_EQ(0u, fwd.find(kUtf8Dir));
EXPECT_EQ(0u, inv.find(kUtf8Dir));
EXPECT_EQ(0u, vec.find(kUtf8Dir));
// Should not contain forward slash on Windows
#if defined(_WIN32) || defined(_WIN64)
EXPECT_EQ(std::string::npos, wal.find('/'));
EXPECT_EQ(std::string::npos, seg.find('/'));
EXPECT_EQ(std::string::npos, fwd.find('/'));
EXPECT_EQ(std::string::npos, inv.find('/'));
EXPECT_EQ(std::string::npos, vec.find('/'));
#endif
}

1
thirdparty/rocksdb/CMakeLists.txt vendored
View File

@ -37,6 +37,7 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE)
set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE)
set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE)
set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE)
set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "use UTF8 as characterset for opening files, regardles of the system code page" FORCE)
# TODO(windows): verify
set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE)
set(FAIL_ON_WARNINGS OFF CACHE BOOL "build with no Werror" FORCE)