diff --git a/src/ailego/io/file.cc b/src/ailego/io/file.cc index 178f731..fa7b36f 100644 --- a/src/ailego/io/file.cc +++ b/src/ailego/io/file.cc @@ -24,6 +24,8 @@ #include #else #include +#include +#include #endif namespace zvec { @@ -405,15 +407,26 @@ bool File::MemoryUnlock(void *addr, size_t len) { #else +namespace { + +bool Utf8PathOk(const char *path, const std::wstring &wide) { + return path && path[0] != '\0' && !wide.empty(); +} + +} // namespace + //! Create a local file bool File::create(const char *path, size_t len, bool direct) { ailego_false_if_false(native_handle_ == File::InvalidHandle && path); + const std::wstring wpath = FileHelper::Utf8ToWide(path); + ailego_false_if_false(Utf8PathOk(path, wpath)); + // Try opening or creating the file HANDLE file_handle = - CreateFileA(path, GENERIC_WRITE | GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + ::CreateFileW(wpath.c_str(), GENERIC_WRITE | GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE); // Truncate the file to the specified size @@ -432,8 +445,8 @@ bool File::create(const char *path, size_t len, bool direct) { } else { // Close and reopen file CloseHandle(file_handle); - file_handle = CreateFileA( - path, GENERIC_WRITE | GENERIC_READ, + file_handle = ::CreateFileW( + wpath.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr); ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE); @@ -448,15 +461,18 @@ bool File::create(const char *path, size_t len, bool direct) { bool File::open(const char *path, bool rdonly, bool direct) { ailego_false_if_false(native_handle_ == File::InvalidHandle && path); + const std::wstring wpath = FileHelper::Utf8ToWide(path); + ailego_false_if_false(Utf8PathOk(path, wpath)); + // Try opening the file DWORD flags = FILE_ATTRIBUTE_NORMAL; if (direct) { flags |= FILE_FLAG_NO_BUFFERING; } - HANDLE file_handle = - CreateFileA(path, (rdonly ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE), - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, OPEN_EXISTING, flags, nullptr); + HANDLE file_handle = ::CreateFileW( + wpath.c_str(), (rdonly ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE), + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, flags, nullptr); ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE); read_only_ = rdonly; diff --git a/src/ailego/utility/file_helper.cc b/src/ailego/utility/file_helper.cc index a5b74ef..5026f40 100644 --- a/src/ailego/utility/file_helper.cc +++ b/src/ailego/utility/file_helper.cc @@ -12,34 +12,238 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include +#include +#include #include #if defined(_WIN32) || defined(_WIN64) #include +#ifdef RemoveDirectory +#undef RemoveDirectory +#endif +#ifdef DeleteFile +#undef DeleteFile +#endif +#ifdef GetFileSize +#undef GetFileSize +#endif #else #if defined(__APPLE__) || defined(__MACH__) #include #endif -#include -#include -#include +#if defined(__FreeBSD__) +#include +#endif #include -#include #include #endif - -#include namespace fs = std::filesystem; -// TODO: refactor all file operations by std::filesystem; namespace zvec { namespace ailego { +namespace { + +thread_local std::error_code g_last_fs_error; + +void ClearFsError() { + g_last_fs_error.clear(); +} + +void SetFsError(std::error_code ec) { + g_last_fs_error = ec; +} + +} // namespace + +// ---------- public UTF-8 / wide helpers ---------- + +fs::path FileHelper::PathFromUtf8(const char *s) { +#if defined(_WIN32) || defined(_WIN64) + if (!s || !*s) { + return fs::path(); + } + return fs::u8path(s); +#else + return fs::path(s ? s : ""); +#endif +} + +fs::path FileHelper::PathFromUtf8(const std::string &s) { + return PathFromUtf8(s.c_str()); +} + +std::string FileHelper::PathToUtf8(const fs::path &p) { + return p.u8string(); +} + +#if defined(_WIN32) || defined(_WIN64) +std::wstring FileHelper::Utf8ToWide(const std::string &src) { + if (src.empty()) { + return {}; + } + int src_len = static_cast(src.size()); + int dst_len = + MultiByteToWideChar(CP_UTF8, 0, src.data(), src_len, nullptr, 0); + if (dst_len <= 0) { + return {}; + } + std::wstring dst(static_cast(dst_len), L'\0'); + if (MultiByteToWideChar(CP_UTF8, 0, src.data(), src_len, dst.data(), + dst_len) != dst_len) { + return {}; + } + return dst; +} + +std::string FileHelper::WideToUtf8(const std::wstring &src) { + if (src.empty()) { + return {}; + } + int src_len = static_cast(src.size()); + int dst_len = WideCharToMultiByte(CP_UTF8, 0, src.data(), src_len, nullptr, 0, + nullptr, nullptr); + if (dst_len <= 0) { + return {}; + } + std::string dst(static_cast(dst_len), '\0'); + if (WideCharToMultiByte(CP_UTF8, 0, src.data(), src_len, dst.data(), dst_len, + nullptr, nullptr) != dst_len) { + return {}; + } + return dst; +} + +#endif + +// ---------- internal helpers ---------- + +namespace { + +static bool GetFileSizeImpl(const fs::path &p, size_t *psz) { + ClearFsError(); + std::error_code ec; + auto sz = fs::file_size(p, ec); + if (ec) { + SetFsError(ec); + return false; + } + *psz = static_cast(sz); + return true; +} + +static bool DeleteFileImpl(const fs::path &p) { + ClearFsError(); + std::error_code ec; + fs::file_status st = fs::symlink_status(p, ec); + if (ec) { + SetFsError(ec); + return false; + } + if (fs::is_directory(st) && !fs::is_symlink(st)) { + ec = std::make_error_code(std::errc::is_a_directory); + SetFsError(ec); + return false; + } + if (!fs::remove(p, ec)) { + SetFsError(ec ? ec + : std::make_error_code(std::errc::no_such_file_or_directory)); + return false; + } + return true; +} + +static bool RenameFileImpl(const fs::path &from, const fs::path &to) { + ClearFsError(); + std::error_code ec; + fs::rename(from, to, ec); + if (ec) { + SetFsError(ec); + return false; + } + return true; +} + +static bool MakePathImpl(const fs::path &p) { + ClearFsError(); + std::error_code ec; + fs::create_directories(p, ec); + if (ec) { + SetFsError(ec); + return false; + } + return true; +} + +static bool RemoveDirectoryImpl(const fs::path &p) { + ClearFsError(); + std::error_code ec; + if (!fs::is_directory(p, ec)) { + if (ec) { + SetFsError(ec); + } + return false; + } + std::uintmax_t n = fs::remove_all(p, ec); + if (ec) { + SetFsError(ec); + return false; + } + (void)n; + return true; +} + +static bool IsExistImpl(const fs::path &p) { + std::error_code ec; + return fs::exists(p, ec); +} + +static bool IsRegularImpl(const fs::path &p) { + std::error_code ec; + return fs::is_regular_file(p, ec); +} + +static bool IsDirectoryImpl(const fs::path &p) { + std::error_code ec; + return fs::is_directory(p, ec); +} + +static bool IsSymbolicLinkImpl(const fs::path &p) { + std::error_code ec; + return fs::is_symlink(p, ec); +} + +static bool IsSameImpl(const fs::path &a, const fs::path &b) { + std::error_code ec; + return fs::equivalent(a, b, ec); +} + +} // namespace + bool FileHelper::GetSelfPath(std::string *path) { #if defined(_WIN32) || defined(_WIN64) - char buf[MAX_PATH]; - DWORD len = GetModuleFileNameA(NULL, buf, MAX_PATH); + std::wstring wbuf(4096, L'\0'); + DWORD n = + GetModuleFileNameW(nullptr, wbuf.data(), static_cast(wbuf.size())); + while (n >= wbuf.size() - 1) { + if (wbuf.size() > 65536) { + return false; + } + wbuf.resize(wbuf.size() * 2); + n = GetModuleFileNameW(nullptr, wbuf.data(), + static_cast(wbuf.size())); + } + if (n == 0) { + return false; + } + wbuf.resize(n); + *path = WideToUtf8(wbuf); + return !path->empty(); #elif defined(__APPLE__) || defined(__MACH__) char buf[PATH_MAX]; size_t len = 0; @@ -62,18 +266,31 @@ bool FileHelper::GetSelfPath(std::string *path) { ssize_t len = readlink("/proc/self/exe", buf, PATH_MAX); #endif +#if !defined(_WIN32) && !defined(_WIN64) if (len <= 0) { return false; } path->assign(buf, len); return true; +#endif } bool FileHelper::GetFilePath(NativeHandle handle, std::string *path) { #if defined(_WIN32) || defined(_WIN64) - char buf[MAX_PATH]; - DWORD len = - GetFinalPathNameByHandleA(handle, buf, MAX_PATH, FILE_NAME_OPENED); + DWORD need = GetFinalPathNameByHandleW(static_cast(handle), nullptr, + 0, FILE_NAME_OPENED); + if (need == 0) { + return false; + } + std::wstring wbuf(static_cast(need) + 1, L'\0'); + DWORD got = GetFinalPathNameByHandleW( + static_cast(handle), wbuf.data(), need + 1, FILE_NAME_OPENED); + if (got == 0 || got > need) { + return false; + } + wbuf.resize(got); + *path = WideToUtf8(wbuf); + return !path->empty(); #elif defined(__linux) || defined(__linux__) char buf[PATH_MAX]; char src[32]; @@ -87,276 +304,75 @@ bool FileHelper::GetFilePath(NativeHandle handle, std::string *path) { } #endif +#if !defined(_WIN32) && !defined(_WIN64) if (len <= 0) { return false; } path->assign(buf, len); return true; -} - -#if !defined(_WIN32) && !defined(_WIN64) - -static inline char *JoinFilePath(const char *prefix, const char *suffix) { - size_t prefix_len = strlen(prefix); - size_t suffix_len = strlen(suffix); - - char *path = (char *)malloc(prefix_len + suffix_len + 2); - if (path) { - memcpy(path, prefix, prefix_len); - memcpy(path + prefix_len + 1, suffix, suffix_len); - path[prefix_len] = '/'; - path[prefix_len + suffix_len + 1] = '\0'; - } - return path; +#endif } bool FileHelper::GetWorkingDirectory(std::string *path) { - char buf[PATH_MAX]; - - if (!getcwd(buf, PATH_MAX)) { + ClearFsError(); + std::error_code ec; + fs::path cwd = fs::current_path(ec); + if (ec) { + SetFsError(ec); return false; } - path->assign(buf); + *path = PathToUtf8(cwd); return !path->empty(); } bool FileHelper::GetFileSize(const char *path, size_t *psz) { - struct stat buf; - if (stat(path, &buf) != 0) { - return false; - } - *psz = buf.st_size; - return true; + return GetFileSizeImpl(PathFromUtf8(path), psz); } bool FileHelper::DeleteFile(const char *path) { - // Delete a file by the path - return (unlink(path) == 0); + return DeleteFileImpl(PathFromUtf8(path)); } bool FileHelper::RenameFile(const char *oldpath, const char *newpath) { - return (rename(oldpath, newpath) == 0); + return RenameFileImpl(PathFromUtf8(oldpath), PathFromUtf8(newpath)); } bool FileHelper::MakePath(const char *path) { - char pathbuf[PATH_MAX]; - char *sp, *pp; - - strncpy(pathbuf, path, sizeof(pathbuf) - 1); - pathbuf[PATH_MAX - 1] = '\0'; - - pp = pathbuf; - while ((sp = strchr(pp, '/')) != nullptr) { - // Neither root nor double slash in path - if (sp != pp) { - *sp = '\0'; - if (mkdir(pathbuf, 0755) == -1 && errno != EEXIST) { - return false; - } - *sp = '/'; - } - pp = sp + 1; - } - return !(*pp != '\0' && mkdir(pathbuf, 0755) == -1 && errno != EEXIST); -} - -bool FileHelper::RemoveDirectory(const char *path) { - DIR *dir = opendir(path); - if (!dir) { - return false; - } - - struct dirent *dent; - while ((dent = readdir(dir)) != nullptr) { - if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) { - continue; - } - char *fullpath = JoinFilePath(path, dent->d_name); - if (!fullpath) { - continue; - } - - if (FileHelper::IsDirectory(fullpath)) { - FileHelper::RemoveDirectory(fullpath); - } else { - FileHelper::DeleteFile(fullpath); - } - free(fullpath); - } - closedir(dir); - return (rmdir(path) == 0); -} - -bool FileHelper::IsExist(const char *path) { - return (access(path, F_OK) == 0); -} - -bool FileHelper::IsRegular(const char *path) { - struct stat buf; - if (stat(path, &buf) != 0) { - return false; - } - return ((buf.st_mode & S_IFREG) != 0); -} - -bool FileHelper::IsDirectory(const char *path) { - struct stat buf; - if (stat(path, &buf) != 0) { - return false; - } - return ((buf.st_mode & S_IFDIR) != 0); -} - -bool FileHelper::IsSymbolicLink(const char *path) { - struct stat buf; - if (stat(path, &buf) != 0) { - return false; - } - return ((buf.st_mode & S_IFLNK) != 0); -} - -bool FileHelper::IsSame(const char *path1, const char *path2) { - char real_path1[PATH_MAX]; - char real_path2[PATH_MAX]; - if (!realpath(path1, real_path1)) { - return false; - } - if (!realpath(path2, real_path2)) { - return false; - } - return (!strcmp(real_path1, real_path2)); -} - -std::string FileHelper::GetLastErrorString() { - return strerror(errno); -} - -#else -#undef RemoveDirectory -#undef DeleteFile -#undef GetFileSize - -static inline char *JoinFilePath(const char *prefix, const char *suffix) { - size_t prefix_len = strlen(prefix); - size_t suffix_len = strlen(suffix); - - char *path = (char *)malloc(prefix_len + suffix_len + 2); - if (path) { - memcpy(path, prefix, prefix_len); - memcpy(path + prefix_len + 1, suffix, suffix_len); - path[prefix_len] = '\\'; - path[prefix_len + suffix_len + 1] = '\0'; - } - return path; -} - -bool FileHelper::GetWorkingDirectory(std::string *path) { - char buf[MAX_PATH]; - DWORD len = GetCurrentDirectoryA(MAX_PATH, buf); - - if (len <= 0) { - return false; - } - path->assign(buf, len); - return true; -} - -bool FileHelper::GetFileSize(const char *path, size_t *psz) { - HANDLE handle = - CreateFileA(path, GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); - if (handle == INVALID_HANDLE_VALUE) { - return false; - } - - LARGE_INTEGER file_size; - if (!GetFileSizeEx(handle, &file_size)) { - CloseHandle(handle); - return false; - } - CloseHandle(handle); - *psz = (size_t)file_size.QuadPart; - return true; -} - -bool FileHelper::DeleteFile(const char *path) { - // Delete a file by the path - return (DeleteFileA(path)); -} - -bool FileHelper::RenameFile(const char *oldpath, const char *newpath) { - return (MoveFileA(oldpath, newpath)); -} - -bool FileHelper::MakePath(const char *path) { - char pathbuf[MAX_PATH]; - char *sp, *pp; - - strncpy(pathbuf, path, sizeof(pathbuf) - 1); - pathbuf[MAX_PATH - 1] = '\0'; - - pp = pathbuf; - while ((sp = strpbrk(pp, "/\\")) != nullptr) { - // Neither root nor double slash in path - if (sp != pp) { - *sp = '\0'; - // Skip Windows drive roots like "C:" — CreateDirectoryA on a bare drive - // letter returns ERROR_ACCESS_DENIED (not ERROR_ALREADY_EXISTS), which - // would cause MakePath to fail even when all parent dirs already exist. - bool is_drive_root = (sp - pathbuf == 2 && pathbuf[1] == ':'); - if (!is_drive_root && !CreateDirectoryA(pathbuf, nullptr) && - GetLastError() != ERROR_ALREADY_EXISTS) { - return false; - } - *sp = '\\'; - } - pp = sp + 1; - } - return !(*pp != '\0' && !CreateDirectoryA(pathbuf, nullptr) && - GetLastError() != ERROR_ALREADY_EXISTS); + return MakePathImpl(PathFromUtf8(path)); } bool FileHelper::RemoveDirectory(const char *path) { if (path == nullptr || *path == '\0') { return false; } - - if (!FileHelper::IsDirectory(path)) { - return false; - } - - std::error_code ec; - fs::remove_all(path, ec); - if (ec) { - return false; - } - return true; + return RemoveDirectoryImpl(PathFromUtf8(path)); } bool FileHelper::IsExist(const char *path) { - DWORD attr = GetFileAttributesA(path); - return (attr != INVALID_FILE_ATTRIBUTES); + return IsExistImpl(PathFromUtf8(path)); } bool FileHelper::IsRegular(const char *path) { - DWORD attr = GetFileAttributesA(path); - return (attr != INVALID_FILE_ATTRIBUTES && - !(attr & FILE_ATTRIBUTE_DIRECTORY)); + return IsRegularImpl(PathFromUtf8(path)); } bool FileHelper::IsDirectory(const char *path) { - DWORD attr = GetFileAttributesA(path); - return (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)); + return IsDirectoryImpl(PathFromUtf8(path)); } bool FileHelper::IsSymbolicLink(const char *path) { - DWORD attr = GetFileAttributesA(path); - return (attr != INVALID_FILE_ATTRIBUTES && - (attr & FILE_ATTRIBUTE_REPARSE_POINT)); + return IsSymbolicLinkImpl(PathFromUtf8(path)); +} + +bool FileHelper::IsSame(const char *path1, const char *path2) { + return IsSameImpl(PathFromUtf8(path1), PathFromUtf8(path2)); } std::string FileHelper::GetLastErrorString() { + if (g_last_fs_error) { + return g_last_fs_error.message(); + } +#if defined(_WIN32) || defined(_WIN64) DWORD err = GetLastError(); if (err == 0) { return "No error"; @@ -366,34 +382,17 @@ std::string FileHelper::GetLastErrorString() { FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, sizeof(buf), nullptr); if (len > 0) { - // Strip trailing newline that FormatMessage often appends while (len > 0 && (buf[len - 1] == '\r' || buf[len - 1] == '\n')) { buf[--len] = '\0'; } return std::string(buf, len); } return "Unknown error " + std::to_string(err); +#else + return strerror(errno); +#endif } -bool FileHelper::IsSame(const char *path1, const char *path2) { - char real_path1[MAX_PATH]; - char real_path2[MAX_PATH]; - char **part_path1 = nullptr; - char **part_path2 = nullptr; - DWORD path1_size = - GetFullPathNameA(path1, sizeof(real_path1), real_path1, part_path1); - DWORD path2_size = - GetFullPathNameA(path2, sizeof(real_path2), real_path2, part_path2); - - if ((part_path1 && *part_path1 != 0) || (part_path2 && *part_path2 != 0) || - (path1_size != path2_size)) { - return false; - } - return (!strcmp(real_path1, real_path2)); -} - -#endif // !_WIN32 && !_WIN64 - bool FileHelper::RemovePath(const char *path) { if (FileHelper::IsDirectory(path)) { return FileHelper::RemoveDirectory(path); @@ -402,4 +401,4 @@ bool FileHelper::RemovePath(const char *path) { } } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/db/collection.cc b/src/db/collection.cc index 8673a92..4e9fa22 100644 --- a/src/db/collection.cc +++ b/src/db/collection.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -1875,7 +1875,7 @@ Status CollectionImpl::init_writing_segment() { } Status CollectionImpl::acquire_file_lock(bool create) { - std::string lock_file_path = ailego::StringHelper::Concat(path_, "/", "LOCK"); + std::string lock_file_path = ailego::FileHelper::PathJoin(path_, "LOCK"); if (create) { if (!lock_file_.create(lock_file_path.c_str(), 0)) { diff --git a/src/db/common/file_helper.cc b/src/db/common/file_helper.cc index c821316..3302e9a 100644 --- a/src/db/common/file_helper.cc +++ b/src/db/common/file_helper.cc @@ -13,20 +13,12 @@ // limitations under the License. #include "file_helper.h" -#include -#include #include #include -#ifdef _MSC_VER +#include #include -#include -#else -#include -#include -#include -#include -#endif -#include +#include + namespace zvec { @@ -37,132 +29,58 @@ const std::string FileHelper::RECOVER_SUFFIX = ".recovering"; bool FileHelper::CopyFile(const std::string &src_file_path, const std::string &dst_file_path) { -#ifdef _MSC_VER std::string dst_file_path_tmp = dst_file_path + ".tmp"; std::error_code ec; - std::filesystem::copy_file(src_file_path, dst_file_path_tmp, - std::filesystem::copy_options::overwrite_existing, - ec); + std::filesystem::copy_file( + ailego::FileHelper::PathFromUtf8(src_file_path), + ailego::FileHelper::PathFromUtf8(dst_file_path_tmp), + std::filesystem::copy_options::overwrite_existing, ec); if (ec) { return false; } - std::filesystem::rename(dst_file_path_tmp, dst_file_path, ec); + std::filesystem::rename(ailego::FileHelper::PathFromUtf8(dst_file_path_tmp), + ailego::FileHelper::PathFromUtf8(dst_file_path), ec); return !ec; -#else - int src_fd = open(src_file_path.c_str(), O_RDONLY, 0); - if (src_fd < 0) { - return false; - } - AILEGO_DEFER([src_fd] { close(src_fd); }); - - std::string dst_file_path_tmp = dst_file_path + ".tmp"; - int dst_fd = - open(dst_file_path_tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (dst_fd < 0) { - return false; - } - AILEGO_DEFER([dst_fd] { close(dst_fd); }); - - ssize_t size; - char buf[BUFSIZ]; - while ((size = read(src_fd, buf, BUFSIZ)) > 0) { - if (size != write(dst_fd, buf, size)) { - return false; - } - } - return rename(dst_file_path_tmp.c_str(), dst_file_path.c_str()) == 0; -#endif } bool FileHelper::CopyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path) { -#ifdef _MSC_VER std::error_code ec; - std::filesystem::copy(src_dir_path, dst_dir_path, + std::filesystem::copy(ailego::FileHelper::PathFromUtf8(src_dir_path), + ailego::FileHelper::PathFromUtf8(dst_dir_path), std::filesystem::copy_options::recursive | std::filesystem::copy_options::overwrite_existing, ec); return !ec; -#else - DIR *dir = opendir(src_dir_path.c_str()); - if (!dir) { - return false; - } - AILEGO_DEFER([dir] { closedir(dir); }); - - if (!ailego::FileHelper::IsExist(dst_dir_path.c_str())) { - if (!ailego::FileHelper::MakePath(dst_dir_path.c_str())) { - return false; - } - } - - struct dirent *dent; - while ((dent = readdir(dir)) != nullptr) { - if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) { - continue; - } - std::string src_full_path = - ailego::StringHelper::Concat(src_dir_path, "/", dent->d_name); - std::string dst_full_path = - ailego::StringHelper::Concat(dst_dir_path, "/", dent->d_name); - - if (ailego::FileHelper::IsDirectory(src_full_path.c_str())) { - if (!CopyDirectory(src_full_path, dst_full_path)) { - return false; - } - } else { - if (!CopyFile(src_full_path, dst_full_path)) { - return false; - } - } - } - return true; -#endif } void FileHelper::CleanupDirectory(const std::string &backup_dir, size_t max_backup_count, const char *prefix_name) { - if (max_backup_count <= 0) { + if (max_backup_count == 0) { return; } - -#ifdef _MSC_VER size_t prefix_len = strlen(prefix_name); std::vector candidates; std::error_code ec; - for (const auto &entry : - std::filesystem::directory_iterator(backup_dir, ec)) { - std::string name = entry.path().filename().string(); + for (const auto &entry : std::filesystem::directory_iterator( + ailego::FileHelper::PathFromUtf8(backup_dir), ec)) { + std::string name = entry.path().filename().u8string(); if (name.compare(0, prefix_len, prefix_name) == 0) { candidates.emplace_back(name); } } -#else - DIR *dir = opendir(backup_dir.c_str()); - if (!dir) { + if (ec) { return; } - - AILEGO_DEFER([dir] { closedir(dir); }); - - size_t prefix_len = strlen(prefix_name); - std::vector candidates; - struct dirent *dent; - while ((dent = readdir(dir)) != nullptr) { - if (strncmp(dent->d_name, prefix_name, prefix_len) == 0) { - candidates.emplace_back(dent->d_name); - } - } -#endif if (candidates.size() <= max_backup_count) { return; } std::sort(candidates.begin(), candidates.end()); for (size_t i = 0; i < candidates.size() - max_backup_count; ++i) { - std::string path = - ailego::StringHelper::Concat(backup_dir, "/", candidates[i].c_str()); - ailego::FileHelper::RemovePath(path.c_str()); + std::string path_str = + ailego::FileHelper::PathJoin(backup_dir, candidates[i]); + ailego::FileHelper::RemovePath(path_str.c_str()); } } diff --git a/src/db/common/file_helper.h b/src/db/common/file_helper.h index 95d5b2a..065c80b 100644 --- a/src/db/common/file_helper.h +++ b/src/db/common/file_helper.h @@ -73,16 +73,17 @@ class FileHelper { public: static const std::string MakeWalPath(const std::string &path, uint32_t seg_id, uint32_t block_id) { - return ailego::StringHelper::Concat(path, "/", seg_id, "/", block_id, - ".wal"); + return ailego::FileHelper::PathJoin( + path, seg_id, ailego::StringHelper::Concat(block_id, ".wal")); } static std::string MakeSegmentPath(const std::string &path, uint32_t id, const std::string &suffix = "") { if (suffix.empty()) { - return ailego::StringHelper::Concat(path, "/", id); + return ailego::FileHelper::PathJoin(path, id); } - return ailego::StringHelper::Concat(path, "/", id, ".", suffix); + return ailego::FileHelper::PathJoin( + path, ailego::StringHelper::Concat(id, ".", suffix)); } static std::string MakeTempSegmentPath(const std::string &path, uint32_t id) { @@ -94,99 +95,105 @@ class FileHelper { uint32_t seg_id, uint32_t block_id, bool use_parquet = false) { - return use_parquet ? MakeForwardBlockPath(path, seg_id, block_id, - std::string("parquet")) - : MakeForwardBlockPath(path, seg_id, block_id, - std::string("ipc")); + return MakeForwardBlockPath(path, seg_id, block_id, + std::string(use_parquet ? "parquet" : "ipc")); } static const std::string MakeForwardBlockPath(const std::string &path, uint32_t seg_id, uint32_t block_id, const std::string &suffix) { - return ailego::StringHelper::Concat(path, "/", seg_id, "/scalar.", block_id, - ".", suffix); + return ailego::FileHelper::PathJoin( + path, seg_id, + ailego::StringHelper::Concat("scalar.", block_id, ".", suffix)); } static const std::string MakeForwardBlockPath(const std::string &seg_path, uint32_t block_id, bool use_parquet = false) { - return use_parquet ? ailego::StringHelper::Concat(seg_path, "/scalar.", - block_id, ".parquet") - : ailego::StringHelper::Concat(seg_path, "/scalar.", - block_id, ".ipc"); + return MakeForwardBlockPath(seg_path, block_id, + std::string(use_parquet ? "parquet" : "ipc")); } static const std::string MakeForwardBlockPath(const std::string &seg_path, uint32_t block_id, const std::string &suffix) { - return ailego::StringHelper::Concat(seg_path, "/scalar.", block_id, ".", - suffix); + return ailego::FileHelper::PathJoin( + seg_path, + ailego::StringHelper::Concat("scalar.", block_id, ".", suffix)); } // e.g.: **/seg1/scalar.index.block.1.rocksdb static const std::string MakeInvertIndexPath(const std::string &path, uint32_t seg_id, uint32_t block_id) { - return ailego::StringHelper::Concat(path, "/", seg_id, "/scalar.index.", - block_id, ".rocksdb"); + return ailego::FileHelper::PathJoin( + path, seg_id, + ailego::StringHelper::Concat("scalar.index.", block_id, ".rocksdb")); } static const std::string MakeInvertIndexPath(const std::string &seg_path, uint32_t block_id) { - return ailego::StringHelper::Concat(seg_path, "/scalar.index.", block_id, - ".rocksdb"); + return ailego::FileHelper::PathJoin( + seg_path, + ailego::StringHelper::Concat("scalar.index.", block_id, ".rocksdb")); } static const std::string MakeVectorIndexPath(const std::string &path, const std::string &column, uint32_t seg_id, uint32_t block_id) { - return ailego::StringHelper::Concat(path, "/", seg_id, "/", column, - ".index.", block_id, ".proxima"); + return ailego::FileHelper::PathJoin( + path, seg_id, + ailego::StringHelper::Concat(column, ".index.", block_id, ".proxima")); } static const std::string MakeVectorIndexPath(const std::string &seg_path, const std::string &column, uint32_t block_id) { - return ailego::StringHelper::Concat(seg_path, "/", column, ".index.", - block_id, ".proxima"); + return ailego::FileHelper::PathJoin( + seg_path, + ailego::StringHelper::Concat(column, ".index.", block_id, ".proxima")); } // e.g.: **/{seg_id}/{column}.index.block.{block_id}.proxima static const std::string MakeQuantizeVectorIndexPath( const std::string &path, const std::string &column, uint32_t seg_id, uint32_t block_id) { - return ailego::StringHelper::Concat(path, "/", seg_id, "/", column, - ".qindex.", block_id, ".proxima"); + return ailego::FileHelper::PathJoin( + path, seg_id, + ailego::StringHelper::Concat(column, ".qindex.", block_id, ".proxima")); } static const std::string MakeQuantizeVectorIndexPath( const std::string &seg_path, const std::string &column, uint32_t block_id) { - return ailego::StringHelper::Concat(seg_path, "/", column, ".qindex.", - block_id, ".proxima"); + return ailego::FileHelper::PathJoin( + seg_path, + ailego::StringHelper::Concat(column, ".qindex.", block_id, ".proxima")); } //! Make file path with ${prefix_path}/${file_name} static std::string MakeFilePath(const std::string &prefix_path, FileID file_id) { - return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id)); + return ailego::FileHelper::PathJoin(prefix_path, GetFileName(file_id)); } //! Make file path with ${prefix_path}/${file_name}.${number} static std::string MakeFilePath(const std::string &prefix_path, FileID file_id, uint32_t number) { - return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id), - ".", number); + return ailego::FileHelper::PathJoin( + prefix_path, + ailego::StringHelper::Concat(GetFileName(file_id), ".", number)); } //! Make file path with ${prefix_path}/${file_name}.${suffix_name}.${number} static std::string MakeFilePath(const std::string &prefix_path, FileID file_id, uint32_t number, const std::string &suffix_name) { - return ailego::StringHelper::Concat(prefix_path, "/", GetFileName(file_id), - ".", suffix_name, ".", number); + return ailego::FileHelper::PathJoin( + prefix_path, ailego::StringHelper::Concat(GetFileName(file_id), ".", + suffix_name, ".", number)); } //! Create directory @@ -273,4 +280,4 @@ class FileHelper { }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/db/common/logger.h b/src/db/common/logger.h index d0f7eb0..d4f4da1 100644 --- a/src/db/common/logger.h +++ b/src/db/common/logger.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "db/common/constants.h" #include "error_code.h" @@ -50,7 +51,8 @@ class LogUtil { if (logger_type == FILE_LOG_TYPE_NAME) { params.set("proxima.file.logger.log_dir", log_dir); params.set("proxima.file.logger.log_file", log_file); - params.set("proxima.file.logger.path", log_dir + "/" + log_file); + params.set("proxima.file.logger.path", + ailego::FileHelper::PathJoin(log_dir, log_file)); std::string program_name = ailego::File::BaseName(gflags::GetArgv0()); params.set("proxima.program.program_name", program_name); params.set("proxima.file.logger.file_size", log_file_size); diff --git a/src/db/index/common/version_manager.cc b/src/db/index/common/version_manager.cc index a0fd42d..1ca095d 100644 --- a/src/db/index/common/version_manager.cc +++ b/src/db/index/common/version_manager.cc @@ -34,7 +34,8 @@ namespace zvec { Status Version::Load(const std::string &path, Version *version) { - std::ifstream ifs(path, std::ios::binary); + std::ifstream ifs; + ailego::FileHelper::OpenIfstream(ifs, path, std::ios::binary); if (!ifs.is_open()) { LOG_ERROR("Failed to open file: %s", path.c_str()); return Status::InternalError("Failed to open file"); @@ -74,7 +75,8 @@ Status Version::Load(const std::string &path, Version *version) { } Status Version::Save(const std::string &path, const Version &version) { - std::ofstream ofs(path, std::ios::binary); + std::ofstream ofs; + ailego::FileHelper::OpenOfstream(ofs, path, std::ios::binary); if (!ofs.is_open()) { LOG_ERROR("Failed to open file: %s, err: %s", path.c_str(), ailego::FileHelper::GetLastErrorString().c_str()); @@ -187,12 +189,13 @@ std::string Version::to_string_formatted(int indent_level) const { Result VersionManager::Recovery(const std::string &path) { namespace fs = std::filesystem; - if (!fs::exists(path)) { + auto u8path = ailego::FileHelper::PathFromUtf8(path); + if (!fs::exists(u8path)) { LOG_ERROR("VersionManager::Recovery: path %s does not exist", path.c_str()); return tl::make_unexpected( Status::NotFound("path ", path, " does not exist")); } - if (!fs::is_directory(path)) { + if (!fs::is_directory(u8path)) { LOG_ERROR("VersionManager::Recovery: path %s is not a directory", path.c_str()); return tl::make_unexpected( @@ -207,14 +210,14 @@ Result VersionManager::Recovery(const std::string &path) { uint64_t max_id = UINT64_MAX; std::string version_path; - for (const auto &entry : fs::directory_iterator(path)) { + for (const auto &entry : fs::directory_iterator(u8path)) { if (entry.is_regular_file()) { - std::string filename = entry.path().filename().string(); + std::string filename = entry.path().filename().u8string(); if (std::regex_match(filename, match, regex)) { uint64_t id = std::stoull(match[1].str()); if (id > max_id || max_id == UINT64_MAX) { max_id = id; - version_path = entry.path().string(); + version_path = entry.path().u8string(); } } } diff --git a/src/db/index/common/version_manager.h b/src/db/index/common/version_manager.h index bed0a72..e43bd85 100644 --- a/src/db/index/common/version_manager.h +++ b/src/db/index/common/version_manager.h @@ -164,7 +164,7 @@ class Version { private: CollectionSchema::Ptr schema_; - bool enable_mmap_; + bool enable_mmap_{false}; std::unordered_map persisted_segment_metas_map_; diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index 138941a..f930e42 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -40,7 +40,8 @@ namespace zvec { inline FileFormat InferFileFormat(const std::string &file_path) { - std::string ext = std::filesystem::path(file_path).extension().string(); + std::string ext = + ailego::FileHelper::PathFromUtf8(file_path).extension().u8string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); if (ext == ".parquet") { return FileFormat::PARQUET; diff --git a/src/db/sqlengine/parser/zvec_parser.cc b/src/db/sqlengine/parser/zvec_parser.cc index cedb082..d56253f 100644 --- a/src/db/sqlengine/parser/zvec_parser.cc +++ b/src/db/sqlengine/parser/zvec_parser.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "db/sqlengine/common/util.h" #include "tree/ParseTree.h" @@ -118,7 +119,8 @@ std::string ZVecParser::to_formatted_string_tree(void *tree, void *parser) { void ZVecParser::save_to_file(const std::string &file_name, const std::string &formatted) { - std::ofstream outfile(file_name); + std::ofstream outfile; + ailego::FileHelper::OpenOfstream(outfile, file_name); outfile << formatted; outfile << std::endl; outfile.close(); diff --git a/src/include/zvec/ailego/utility/file_helper.h b/src/include/zvec/ailego/utility/file_helper.h index 0c481c1..c607cb4 100644 --- a/src/include/zvec/ailego/utility/file_helper.h +++ b/src/include/zvec/ailego/utility/file_helper.h @@ -15,8 +15,11 @@ #pragma once #include +#include +#include #include #include +#include namespace zvec { namespace ailego { @@ -41,10 +44,11 @@ struct FileHelper { //! Retrieve the final path for the specified file static bool GetFilePath(NativeHandle handle, std::string *path); - //! Retrieve current working directory + //! Retrieve current working directory (UTF-8 bytes in \p *path) static bool GetWorkingDirectory(std::string *path); - //! Get the size of a file + //! Narrow paths are UTF-8 on all platforms (on Windows, decoded as UTF-8; + //! on POSIX, native narrow encoding is typically UTF-8). static bool GetFileSize(const char *path, size_t *psz); //! Delete a name and possibly the file it refers to @@ -77,8 +81,8 @@ struct FileHelper { //! Retrieve non-zero if two paths are pointing to the same file static bool IsSame(const char *path1, const char *path2); - //! Retrieve a human-readable string for the most recent OS error - //! (GetLastError() on Windows, strerror(errno) on POSIX) + //! Human-readable error: last std::filesystem failure on this thread if any, + //! else GetLastError() on Windows or strerror(errno) on POSIX. static std::string GetLastErrorString(); //! Retrieve the size of a file @@ -96,6 +100,54 @@ struct FileHelper { } return (output ? output + 1 : path); } + + //! Build std::filesystem::path from a UTF-8 string (handles Windows codepage) + static std::filesystem::path PathFromUtf8(const std::string &s); + static std::filesystem::path PathFromUtf8(const char *s); + + //! Convert std::filesystem::path back to a UTF-8 std::string + static std::string PathToUtf8(const std::filesystem::path &p); + + //! Concatenate path segments with the native separator. + //! Accepts strings, c-strings, and numbers (anything Alphameric accepts). + //! E.g. PathJoin(dir, seg_id, "file.ext") + static std::string PathJoin(const internal::Alphameric &a, + const internal::Alphameric &b) { + return StringHelper::Concat(a, kPathSep, b); + } + template + static std::string PathJoin(const internal::Alphameric &a, + const internal::Alphameric &b, + const Rest &...rest) { + return PathJoin(StringHelper::Concat(a, kPathSep, b), rest...); + } + + private: +#if defined(_WIN32) || defined(_WIN64) + static constexpr const char *kPathSep = "\\"; +#else + static constexpr const char *kPathSep = "/"; +#endif + public: +#if defined(_WIN32) || defined(_WIN64) + //! UTF-8 narrow string -> wide string (Win32 API ready) + static std::wstring Utf8ToWide(const std::string &utf8); + + //! Wide string -> UTF-8 narrow string + static std::string WideToUtf8(const std::wstring &ws); +#endif + + //! Open a std::ifstream from a UTF-8 path + static void OpenIfstream(std::ifstream &ifs, const std::string &path, + std::ios_base::openmode mode = std::ios_base::in) { + ifs.open(PathFromUtf8(path), mode); + } + + //! Open a std::ofstream from a UTF-8 path + static void OpenOfstream(std::ofstream &ofs, const std::string &path, + std::ios_base::openmode mode = std::ios_base::out) { + ofs.open(PathFromUtf8(path), mode); + } }; } // namespace ailego diff --git a/src/include/zvec/core/framework/index_bundle.h b/src/include/zvec/core/framework/index_bundle.h index ea08959..96cd946 100644 --- a/src/include/zvec/core/framework/index_bundle.h +++ b/src/include/zvec/core/framework/index_bundle.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace zvec { namespace core { @@ -210,7 +211,7 @@ class MMapFileIndexBundle : public IndexBundle { //! Create a memory mapping file in bundle bool create(const std::string &prefix, const std::string &key, size_t len) { ailego::MMapFile file; - if (!file.create(prefix + '/' + key, len)) { + if (!file.create(ailego::FileHelper::PathJoin(prefix, key), len)) { return false; } map_[key] = std::move(file); @@ -220,7 +221,7 @@ class MMapFileIndexBundle : public IndexBundle { //! Create a memory mapping file in bundle bool create(const std::string &prefix, std::string &&key, size_t len) { ailego::MMapFile file; - if (!file.create(prefix + '/' + key, len)) { + if (!file.create(ailego::FileHelper::PathJoin(prefix, key), len)) { return false; } map_[std::move(key)] = std::move(file); @@ -240,7 +241,7 @@ class MMapFileIndexBundle : public IndexBundle { //! Open a memory mapping file in bundle bool open(const std::string &prefix, const std::string &key, bool rdonly) { ailego::MMapFile file; - if (!file.open(prefix + '/' + key, rdonly)) { + if (!file.open(ailego::FileHelper::PathJoin(prefix, key), rdonly)) { return false; } map_[key] = std::move(file); @@ -250,7 +251,7 @@ class MMapFileIndexBundle : public IndexBundle { //! Open a memory mapping file in bundle bool open(const std::string &prefix, std::string &&key, bool rdonly) { ailego::MMapFile file; - if (!file.open(prefix + '/' + key, rdonly)) { + if (!file.open(ailego::FileHelper::PathJoin(prefix, key), rdonly)) { return false; } map_[std::move(key)] = std::move(file); diff --git a/tests/ailego/io/file_test.cc b/tests/ailego/io/file_test.cc index 84b916c..308f987 100644 --- a/tests/ailego/io/file_test.cc +++ b/tests/ailego/io/file_test.cc @@ -45,7 +45,9 @@ TEST(File, General) { } TEST(File, MakePath) { - EXPECT_TRUE(File::MakePath("")); + EXPECT_FALSE(File::MakePath("")); + std::cout << FileHelper::GetLastErrorString() << std::endl; + EXPECT_TRUE(File::MakePath(".")); EXPECT_TRUE(File::MakePath("..")); EXPECT_TRUE(File::MakePath("../")); diff --git a/tests/ailego/io/utf8_path_test.cc b/tests/ailego/io/utf8_path_test.cc new file mode 100644 index 0000000..aa5c50e --- /dev/null +++ b/tests/ailego/io/utf8_path_test.cc @@ -0,0 +1,287 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(_WIN32) || defined(_WIN64) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#undef DeleteFile +#undef RemoveDirectory +#undef CreateFile +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace zvec::ailego; + +// UTF-8 encoded Chinese / Japanese test strings. +// These contain characters outside the Windows ANSI codepage, so any path +// handling that falls back to narrow Win32 APIs will produce the mojibake. +static const std::string kChinese = + "\xe4\xb8\xad\xe6\x96\x87\xe8\xb7\xaf\xe5\xbe\x84"; // 中文路径 +static const std::string kJapanese = + "\xe3\x83\x86\xe3\x82\xb9\xe3\x83\x88"; // テスト +static const std::string kMixed = "data_测试_2025"; // data_测试_2025 + +class Utf8PathTest : public ::testing::Test { + protected: + void TearDown() override { + for (auto &p : cleanup_) { + FileHelper::RemovePath(p.c_str()); + } + } + void ScheduleCleanup(const std::string &path) { + cleanup_.push_back(path); + } + + private: + std::vector cleanup_; +}; + +// --------------------------------------------------------------------------- +// 1. Directory create / query / remove with Chinese path +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, MakePath_Chinese) { + std::string dir = "utf8_test_" + kChinese; + ScheduleCleanup(dir); + + ASSERT_TRUE(FileHelper::MakePath(dir.c_str())); + EXPECT_TRUE(FileHelper::IsExist(dir.c_str())); + EXPECT_TRUE(FileHelper::IsDirectory(dir.c_str())); + EXPECT_FALSE(FileHelper::IsRegular(dir.c_str())); + + ASSERT_TRUE(FileHelper::RemoveDirectory(dir.c_str())); + EXPECT_FALSE(FileHelper::IsExist(dir.c_str())); +} + +// --------------------------------------------------------------------------- +// 2. Nested directories with mixed UTF-8 segments +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, MakePath_Nested) { + std::string root = "utf8_test_nested_" + kChinese; + std::string nested = FileHelper::PathJoin(root, kJapanese, kMixed); + ScheduleCleanup(root); + + ASSERT_TRUE(FileHelper::MakePath(nested.c_str())); + EXPECT_TRUE(FileHelper::IsDirectory(nested.c_str())); + + ASSERT_TRUE(FileHelper::RemoveDirectory(root.c_str())); + EXPECT_FALSE(FileHelper::IsExist(root.c_str())); +} + +// --------------------------------------------------------------------------- +// 3. File create / read / write through ailego::File with UTF-8 path +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, FileCreateAndReadWrite) { + std::string dir = "utf8_test_file_" + kChinese; + std::string file_path = FileHelper::PathJoin(dir, kJapanese + ".dat"); + ScheduleCleanup(dir); + + ASSERT_TRUE(FileHelper::MakePath(dir.c_str())); + + const std::string payload = + "hello_utf8_\xe4\xbd\xa0\xe5\xa5\xbd"; // hello_utf8_你好 + { + File f; + ASSERT_TRUE(f.create(file_path.c_str(), 0)); + EXPECT_EQ(payload.size(), f.write(payload.data(), payload.size())); + EXPECT_TRUE(f.flush()); + } + + EXPECT_TRUE(FileHelper::IsRegular(file_path.c_str())); + + size_t sz = 0; + ASSERT_TRUE(FileHelper::GetFileSize(file_path.c_str(), &sz)); + EXPECT_EQ(payload.size(), sz); + + { + File f; + ASSERT_TRUE(f.open(file_path.c_str(), true)); + std::string buf(sz, '\0'); + EXPECT_EQ(sz, f.read(buf.data(), sz)); + EXPECT_EQ(payload, buf); + } + + ASSERT_TRUE(FileHelper::DeleteFile(file_path.c_str())); + EXPECT_FALSE(FileHelper::IsExist(file_path.c_str())); +} + +// --------------------------------------------------------------------------- +// 4. FileHelper::RenameFile across UTF-8 paths +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, RenameFile) { + std::string dir = "utf8_test_rename_" + kChinese; + std::string src = FileHelper::PathJoin(dir, "src_" + kJapanese); + std::string dst = FileHelper::PathJoin(dir, "dst_" + kMixed); + ScheduleCleanup(dir); + + ASSERT_TRUE(FileHelper::MakePath(dir.c_str())); + { + File f; + ASSERT_TRUE(f.create(src.c_str(), 0)); + } + EXPECT_TRUE(FileHelper::IsExist(src.c_str())); + + ASSERT_TRUE(FileHelper::RenameFile(src.c_str(), dst.c_str())); + EXPECT_FALSE(FileHelper::IsExist(src.c_str())); + EXPECT_TRUE(FileHelper::IsExist(dst.c_str())); +} + +// --------------------------------------------------------------------------- +// 5. OpenIfstream / OpenOfstream with UTF-8 path +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, FstreamUtf8) { + std::string dir = "utf8_test_fstream_" + kChinese; + std::string file_path = FileHelper::PathJoin(dir, kMixed + ".txt"); + ScheduleCleanup(dir); + + ASSERT_TRUE(FileHelper::MakePath(dir.c_str())); + + const std::string content = + "line1\nline2_\xe6\xb5\x8b\xe8\xaf\x95\n"; // line2_测试 + { + std::ofstream ofs; + FileHelper::OpenOfstream(ofs, file_path, std::ios::binary); + ASSERT_TRUE(ofs.is_open()); + ofs.write(content.data(), content.size()); + } + + { + std::ifstream ifs; + FileHelper::OpenIfstream(ifs, file_path, std::ios::binary); + ASSERT_TRUE(ifs.is_open()); + std::string buf((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + EXPECT_EQ(content, buf); + } +} + +// --------------------------------------------------------------------------- +// 6. PathJoin produces correct separators +// --------------------------------------------------------------------------- +TEST_F(Utf8PathTest, PathJoinSeparator) { + std::string result = FileHelper::PathJoin("root", "sub", "file.txt"); +#if defined(_WIN32) || defined(_WIN64) + EXPECT_EQ("root\\sub\\file.txt", result); +#else + EXPECT_EQ("root/sub/file.txt", result); +#endif +} + +// --------------------------------------------------------------------------- +// 7. Verify NO mojibake on actual filesystem via "dir" (Windows) / "ls" (POSIX) +// We create a directory + file with Chinese names, then list the parent +// using the OS shell and check the expected name appears verbatim. +// --------------------------------------------------------------------------- +#if defined(_WIN32) || defined(_WIN64) +TEST_F(Utf8PathTest, NoGarbledNames_FindFirstFile) { + std::string parent = "utf8_test_dir_verify"; + std::string child_dir = kChinese; + std::string child_file = kJapanese + ".dat"; + std::string full_dir = FileHelper::PathJoin(parent, child_dir); + std::string full_file = FileHelper::PathJoin(parent, child_file); + ScheduleCleanup(parent); + + ASSERT_TRUE(FileHelper::MakePath(full_dir.c_str())); + { + File f; + ASSERT_TRUE(f.create(full_file.c_str(), 0)); + } + + // Use FindFirstFileW / FindNextFileW to enumerate the parent directory. + // This is the most reliable way to verify the OS actually stored correct + // Unicode names (no mojibake), bypassing any console codepage issues. + std::wstring pattern = FileHelper::Utf8ToWide(parent) + L"\\*"; + WIN32_FIND_DATAW fd; + HANDLE hFind = FindFirstFileW(pattern.c_str(), &fd); + ASSERT_NE(INVALID_HANDLE_VALUE, hFind) << "FindFirstFileW failed"; + + std::vector entries; + do { + std::wstring name(fd.cFileName); + if (name != L"." && name != L"..") { + entries.push_back(name); + } + } while (FindNextFileW(hFind, &fd)); + FindClose(hFind); + + std::wstring expected_dir = FileHelper::Utf8ToWide(child_dir); + std::wstring expected_file = FileHelper::Utf8ToWide(child_file); + + bool found_dir = false, found_file = false; + std::cout << "[NoGarbledNames] Entries in " << parent << ":" << std::endl; + for (const auto &e : entries) { + std::string utf8_name = FileHelper::WideToUtf8(e); + std::cout << " " << utf8_name << std::endl; + if (e == expected_dir) found_dir = true; + if (e == expected_file) found_file = true; + } + + EXPECT_TRUE(found_dir) << "Directory '" << child_dir + << "' not found (garbled Unicode?)"; + EXPECT_TRUE(found_file) << "File '" << (kJapanese + ".dat") + << "' not found (garbled Unicode?)"; +} + +TEST_F(Utf8PathTest, NoGarbledNames_DirCmd) { + std::string parent = "utf8_test_dir_cmd"; + std::string child = kChinese; + std::string full = FileHelper::PathJoin(parent, child); + std::string tmpfile = "utf8_test_dir_cmd_output.txt"; + ScheduleCleanup(parent); + + ASSERT_TRUE(FileHelper::MakePath(full.c_str())); + + // "cmd /u" makes built-in commands output UTF-16LE when redirected to file. + std::wstring wparent = FileHelper::Utf8ToWide(parent); + std::wstring wtmp = FileHelper::Utf8ToWide(tmpfile); + std::wstring cmd = + L"cmd /u /c dir /b \"" + wparent + L"\" >\"" + wtmp + L"\""; + _wsystem(cmd.c_str()); + + // Read back the file as raw bytes (UTF-16LE) + std::ifstream ifs; + FileHelper::OpenIfstream(ifs, tmpfile, std::ios::binary); + ASSERT_TRUE(ifs.is_open()) << "Could not read dir output file"; + std::string raw((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + ifs.close(); + FileHelper::DeleteFile(tmpfile.c_str()); + + // Interpret as UTF-16LE: skip BOM if present, then convert to UTF-8 + const wchar_t *wdata = reinterpret_cast(raw.data()); + size_t wlen = raw.size() / sizeof(wchar_t); + if (wlen > 0 && wdata[0] == L'\xFEFF') { + wdata++; + wlen--; + } + std::wstring woutput(wdata, wlen); + std::string output = FileHelper::WideToUtf8(woutput); + + std::cout << "[NoGarbledNames_DirCmd] dir output: [" << output << "]"; + + EXPECT_NE(std::string::npos, output.find(child)) + << "Chinese directory name not found in 'dir' output (garbled?).\n" + << "Got: [" << output << "]"; +} +#endif diff --git a/tests/db/collection_test.cc b/tests/db/collection_test.cc index d50701d..6053ad0 100644 --- a/tests/db/collection_test.cc +++ b/tests/db/collection_test.cc @@ -216,7 +216,7 @@ TEST_F(CollectionTest, Feature_CreateAndOpen_PathValidate) { auto schema = TestHelper::CreateNormalSchema(); { - std::vector valid_paths = {"abc", + std::vector valid_paths = {"你好", "data123", "my_collection", "v1.2_alpha-beta", @@ -234,6 +234,8 @@ TEST_F(CollectionTest, Feature_CreateAndOpen_PathValidate) { auto result = Collection::CreateAndOpen(path, *schema, options); if (!result.has_value()) { std::cout << result.error().message() << std::endl; + std::cout << "File error:" << ailego::FileHelper::GetLastErrorString() + << std::endl; } ASSERT_TRUE(result.has_value()); diff --git a/tests/db/crash_recovery/collection_optimizer.cc b/tests/db/crash_recovery/collection_optimizer.cc index ea9baa6..639927b 100644 --- a/tests/db/crash_recovery/collection_optimizer.cc +++ b/tests/db/crash_recovery/collection_optimizer.cc @@ -85,8 +85,8 @@ int main(int argc, char **argv) { // Scope 'result' so its shared_ptr is released before we call _exit(). zvec::Collection::Ptr collection; { - auto result = - zvec::Collection::Open(config.path, zvec::CollectionOptions{false, true}); + auto result = zvec::Collection::Open(config.path, + zvec::CollectionOptions{false, true}); if (!result) { LOG_ERROR("Failed to open collection[%s]: %s", config.path.c_str(), result.error().c_str()); diff --git a/tests/db/index/common/version_manager_test.cc b/tests/db/index/common/version_manager_test.cc index 7a16a0c..08cdf4e 100644 --- a/tests/db/index/common/version_manager_test.cc +++ b/tests/db/index/common/version_manager_test.cc @@ -27,9 +27,10 @@ class VersionManagerTest : public ::testing::Test { protected: void SetUp() override { // Create a temporary directory for testing - test_path_ = "./version_manager_test"; + test_path_ = "./test_version_manager"; FileHelper::RemoveDirectory(test_path_); - FileHelper::CreateDirectory(test_path_); + ASSERT_TRUE(FileHelper::CreateDirectory(test_path_)) + << ailego::FileHelper::GetLastErrorString(); } void TearDown() override { @@ -131,7 +132,7 @@ TEST_F(VersionManagerTest, VersionManagerCreateAndRecover) { // Create VersionManager auto create_result = VersionManager::Create(version_path, initial_version); - EXPECT_TRUE(create_result.has_value()); + ASSERT_TRUE(create_result.has_value()); auto version_manager = create_result.value(); @@ -149,7 +150,7 @@ TEST_F(VersionManagerTest, VersionManagerCreateAndRecover) { // Recover VersionManager auto recover_result = VersionManager::Recovery(version_path); - EXPECT_TRUE(recover_result.has_value()); + ASSERT_TRUE(recover_result.has_value()); auto recovered_manager = recover_result.value(); auto recovered_version = recovered_manager->get_current_version(); diff --git a/tests/db/utf8_collection_test.cc b/tests/db/utf8_collection_test.cc new file mode 100644 index 0000000..4c0e5d1 --- /dev/null +++ b/tests/db/utf8_collection_test.cc @@ -0,0 +1,211 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "db/common/file_helper.h" +#include "index/utils/utils.h" +#include "zvec/db/collection.h" +#include "zvec/db/doc.h" +#include "zvec/db/options.h" +#include "zvec/db/schema.h" +#include "zvec/db/status.h" + +#if defined(_WIN32) || defined(_WIN64) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#undef DeleteFile +#undef RemoveDirectory +#undef CreateFile +#undef DELETE +#endif + +using namespace zvec; +using namespace zvec::test; + +// UTF-8 test path containing Chinese characters. +// If the path handling is broken, the OS will show garbled names (mojibake). +static const std::string kUtf8Dir = + "utf8_col_\xe4\xb8\xad\xe6\x96\x87\xe6\xb5\x8b\xe8\xaf\x95"; // utf8_col_中文测试 + +class Utf8CollectionTest : public ::testing::Test { + protected: + void SetUp() override { + ailego::FileHelper::RemovePath(kUtf8Dir.c_str()); + } + void TearDown() override { + ailego::FileHelper::RemovePath(kUtf8Dir.c_str()); + } +}; + +// --------------------------------------------------------------------------- +// 1. Create a collection in a UTF-8 path, insert docs, flush, reopen +// --------------------------------------------------------------------------- +TEST_F(Utf8CollectionTest, CreateInsertFlushReopen) { + CollectionOptions opts; + opts.read_only_ = false; + opts.enable_mmap_ = true; + + auto schema = TestHelper::CreateNormalSchema(); + auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); + if (!result.has_value()) { + std::cout << result.error().message() << std::endl; + } + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(ailego::FileHelper::IsExist(kUtf8Dir.c_str())); + + auto col = std::move(result).value(); + ASSERT_EQ(col->Path(), kUtf8Dir); + + const int kDocCount = 10; + auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount); + ASSERT_TRUE(s.ok()) << s.message(); + + ASSERT_TRUE(col->Flush().ok()); + + auto stats = col->Stats().value(); + ASSERT_EQ(stats.doc_count, kDocCount); + + col.reset(); + + // Reopen and verify doc count survives + auto reopen = Collection::Open(kUtf8Dir, opts); + ASSERT_TRUE(reopen.has_value()) << reopen.error().message(); + auto col2 = std::move(reopen).value(); + auto stats2 = col2->Stats().value(); + ASSERT_EQ(stats2.doc_count, kDocCount); +} + +// --------------------------------------------------------------------------- +// 2. Destroy a collection in a UTF-8 path +// --------------------------------------------------------------------------- +TEST_F(Utf8CollectionTest, CreateAndDestroy) { + CollectionOptions opts; + opts.read_only_ = false; + opts.enable_mmap_ = true; + + auto schema = TestHelper::CreateNormalSchema(); + auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); + ASSERT_TRUE(result.has_value()); + + auto col = std::move(result).value(); + ASSERT_EQ(col->Destroy(), Status::OK()); + ASSERT_FALSE(ailego::FileHelper::IsExist(kUtf8Dir.c_str())); +} + +// --------------------------------------------------------------------------- +// 3. Verify the on-disk layout contains correct UTF-8 names (no mojibake) +// by listing files via the wide-char Windows API or POSIX readdir. +// --------------------------------------------------------------------------- +#if defined(_WIN32) || defined(_WIN64) +TEST_F(Utf8CollectionTest, NoDirGarble) { + CollectionOptions opts; + opts.read_only_ = false; + opts.enable_mmap_ = true; + + auto schema = TestHelper::CreateNormalSchema(); + auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); + ASSERT_TRUE(result.has_value()); + auto col = std::move(result).value(); + + const int kDocCount = 5; + auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount); + ASSERT_TRUE(s.ok()) << s.message(); + ASSERT_TRUE(col->Flush().ok()); + + // --- Check parent directory via FindFirstFileW --- + { + WIN32_FIND_DATAW fd; + HANDLE hFind = FindFirstFileW(L".\\*", &fd); + ASSERT_NE(INVALID_HANDLE_VALUE, hFind); + + std::wstring expected = ailego::FileHelper::Utf8ToWide(kUtf8Dir); + bool found = false; + do { + if (std::wstring(fd.cFileName) == expected) { + found = true; + break; + } + } while (FindNextFileW(hFind, &fd)); + FindClose(hFind); + + EXPECT_TRUE(found) << "Collection dir '" << kUtf8Dir + << "' not found via FindFirstFileW (garbled Unicode?)"; + } + + // --- Check inside collection via "cmd /u /c dir /b" redirected to file --- + { + std::wstring wdir = ailego::FileHelper::Utf8ToWide(kUtf8Dir); + std::string tmpfile = "utf8_col_dir_output.txt"; + std::wstring wtmp = ailego::FileHelper::Utf8ToWide(tmpfile); + std::wstring cmd = L"cmd /u /c dir /b \"" + wdir + L"\" >\"" + wtmp + L"\""; + _wsystem(cmd.c_str()); + + std::ifstream ifs; + ailego::FileHelper::OpenIfstream(ifs, tmpfile, std::ios::binary); + ASSERT_TRUE(ifs.is_open()); + std::string raw((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + ifs.close(); + ailego::FileHelper::DeleteFile(tmpfile.c_str()); + + const wchar_t *wdata = reinterpret_cast(raw.data()); + size_t wlen = raw.size() / sizeof(wchar_t); + if (wlen > 0 && wdata[0] == L'\xFEFF') { + wdata++; + wlen--; + } + std::wstring woutput(wdata, wlen); + std::string output = ailego::FileHelper::WideToUtf8(woutput); + + EXPECT_FALSE(output.empty()) << "dir listing inside collection is empty"; + std::cout << "[NoDirGarble] Contents of " << kUtf8Dir << ":\n" + << output << std::endl; + } +} +#endif + +// --------------------------------------------------------------------------- +// 4. MakeWalPath / MakeSegmentPath etc. produce correct UTF-8 results +// --------------------------------------------------------------------------- +TEST_F(Utf8CollectionTest, FileHelperPaths) { + std::string wal = FileHelper::MakeWalPath(kUtf8Dir, 0, 1); + std::string seg = FileHelper::MakeSegmentPath(kUtf8Dir, 0); + std::string fwd = FileHelper::MakeForwardBlockPath(kUtf8Dir, 0, 1, true); + std::string inv = FileHelper::MakeInvertIndexPath(kUtf8Dir, 0, 1); + std::string vec = FileHelper::MakeVectorIndexPath(kUtf8Dir, "vec1", 0, 1); + + // All paths should start with our UTF-8 base + EXPECT_EQ(0u, wal.find(kUtf8Dir)); + EXPECT_EQ(0u, seg.find(kUtf8Dir)); + EXPECT_EQ(0u, fwd.find(kUtf8Dir)); + EXPECT_EQ(0u, inv.find(kUtf8Dir)); + EXPECT_EQ(0u, vec.find(kUtf8Dir)); + + // Should not contain forward slash on Windows +#if defined(_WIN32) || defined(_WIN64) + EXPECT_EQ(std::string::npos, wal.find('/')); + EXPECT_EQ(std::string::npos, seg.find('/')); + EXPECT_EQ(std::string::npos, fwd.find('/')); + EXPECT_EQ(std::string::npos, inv.find('/')); + EXPECT_EQ(std::string::npos, vec.find('/')); +#endif +} diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt index 30b1a87..5f1bc3c 100644 --- a/thirdparty/rocksdb/CMakeLists.txt +++ b/thirdparty/rocksdb/CMakeLists.txt @@ -37,6 +37,7 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE) set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE) set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE) set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE) +set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "use UTF8 as characterset for opening files, regardles of the system code page" FORCE) # TODO(windows): verify set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE) set(FAIL_ON_WARNINGS OFF CACHE BOOL "build with no Werror" FORCE)