diff --git a/.github/workflows/03-macos-linux-build.yml b/.github/workflows/03-macos-linux-build.yml index 4ea8c74..a9bcfc5 100644 --- a/.github/workflows/03-macos-linux-build.yml +++ b/.github/workflows/03-macos-linux-build.yml @@ -93,6 +93,7 @@ jobs: cmake==3.30.0 \ ninja==1.11.1 \ pytest \ + pytest-xdist \ scikit-build-core \ setuptools_scm shell: bash diff --git a/.github/workflows/05-windows-build.yml b/.github/workflows/05-windows-build.yml index 9922e03..0db11c2 100644 --- a/.github/workflows/05-windows-build.yml +++ b/.github/workflows/05-windows-build.yml @@ -64,6 +64,7 @@ jobs: cmake==3.30.0 ` ninja==1.11.1 ` pytest ` + pytest-xdist ` scikit-build-core ` setuptools_scm shell: powershell diff --git a/.github/workflows/nightly_coverage.yml b/.github/workflows/nightly_coverage.yml index 1279496..458044d 100644 --- a/.github/workflows/nightly_coverage.yml +++ b/.github/workflows/nightly_coverage.yml @@ -59,6 +59,7 @@ jobs: ninja==1.11.1 \ pytest \ pytest-cov \ + pytest-xdist \ scikit-build-core \ setuptools_scm shell: bash diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index 262f721..d8ddba0 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -318,10 +318,20 @@ if(NOT TARGET unittest) # iOS: build-only target; tests are run on simulator separately add_custom_target(unittest) else() + include(ProcessorCount) + ProcessorCount(NPROC) + if(NPROC EQUAL 0) + set(NPROC 1) + endif() + math(EXPR PARALLEL_JOBS "${NPROC} - 1") + if(PARALLEL_JOBS LESS 1) + set(PARALLEL_JOBS 1) + endif() add_custom_target( unittest COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --build-config $ + --parallel ${PARALLEL_JOBS} ) endif() endif() @@ -1110,16 +1120,18 @@ function(cc_test) "${CC_ARGS_UNPARSED_ARGUMENTS}" ) add_dependencies(unittest ${CC_ARGS_NAME}) + set(TEST_WORKING_DIR "${CMAKE_BINARY_DIR}/test_tmp/${CC_ARGS_NAME}") + file(MAKE_DIRECTORY "${TEST_WORKING_DIR}") add_custom_target( unittest.${CC_ARGS_NAME} COMMAND $ "${CC_ARGS_ARGS}" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${TEST_WORKING_DIR} DEPENDS ${CC_ARGS_NAME} ) add_test( NAME ${CC_ARGS_NAME} COMMAND $ "${CC_ARGS_ARGS}" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${TEST_WORKING_DIR} ) endfunction() @@ -1925,16 +1937,18 @@ function(cuda_test) "${CUDA_ARGS_UNPARSED_ARGUMENTS}" ) add_dependencies(unittest ${CUDA_ARGS_NAME}) + set(TEST_WORKING_DIR "${CMAKE_BINARY_DIR}/test_tmp/${CUDA_ARGS_NAME}") + file(MAKE_DIRECTORY "${TEST_WORKING_DIR}") add_custom_target( unittest.${CUDA_ARGS_NAME} COMMAND $ "${CUDA_ARGS_ARGS}" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${TEST_WORKING_DIR} DEPENDS ${CUDA_ARGS_NAME} ) add_test( NAME ${CUDA_ARGS_NAME} COMMAND $ "${CUDA_ARGS_ARGS}" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${TEST_WORKING_DIR} ) endfunction() diff --git a/pyproject.toml b/pyproject.toml index c7125a3..2b7908f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ test = [ "pytest >=8.0", "pytest-cov >=4.1", "pytest-mock >=3.12", + "pytest-xdist >=3.5", "cibuildwheel == 3.4.0", ] docs = [ @@ -142,6 +143,7 @@ addopts = [ "--strict-markers", "--strict-config", "--tb=short", + "-n=auto", ] xfail_strict = true log_cli_level = "INFO" @@ -169,7 +171,7 @@ build = [ "cp314-*", ] build-frontend = "build" -test-requires = ["pytest", "numpy"] +test-requires = ["pytest", "pytest-xdist", "numpy"] test-command = "cd {project} && pytest python/tests -v --tb=short" build-verbosity = 1 diff --git a/tests/db/crash_recovery/CMakeLists.txt b/tests/db/crash_recovery/CMakeLists.txt index f737b40..7a87243 100644 --- a/tests/db/crash_recovery/CMakeLists.txt +++ b/tests/db/crash_recovery/CMakeLists.txt @@ -77,5 +77,8 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) ) add_dependencies(${CC_TARGET} data_generator) add_dependencies(${CC_TARGET} collection_optimizer) + set_tests_properties(${CC_TARGET} PROPERTIES + ENVIRONMENT "TEST_BINARY_DIR=${PROJECT_BINARY_DIR}" + ) cc_test_suite(zvec_crash_recovery ${CC_TARGET}) endforeach() diff --git a/tests/db/crash_recovery/optimize_recovery_test.cc b/tests/db/crash_recovery/optimize_recovery_test.cc index 620c5df..a2a7f66 100644 --- a/tests/db/crash_recovery/optimize_recovery_test.cc +++ b/tests/db/crash_recovery/optimize_recovery_test.cc @@ -46,38 +46,7 @@ const int num_batches{1000}; static std::string LocateOptimizeGenerator() { - namespace fs = std::filesystem; - std::cout << "Current path: " << fs::current_path() << std::endl; - - const std::string base_name = "collection_optimizer"; - std::vector candidates; - const std::vector search_paths = {"./", "./bin/"}; - - for (const auto &p : search_paths) { - candidates.push_back(p); - } - -// TODO(windows): unify _WIN32/_WIN64/MSCV_VER -#ifdef _WIN32 - for (const auto &p : search_paths) { - candidates.push_back(p + "Debug/"); - candidates.push_back(p + "Release/"); - } -#endif - - for (auto &p : candidates) { - p += base_name; -#ifdef _WIN32 - p += ".exe"; -#endif - } - - for (const auto &p : candidates) { - if (fs::exists(p)) { - return fs::canonical(p).string(); - } - } - throw std::runtime_error("collection_optimizer binary not found"); + return LocateBinary("collection_optimizer"); } diff --git a/tests/db/crash_recovery/utility.h b/tests/db/crash_recovery/utility.h index 4f1a0bf..826111c 100644 --- a/tests/db/crash_recovery/utility.h +++ b/tests/db/crash_recovery/utility.h @@ -16,6 +16,13 @@ #pragma once +#include +#include +#include +#include +#include +#include + #include #include @@ -149,4 +156,50 @@ inline Doc CreateTestDoc(uint64_t doc_id, int version) { } + +/** + * @brief Locate a binary by name, searching common paths and TEST_BINARY_DIR. + * + * @param binary_name The base name of the binary (e.g. "data_generator") + * @return std::string The canonical path to the found binary + * @throws std::runtime_error if the binary is not found + */ +inline std::string LocateBinary(const std::string &binary_name) { + namespace fs = std::filesystem; + std::cout << "Current path: " << fs::current_path() << std::endl; + + std::vector candidates; + const std::vector search_paths = {"./", "./bin/"}; + + for (const auto &p : search_paths) { + candidates.push_back(p); + } +#ifdef _WIN32 + for (const auto &p : search_paths) { + candidates.push_back(p + "Debug/"); + candidates.push_back(p + "Release/"); + } +#endif + + const char *test_binary_dir = std::getenv("TEST_BINARY_DIR"); + if (test_binary_dir != nullptr) { + candidates.push_back(std::string(test_binary_dir) + "/"); + candidates.push_back(std::string(test_binary_dir) + "/bin/"); + } + + for (auto &p : candidates) { + p += binary_name; +#ifdef _WIN32 + p += ".exe"; +#endif + } + + for (const auto &p : candidates) { + if (fs::exists(p)) { + return fs::canonical(p).string(); + } + } + throw std::runtime_error(binary_name + " binary not found"); +} + } // namespace zvec diff --git a/tests/db/crash_recovery/write_recovery_test.cc b/tests/db/crash_recovery/write_recovery_test.cc index 6eed2e0..d460b7b 100644 --- a/tests/db/crash_recovery/write_recovery_test.cc +++ b/tests/db/crash_recovery/write_recovery_test.cc @@ -44,38 +44,7 @@ const zvec::CollectionOptions options_{false, true, 256 * 1024}; static std::string LocateDataGenerator() { - namespace fs = std::filesystem; - std::cout << "Current path: " << fs::current_path() << std::endl; - - const std::string base_name = "data_generator"; - std::vector candidates; - // Define potential search locations relative to the current working directory - const std::vector search_paths = {"./", "./bin/"}; - - for (const auto &p : search_paths) { - candidates.push_back(p); - } -#ifdef _WIN32 - for (const auto &p : search_paths) { - candidates.push_back(p + "Debug/"); - candidates.push_back(p + "Release/"); - } -#endif - - - for (auto &p : candidates) { - p += base_name; -#ifdef _WIN32 - p += ".exe"; // Append .exe suffix for Windows compatibility -#endif - } - - for (const auto &p : candidates) { - if (fs::exists(p)) { - return fs::canonical(p).string(); - } - } - throw std::runtime_error("data_generator binary not found"); + return LocateBinary("data_generator"); }