From d91867e00edb59a6fdc9a63fabd0b6db5f0d699d Mon Sep 17 00:00:00 2001 From: Cuiys Date: Tue, 12 May 2026 15:48:48 +0800 Subject: [PATCH] ci: add ccache/sccache compilation caching to speed up CI builds (#360) * ci: add ccache/sccache compilation caching to speed up CI builds - Use hendrikmuhs/ccache-action@v1.2 for Linux/macOS/iOS/Android/clang-tidy (auto-installs ccache, manages cache, sets env vars, shows stats) - Use mozilla-actions/sccache-action@v0.0.9 for Windows (MSVC compatible) - Add CMAKE_C/CXX_COMPILER_LAUNCHER to all CMake build steps - Exclude wheel build and nightly coverage workflows per decision * ci: switch MacOS & Linux build from Unix Makefiles to Ninja generator - Replace CMAKE_GENERATOR='Unix Makefiles' with 'Ninja' in pip build - Replace 'make unittest -j' with 'cmake --build --target unittest --parallel' - Add '-G Ninja' to C++ and C example cmake configure steps - Replace 'make -j' with 'cmake --build --parallel' for examples - Aligns with Windows and Android workflows which already use Ninja * ci: enable parallel ctest execution with -j and --timeout - Use CMake ProcessorCount module to detect available CPU cores - Add -j ${NPROC} to ctest command for parallel test execution - Add --timeout 300 to prevent individual tests from hanging CI - Fallback to NPROC=1 when ProcessorCount returns 0 - iOS target unchanged (build-only, no test execution) * Revert "ci: enable parallel ctest execution with -j and --timeout" This reverts commit d196dac5f17b1f7cae443360c88bbd8c935dc145. * fix(ci): remove sccache from Windows, fix cmake.define quote issues Windows (05-windows-build.yml): - Remove mozilla-actions/sccache-action: sccache incompatible with MSVC /FS flag - Remove SCCACHE_GHA_ENABLED env var - Remove CMAKE_C/CXX_COMPILER_LAUNCHER=sccache from build steps - Remove 'Show sccache statistics' step - MSVC /FS (global PDB concurrency flag) causes fatal C1041 when used with sccache MacOS & Linux (03-macos-linux-build.yml): - Fix cmake.define values: remove extra quotes around 'ccache' and 'ON' - Bare values required: cmake.define.FOO=bar not cmake.define.FOO="bar" * feat: cache key with platform and os * ci: add compiler to ccache key to avoid cache pollution * ci: optimize cache usage to reduce bloat - Add max-size limits to all ccache configs (150M general, 300M Android, 100M clang-tidy) to prevent unbounded cache growth - Remove redundant iOS full build directory cache (~1.2 GB) since ccache already handles incremental compilation - Fix iOS protoc cache key to use thirdparty/protobuf/** instead of src/**, avoiding unnecessary cache misses on business code changes * ci: trigger CI run --- .github/workflows/03-macos-linux-build.yml | 26 ++++++++++++++++------ .github/workflows/04-android-build.yml | 17 +++++--------- .github/workflows/06-ios-build.yml | 22 ++++++++++-------- .github/workflows/clang_tidy.yml | 10 ++++++++- 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/.github/workflows/03-macos-linux-build.yml b/.github/workflows/03-macos-linux-build.yml index 1dd1e45..4ea8c74 100644 --- a/.github/workflows/03-macos-linux-build.yml +++ b/.github/workflows/03-macos-linux-build.yml @@ -40,6 +40,12 @@ jobs: with: submodules: recursive + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ inputs.platform }}-${{ inputs.os }}-${{ inputs.compiler || 'gcc' }} + max-size: 150M + - name: Set up Python uses: actions/setup-python@v6 with: @@ -95,18 +101,20 @@ jobs: run: | cd "$GITHUB_WORKSPACE" - CMAKE_GENERATOR="Unix Makefiles" \ + CMAKE_GENERATOR="Ninja" \ CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \ python -m pip install -v . \ --no-build-isolation \ - --config-settings='cmake.define.BUILD_TOOLS="ON"' \ + --config-settings='cmake.define.BUILD_TOOLS=ON' \ + --config-settings='cmake.define.CMAKE_C_COMPILER_LAUNCHER=ccache' \ + --config-settings='cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=ccache' \ ${{ matrix.arch_flag }} shell: bash - name: Run C++ Tests run: | cd "$GITHUB_WORKSPACE/build" - make unittest -j$NPROC + cmake --build . --target unittest --parallel $NPROC shell: bash - name: Run Python Tests @@ -119,8 +127,10 @@ jobs: run: | cd "$GITHUB_WORKSPACE/examples/c++" mkdir build && cd build - cmake .. -DCMAKE_BUILD_TYPE=Release - make -j $NPROC + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build . --parallel $NPROC ./db-example ./core-example ./ailego-example @@ -130,8 +140,10 @@ jobs: run: | cd "$GITHUB_WORKSPACE/examples/c" mkdir build && cd build - cmake .. -DCMAKE_BUILD_TYPE=Release - make -j $NPROC + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build . --parallel $NPROC ./c_api_basic_example ./c_api_collection_schema_example ./c_api_doc_example diff --git a/.github/workflows/04-android-build.yml b/.github/workflows/04-android-build.yml index 092bacb..b635205 100644 --- a/.github/workflows/04-android-build.yml +++ b/.github/workflows/04-android-build.yml @@ -25,11 +25,6 @@ jobs: matrix: abi: [x86_64] api: ${{ github.event.inputs.api && fromJSON(format('["{0}"]', github.event.inputs.api)) || fromJSON('["34"]') }} - env: - CCACHE_BASEDIR: ${{ github.workspace }} - CCACHE_NOHASHDIR: '1' - CCACHE_SLOPPINESS: clang_index_store,file_stat_matches,include_file_mtime,locale,time_macros - steps: # ── Environment setup ────────────────────────────────────────────── - name: Checkout @@ -42,15 +37,13 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ cmake ninja-build git ca-certificates python3 \ - build-essential make ccache unzip curl + build-essential make unzip curl - - name: Cache ccache - uses: actions/cache@v5 + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 with: - path: ~/.ccache - key: ${{ runner.os }}-ccache-android-${{ matrix.abi }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-ccache-android-${{ matrix.abi }}- + key: android-${{ matrix.abi }} + max-size: 300M - name: Setup Java 17 uses: actions/setup-java@v5 diff --git a/.github/workflows/06-ios-build.yml b/.github/workflows/06-ios-build.yml index 26ffbbf..802716d 100644 --- a/.github/workflows/06-ios-build.yml +++ b/.github/workflows/06-ios-build.yml @@ -31,29 +31,31 @@ jobs: with: submodules: recursive + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ios-${{ matrix.platform }} + max-size: 150M + - name: Cache host protoc build uses: actions/cache@v5 with: path: build_host - key: macos-host-protoc-${{ hashFiles('src/**', 'CMakeLists.txt') }} + key: macos-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }} restore-keys: | macos-host-protoc- - name: Build host protoc run: | if [ ! -f "build_host/bin/protoc" ]; then - cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build_host --target protoc --parallel $(sysctl -n hw.ncpu) else echo "Using cached host protoc" fi - - name: Cache iOS build - uses: actions/cache@v5 - with: - path: build_ios_${{ matrix.platform }} - key: ios-build-${{ matrix.platform }}-${{ hashFiles('src/**', 'CMakeLists.txt', 'cmake/**', 'thirdparty/**') }} - - name: Configure and Build run: | git submodule foreach --recursive 'git stash --include-untracked' || true @@ -72,7 +74,9 @@ jobs: -DCMAKE_INSTALL_PREFIX="./install" \ -DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \ -DIOS=ON \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build_ios_${{ matrix.platform }} --parallel $NPROC diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 0b595b6..aba45c5 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -29,11 +29,19 @@ jobs: sudo apt-get update sudo apt-get install -y clang-tidy=1:18.0-59~exp2 cmake ninja-build libomp-dev + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: clang-tidy + max-size: 100M + - name: Configure CMake and export compile commands run: | cmake -S . -B build -G Ninja \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DBUILD_TOOLS=ON + -DBUILD_TOOLS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Collect changed C/C++ files id: changed_files