From 0ffdb140bf608e4904022f7022d6c2bd9daa69ed Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 9 Sep 2025 14:13:59 -0400 Subject: [PATCH 1/5] ci: add newdeps job testing newest versions of cmake and capnproto --- .github/workflows/ci.yml | 4 ++-- ci/README.md | 1 + ci/configs/newdeps.bash | 6 ++++++ ci/scripts/ci.sh | 17 +++++++++++++++++ shell.nix | 7 ++++++- 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 ci/configs/newdeps.bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87b1706..204e5b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: uses: vmactions/freebsd-vm@v1 with: prepare: | - pkg install -y cmake ninja bash capnproto + pkg install -y cmake ninja bash capnproto git sync: 'rsync' copyback: false @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - config: [default, llvm, gnu32, sanitize, olddeps] + config: [default, llvm, gnu32, sanitize, olddeps, newdeps] name: build • ${{ matrix.config }} diff --git a/ci/README.md b/ci/README.md index 2ddaa9e..25c0a7e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,6 +21,7 @@ CI_CONFIG=ci/configs/llvm.bash ci/scripts/run.sh CI_CONFIG=ci/configs/gnu32.bash ci/scripts/run.sh CI_CONFIG=ci/configs/sanitize.bash ci/scripts/run.sh CI_CONFIG=ci/configs/olddeps.bash ci/scripts/run.sh +CI_CONFIG=ci/configs/newdeps.bash ci/scripts/run.sh ``` By default CI jobs will reuse their build directories. `CI_CLEAN=1` can be specified to delete them before running instead. diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash new file mode 100644 index 0000000..e05913a --- /dev/null +++ b/ci/configs/newdeps.bash @@ -0,0 +1,6 @@ +CI_DESC="CI job using newest Cap'n Proto and cmake versions" +CI_DIR=build-newdeps +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" +CAPNP_CHECKOUT=master +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.1.1") +BUILD_ARGS=(-k) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index d989e9f..77bec68 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -21,9 +21,26 @@ cmake --version cmake_ver=$(cmake --version | awk '/version/{print $3; exit}') ver_ge() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ]; } +# If CAPNP_CHECKOUT was requested, clone and install requested Cap'n Proto branch or tag +capnp_prefix= +if [ -n "${CAPNP_CHECKOUT-}" ]; then + capnp_prefix="$PWD/capnp-install" + [ -e "capnp" ] || git clone -b "${CAPNP_CHECKOUT}" "https://github.com/capnproto/capnproto" capnp + mkdir -p capnp/build + ( + cd capnp/build + git --no-pager log -1 || true + CXXFLAGS="-std=c++20" cmake .. "-DCMAKE_INSTALL_PREFIX=${capnp_prefix}" -DBUILD_TESTING=OFF -DWITH_OPENSSL=OFF -DWITH_ZLIB=OFF + cmake --build . + cmake --install . + ) + export CMAKE_PREFIX_PATH="${capnp_prefix}:${CMAKE_PREFIX_PATH-}" +fi + src_dir=$PWD mkdir -p "$CI_DIR" cd "$CI_DIR" +git --no-pager log -1 || true cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" if ver_ge "$cmake_ver" "3.15"; then cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" diff --git a/shell.nix b/shell.nix index 24d3820..6d7552e 100644 --- a/shell.nix +++ b/shell.nix @@ -40,6 +40,7 @@ let clang-tools = llvm.clang-tools.override { inherit enableLibcxx; }; cmakeHashes = { "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; + "4.1.1" = "sha256-sp9vGXM6oiS3djUHoQikJ+1Ixojh+vIrKcROHDBUkoI="; }; cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { version = cmakeVersion; @@ -50,11 +51,12 @@ let patches = []; })).override { isMinimalBuild = true; }; in crossPkgs.mkShell { - buildInputs = [ + buildInputs = lib.optionals (capnprotoVersion != "none") [ capnproto ]; nativeBuildInputs = with pkgs; [ cmakeBuild + git include-what-you-use ninja ] ++ lib.optionals (!minimal) [ @@ -64,4 +66,7 @@ in crossPkgs.mkShell { # Tell IWYU where its libc++ mapping lives IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null; + + # Avoid "SSL certificate problem: unable to get local issuer certificate" error during git clone in ci/scripts/ci.sh + NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; } From 6ba1050bef0b5ea5df35d3623c9e101cceb82677 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 10 Sep 2025 07:06:21 -0400 Subject: [PATCH 2/5] ci: add cmake debug output and --parallel build option --- ci/configs/sanitize.bash | 2 +- ci/scripts/ci.sh | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ci/configs/sanitize.bash b/ci/configs/sanitize.bash index ce920f4..fc24960 100644 --- a/ci/configs/sanitize.bash +++ b/ci/configs/sanitize.bash @@ -3,5 +3,5 @@ CI_DIR=build-sanitize export CXX=clang++ export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety-analysis -Wno-unused-parameter -fsanitize=thread" CMAKE_ARGS=() -BUILD_ARGS=(-k -j4) +BUILD_ARGS=(-k) BUILD_TARGETS=(mptest) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index 77bec68..58b8d34 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -41,14 +41,23 @@ src_dir=$PWD mkdir -p "$CI_DIR" cd "$CI_DIR" git --no-pager log -1 || true -cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" +cmake_args=("${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}") +if ! cmake "$src_dir" "${cmake_args[@]}"; then + # If cmake failed, try it again with debug options. + # Could add --trace / --trace-expand here too but they are very verbose. + cmake_args+=(--debug-find --debug-output --debug-trycompile --log-level=DEBUG) + cmake "$src_dir" "${cmake_args[@]}" || : "cmake exited with $?" + cat CMakeFiles/CMakeConfigureLog.yaml || true + find . -ls || true + false +fi if ver_ge "$cmake_ver" "3.15"; then - cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" + cmake --build . --parallel -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" else # Older versions of cmake can only build one target at a time with --target, # and do not support -t shortcut for t in "${BUILD_TARGETS[@]}"; do - cmake --build . --target "$t" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" + cmake --build . --parallel --target "$t" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" done fi ctest --output-on-failure From 9441e5a8f370d872a735c73c916f4d5e0b6258bf Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 10 Sep 2025 06:55:28 -0400 Subject: [PATCH 3/5] cmake: fix find_package(Threads REQUIRED) error with new cmake policies With latest cmake policies, find_package(Threads REQUIRED) fails in freebsd and openbsd builds. The specific failures in these jobs happens due to the CMP0155 policy turning CMAKE_CXX_SCAN_FOR_MODULES on, which is turned off in the next commit. But there are other things that could cause the threads package not to be found, and there are platforms where it may not be required, so it is better to make it an optional instead of required dependency. Technically this change is not needed to make all CI jobs pass as long as the next commit is present. But since this find_package error obscures other errors that would be clearer, and since there are other conditions that could trigger it, it is worth fixing generally. For example, this same failure also seems to happen in the llvm job as well when CMP0137 is disabled or CMAKE_TRY_COMPILE_NO_PLATFORM_VARIABLES is set to true. Error looks like: + cmake /home/runner/work/libmultiprocess/libmultiprocess -G Ninja -- The CXX compiler identification is Clang 16.0.6 -- Detecting CXX compiler ABI info -- Detecting CXX compiler ABI info - done -- Check for working CXX compiler: /usr/bin/c++ - skipped -- Detecting CXX compile features -- Detecting CXX compile features - done -- Performing Test CMAKE_HAVE_LIBC_PTHREAD -- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed -- Looking for pthread_create in pthreads -- Looking for pthread_create in pthreads - not found -- Looking for pthread_create in pthread -- Looking for pthread_create in pthread - not found -- Check if compiler accepts -pthread -- Check if compiler accepts -pthread - no CMake Error at /usr/local/share/cmake/Modules/FindPackageHandleStandardArgs.cmake:233 (message): Could NOT find Threads (missing: Threads_FOUND) Call Stack (most recent call first): /usr/local/share/cmake/Modules/FindPackageHandleStandardArgs.cmake:603 (_FPHSA_FAILURE_MESSAGE) /usr/local/share/cmake/Modules/FindThreads.cmake:226 (FIND_PACKAGE_HANDLE_STANDARD_ARGS) CMakeLists.txt:41 (find_package) and inside the CMakeConfigureLog.yaml file there are "/bin/sh: CMAKE_CXX_COMPILER_CLANG_SCAN_DEPS-NOTFOUND: not found" errors. --- CMakeLists.txt | 11 +++++++++-- cmake/pthread_checks.cmake | 2 +- example/CMakeLists.txt | 6 +++--- src/mp/util.cpp | 4 ++-- test/CMakeLists.txt | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a36023b..fc92edf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,14 @@ endif() include("cmake/compat_find.cmake") -find_package(Threads REQUIRED) +add_library(mpdeps INTERFACE) + +find_package(Threads) + +if(Threads_FOUND) + target_link_libraries(mpdeps INTERFACE Threads::Threads) +endif() + find_package(CapnProto 0.7 QUIET NO_MODULE) if(NOT CapnProto_FOUND) message(FATAL_ERROR @@ -202,7 +209,7 @@ target_link_libraries(mpgen PRIVATE CapnProto::capnp) target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) target_link_libraries(mpgen PRIVATE CapnProto::capnpc) target_link_libraries(mpgen PRIVATE CapnProto::kj) -target_link_libraries(mpgen PRIVATE Threads::Threads) +target_link_libraries(mpgen PRIVATE mpdeps) set_target_properties(mpgen PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) set_target_properties(mpgen PROPERTIES diff --git a/cmake/pthread_checks.cmake b/cmake/pthread_checks.cmake index 241978d..a8b59c2 100644 --- a/cmake/pthread_checks.cmake +++ b/cmake/pthread_checks.cmake @@ -9,7 +9,7 @@ include(CMakePushCheckState) include(CheckCXXSourceCompiles) cmake_push_check_state() -set(CMAKE_REQUIRED_LIBRARIES Threads::Threads) +set(CMAKE_REQUIRED_LIBRARIES mpdeps) check_cxx_source_compiles(" #include int main(int argc, char** argv) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 7da049c..a8961c5 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -9,20 +9,20 @@ add_executable(mpcalculator ) target_capnp_sources(mpcalculator ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) target_include_directories(mpcalculator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpcalculator PRIVATE Threads::Threads) +target_link_libraries(mpcalculator PRIVATE mpdeps) add_executable(mpprinter printer.cpp ) target_capnp_sources(mpprinter ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) target_include_directories(mpprinter PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpprinter PRIVATE Threads::Threads) +target_link_libraries(mpprinter PRIVATE mpdeps) add_executable(mpexample example.cpp ) target_capnp_sources(mpexample ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) target_include_directories(mpexample PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpexample PRIVATE Threads::Threads) +target_link_libraries(mpexample PRIVATE mpdeps) add_custom_target(mpexamples DEPENDS mpexample mpcalculator mpprinter) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index b712462..71263aa 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include // IWYU pragma: keep #include #include #include #include #include -#include +#include // IWYU pragma: keep #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2a1a7e9..2cdb317 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,7 +32,7 @@ if(BUILD_TESTING AND TARGET CapnProto::kj-test) target_capnp_sources(mptest ${CMAKE_CURRENT_SOURCE_DIR} mp/test/foo.capnp) target_include_directories(mptest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(mptest PRIVATE CapnProto::kj-test) - target_link_libraries(mptest PRIVATE Threads::Threads) + target_link_libraries(mptest PRIVATE mpdeps) add_dependencies(mptests mptest) add_test(NAME mptest COMMAND mptest) From e416d45528faa7d4a15dc6e46e265778e611a26c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 10 Sep 2025 10:50:44 -0400 Subject: [PATCH 4/5] cmake: fix CMAKE_CXX_COMPILER_CLANG_SCAN_DEPS-NOTFOUND error with new cmake policies With latest cmake policies, openbsd and freebsd CI jobs fail due to a lack of a clang-scan-deps tool. The tool could potentially be installed on these platforms but it is unclear how to do that and the project isn't using modules anyway, so just disable them here. Errors look like: + cmake --build . --parallel -t all tests mpexamples -- -k 0 [1/114] Scanning /home/runner/work/libmultiprocess/libmultiprocess/src/mp/util.cpp for CXX dependencies FAILED: CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi "CMAKE_CXX_COMPILER_CLANG_SCAN_DEPS-NOTFOUND" -format=p1689 -- /usr/bin/c++ -I/home/runner/work/libmultiprocess/libmultiprocess/include -I/home/runner/work/libmultiprocess/libmultiprocess/build-openbsd/include -isystem /usr/local/include -Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -std=gnu++20 -x c++ /home/runner/work/libmultiprocess/libmultiprocess/src/mp/util.cpp -c -o CMakeFiles/mputil.dir/src/mp/util.cpp.o -resource-dir "/usr/lib/clang/16" -MT CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi -MD -MF CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi.d > CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi.tmp && mv CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi.tmp CMakeFiles/mputil.dir/src/mp/util.cpp.o.ddi /bin/sh: CMAKE_CXX_COMPILER_CLANG_SCAN_DEPS-NOTFOUND: not found --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc92edf..96b9dcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,11 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(CMAKE_CXX_STANDARD_REQUIRED YES) endif() +# Disable automatic C++20 module dependency scanning. +# CMake >=3.28 tries to use `clang-scan-deps` by default, which may not +# be installed on all platforms. We don't use named modules, so turn this off. +set(CMAKE_CXX_SCAN_FOR_MODULES OFF) + include("cmake/compat_find.cmake") add_library(mpdeps INTERFACE) From 1c2958bfffd60c8cf9ec9715699c1496f6377922 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 9 Sep 2025 14:20:44 -0400 Subject: [PATCH 5/5] cmake: Increase cmake policy version Increase cmake policy version from 3.12 to 4.1 in standalone builds to stop using very old and deprecated CMake policies in standalone builds. Also stop overriding policy version if a parent project has already set one, so parent projects are able to control which policies are used by default based on their own practices and needs. In the Bitcoin Core subtree, this change causes the libmultiprocess policy version to increase from 3.12 to 3.22, which is the version Bitcoin Core sets. This commit only changes the policy version, it does not change the specified minimum version of cmake required to build the project, which is 3.12. --- CMakeLists.txt | 28 +++++++++++++++++++++++++++- ci/configs/newdeps.bash | 1 + ci/configs/olddeps.bash | 1 + 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96b9dcf..4e21e82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,33 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -cmake_minimum_required(VERSION 3.12) +# Call cmake_minimum_required() only if it was not already called, so parent +# projects set the policy version when this project is included via +# add_subdirectory(). +# +# Rationale: Different projects have different practices for choosing policy +# versions. For example, the Bitcoin Core project sets an old policy version, +# causing CMake to use deprecated behaviors instead of new behaviors and +# maximize compatibility with a single old version of CMake, reducing variance +# between builds with newer CMake versions. By contrast CMake documentation +# recommends setting the policy version to the latest supported version of +# CMake, as an upgrading mechanism rather than a pinning mechanism, to let +# project authors fix problems before enabling newer policies, while not opting +# into deprecated policies on a longer term basis. +if(NOT DEFINED CMAKE_MINIMUM_REQUIRED_VERSION + OR CMAKE_MINIMUM_REQUIRED_VERSION VERSION_LESS 3.12) + # The left number in the range below is the minimum CMake version required to + # run this project. The right number is the CMake policy version. + # + # The purpose of the minimum version is to trigger a helpful error if a version + # of CMake is being used that is too old to work. If this number is changed, + # the version in ci/configs/olddeps.bash should be changed to match. + # + # The purpose of the policy version is to opt out of policies introduced in + # newer versions of CMake until they have been tested. If this number is + # changed, the version in ci/configs/newdeps.bash should be changed to match. + cmake_minimum_required(VERSION 3.12...4.1 FATAL_ERROR) +endif() project("Libmultiprocess" CXX) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash index e05913a..c59a76d 100644 --- a/ci/configs/newdeps.bash +++ b/ci/configs/newdeps.bash @@ -2,5 +2,6 @@ CI_DESC="CI job using newest Cap'n Proto and cmake versions" CI_DIR=build-newdeps export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" CAPNP_CHECKOUT=master +# cmakeVersion here should match policy version in CMakeLists.txt NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.1.1") BUILD_ARGS=(-k) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 95f4412..2a6329c 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -1,5 +1,6 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" +# cmakeVersion here should match minimum version in CMakeLists.txt NIX_ARGS=(--argstr capnprotoVersion "0.7.1" --argstr cmakeVersion "3.12.4") BUILD_ARGS=(-k)