From dbb412729b2c8bc9567565d7d7832a47150c5c37 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Fri, 2 Feb 2024 11:15:48 +0100 Subject: [PATCH] chore: macOS build tweaks --- CMakeLists.txt | 76 +++++++++++++++++++++++++++++++-------------- README.md | 67 +++++++++++++++++---------------------- cmake/osx.sh | 49 ----------------------------- test/tools_test.cpp | 36 +++++++++++++++++++-- 4 files changed, 115 insertions(+), 113 deletions(-) delete mode 100755 cmake/osx.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index f7ad3919..ba3667b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,10 @@ option(WITH_MAN_OPTION "build with --man option" ON) option(ENABLE_PERFMON "enable performance monitor in all tools" ON) option(ENABLE_FLAC "build with FLAC support" ON) option(ENABLE_RICEPP "build with RICEPP compression support" ON) +option(WITH_UNIVERSAL_BINARY "build with universal binary" ON) +if(APPLE) + option(USE_HOMEBREW_LIBARCHIVE "use libarchive from homebrew" ON) +endif() if(WIN32) set(PREFER_SYSTEM_LIBFMT ON) set(PREFER_SYSTEM_ZSTD ON) @@ -230,6 +234,15 @@ find_package(PkgConfig) if(APPLE) # For whatever reason, thrift is unhappy if we don't do this find_package(OpenSSL 1.1.1 MODULE REQUIRED) + + if(USE_HOMEBREW_LIBARCHIVE) + find_program(HOMEBREW_EXE brew) + execute_process( + COMMAND ${HOMEBREW_EXE} --prefix libarchive + OUTPUT_VARIABLE LIBARCHIVE_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + list(PREPEND CMAKE_PREFIX_PATH ${LIBARCHIVE_PREFIX}) + endif() endif() if(STATIC_BUILD_DO_NOT_USE) @@ -576,22 +589,28 @@ add_executable(mkdwarfs src/mkdwarfs.cpp) add_executable(dwarfsck src/dwarfsck.cpp) add_executable(dwarfsbench src/dwarfsbench.cpp) add_executable(dwarfsextract src/dwarfsextract.cpp) -add_executable(dwarfsuniversal src/universal.cpp) +if(WITH_UNIVERSAL_BINARY) + add_executable(dwarfsuniversal src/universal.cpp) +endif() target_link_libraries(mkdwarfs mkdwarfs_main) target_link_libraries(dwarfsck dwarfsck_main) target_link_libraries(dwarfsbench dwarfsbench_main) target_link_libraries(dwarfsextract dwarfsextract_main) -target_link_libraries(dwarfsuniversal mkdwarfs_main dwarfsck_main - dwarfsextract_main) -set_target_properties(dwarfsuniversal PROPERTIES - RUNTIME_OUTPUT_DIRECTORY universal - OUTPUT_NAME dwarfs-universal) +if(WITH_UNIVERSAL_BINARY) + target_link_libraries(dwarfsuniversal mkdwarfs_main dwarfsck_main + dwarfsextract_main) + set_target_properties(dwarfsuniversal PROPERTIES + RUNTIME_OUTPUT_DIRECTORY universal + OUTPUT_NAME dwarfs-universal) +endif() install(TARGETS mkdwarfs dwarfsck dwarfsbench dwarfsextract RUNTIME DESTINATION bin) list(APPEND BINARY_TARGETS mkdwarfs dwarfsck dwarfsbench dwarfsextract) -list(APPEND BINARY_TARGETS dwarfsuniversal) +if(WITH_UNIVERSAL_BINARY) + list(APPEND BINARY_TARGETS dwarfsuniversal) +endif() list(APPEND MAIN_TARGETS mkdwarfs_main dwarfsck_main dwarfsbench_main dwarfsextract_main) @@ -603,8 +622,10 @@ if(FUSE3_FOUND OR WINFSP OR APPLE) DWARFS_FUSE_LOWLEVEL=0) target_include_directories(dwarfs_main SYSTEM PRIVATE "${WINFSP_PATH}/inc") target_link_libraries(dwarfs_main ${WINFSP}) - target_link_libraries(dwarfsuniversal delayimp.lib) - target_link_options(dwarfsuniversal PRIVATE /DELAYLOAD:winfsp-x64.dll) + if(WITH_UNIVERSAL_BINARY) + target_link_libraries(dwarfsuniversal delayimp.lib) + target_link_options(dwarfsuniversal PRIVATE /DELAYLOAD:winfsp-x64.dll) + endif() elseif(APPLE) target_compile_definitions(dwarfs_main PRIVATE FUSE_USE_VERSION=29) target_link_libraries(dwarfs_main PkgConfig::FUSE) @@ -615,7 +636,9 @@ if(FUSE3_FOUND OR WINFSP OR APPLE) add_executable(dwarfs-bin src/dwarfs.cpp) target_link_libraries(dwarfs-bin dwarfs_main) set_target_properties(dwarfs-bin PROPERTIES OUTPUT_NAME dwarfs) - target_link_libraries(dwarfsuniversal dwarfs_main) + if(WITH_UNIVERSAL_BINARY) + target_link_libraries(dwarfsuniversal dwarfs_main) + endif() if(WINFSP) install(TARGETS dwarfs-bin RUNTIME DESTINATION bin) else() @@ -637,7 +660,7 @@ if(FUSE_FOUND AND (NOT APPLE) AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND)) target_link_libraries(dwarfs2_main PkgConfig::FUSE) add_executable(dwarfs2-bin src/dwarfs.cpp) target_link_libraries(dwarfs2-bin dwarfs2_main) - if(NOT FUSE3_FOUND) + if(WITH_UNIVERSAL_BINARY AND (NOT FUSE3_FOUND)) target_link_libraries(dwarfsuniversal dwarfs2_main) endif() set_target_properties(dwarfs2-bin PROPERTIES OUTPUT_NAME dwarfs2) @@ -714,6 +737,11 @@ if(WITH_TESTS) list(APPEND TEST_TARGETS ${test}) endforeach() + set_source_files_properties(test/tools_test.cpp PROPERTIES + COMPILE_DEFINITIONS + $<$:DWARFS_HAVE_UNIVERSAL_BINARY> + ) + foreach(tgt ${TEST_TARGETS}) gtest_discover_tests(${tgt} DISCOVERY_TIMEOUT 120) endforeach() @@ -1145,21 +1173,23 @@ if(PRJ_VERSION_FULL) endif() if(STATIC_BUILD_DO_NOT_USE OR WIN32) - find_program(UPX_EXE upx upx.exe PATHS "c:/bin" DOC "ultimate packer for executables" REQUIRED) + if (WITH_UNIVERSAL_BINARY) + find_program(UPX_EXE upx upx.exe PATHS "c:/bin" DOC "ultimate packer for executables" REQUIRED) - set(UNIVERSAL_PACKED - "dwarfs-universal-${DWARFS_ARTIFACT_ID}${CMAKE_EXECUTABLE_SUFFIX}") + set(UNIVERSAL_PACKED + "dwarfs-universal-${DWARFS_ARTIFACT_ID}${CMAKE_EXECUTABLE_SUFFIX}") - # upx -9 is a good compromise between compression ratio and speed - # also, anything above --best increases the startup time of the compressed - # executable significantly - add_custom_command( - OUTPUT ${UNIVERSAL_PACKED} - COMMAND ${UPX_EXE} -9 -o ${UNIVERSAL_PACKED} $ - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) + # upx -9 is a good compromise between compression ratio and speed + # also, anything above --best increases the startup time of the compressed + # executable significantly + add_custom_command( + OUTPUT ${UNIVERSAL_PACKED} + COMMAND ${UPX_EXE} -9 -o ${UNIVERSAL_PACKED} $ + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) - add_custom_target(universal_upx DEPENDS ${UNIVERSAL_PACKED}) + add_custom_target(universal_upx DEPENDS ${UNIVERSAL_PACKED}) + endif() if(DEFINED ENV{GITHUB_REF_TYPE}) message(STATUS "GITHUB_REF_TYPE: $ENV{GITHUB_REF_TYPE}") diff --git a/README.md b/README.md index b59641c2..abe3c35e 100644 --- a/README.md +++ b/README.md @@ -519,63 +519,54 @@ The macOS version of the DwarFS filesystem driver relies on the awesome ### Building on macOS Building on macOS involves a few steps, but should be relatively -straightforward. This has so far only been tested on a 2014 iMac running -macOS Big Sur. However, it seems to be possible to build universal (fat) -binaries even on such an old platform. +straightforward: -- First, install [macFUSE](https://https://osxfuse.github.io/). +- Install [Homebrew](https://brew.sh/) -- Install the [mistletoe](https://pypi.org/project/mistletoe/) Python - library, e.g.: +- Use Homebrew to install the necessary dependencies: ``` -$ pip3 install --user mistletoe +$ brew install cmake ninja ronn macfuse python3 brotli howard-hinnant-date \ + double-conversion fmt glog libarchive libevent flac openssl \ + pkg-config range-v3 utf8cpp xxhash boost zstd jemalloc ``` -- Install `cmake`, `ninja` and `ronn` using Homebrew (you can - optionally also install `ccache`): +- When installing macFUSE for the first time, you'll need to explicitly + allow the sofware in *System Preferences* / *Privacy & Security*. It's + quite likely that you'll have to reboot after this. + +- Clone the DwarFS repository: ``` -$ brew install cmake -$ brew install ninja -$ brew install ronn -``` - -- Clone the [vcpkg](https://vcpkg.io/), - [lipo-dir-merge](https://github.com/faaxm/lipo-dir-merge) and - DwarFS repositories. If you want the files to go to a different - location, you'll need set `VCPKG_BASEDIR` later. - -``` -$ cd ~ -$ mkdir git -$ cd git -$ git clone https://github.com/Microsoft/vcpkg.git -$ git clone https://github.com/faaxm/lipo-dir-merge $ git clone --recurse-submodules https://github.com/mhx/dwarfs ``` -- Bootstrap `vcpkg`: - -``` -$ ./vcpkg/bootstrap-vcpkg.sh -``` - -- Build DwarFS: +- Prepare the build by installing the `mistletoe` python module + in a virtualenv: ``` $ cd dwarfs -$ git checkout experimental/osx-build -$ mkdir build -$ cd build -$ export VCPKG_BASEDIR=$HOME/git # optional -$ sh ../cmake/osx.sh rebuild-vcpkg -$ sh ../cmake/osx.sh +$ python3 -m venv @buildenv +$ source ./@buildenv/bin/activate +$ pip3 install mistletoe +``` + +- Build DwarFS and run its tests: + +``` +$ mkdir build && cd build +$ cmake .. -GNinja -DWITH_TESTS=ON $ ninja $ export CTEST_PARALLEL_LEVEL=$(sysctl -n hw.logicalcpu) $ ninja test ``` +- Install DwarFS: + +``` +$ ninja install +``` + That's it! ## Extended Attributes diff --git a/cmake/osx.sh b/cmake/osx.sh deleted file mode 100755 index cd76dabc..00000000 --- a/cmake/osx.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -z "${VCPKG_BASEDIR}" ]]; then - VCPKG_BASEDIR=$HOME/git -fi - -if [[ -z "${VCPKG_ROOT}" ]]; then - VCPKG_ROOT=$VCPKG_BASEDIR/vcpkg -fi - -if [[ -z "${VCPKG_INSTALL_ROOT}" ]]; then - VCPKG_INSTALL_ROOT=$VCPKG_BASEDIR/@vcpkg-install -fi - -if [[ -z "${LIPO_DIR_MERGE}" ]]; then - LIPO_DIR_MERGE=$VCPKG_BASEDIR/lipo-dir-merge/lipo-dir-merge.py -fi - -build_mode=$1 - -if [[ -z $build_mode ]]; then - build_mode="Release" -fi - -if [[ $build_mode == "rebuild-vcpkg" ]]; then - for triplet in x64-osx arm64-osx; do - rm -rf $VCPKG_INSTALL_ROOT/tmp/$triplet - $VCPKG_ROOT/vcpkg install --triplet=$triplet --x-install-root=$VCPKG_INSTALL_ROOT/tmp/$triplet - done - - rm -rf $VCPKG_INSTALL_ROOT/uni-osx - - echo "merging x64-osx and arm64-osx to uni-osx..." - - python3 $LIPO_DIR_MERGE \ - $VCPKG_INSTALL_ROOT/tmp/x64-osx/x64-osx \ - $VCPKG_INSTALL_ROOT/tmp/arm64-osx/arm64-osx \ - $VCPKG_INSTALL_ROOT/uni-osx - - echo "DONE" -else - cmake .. -GNinja \ - -DWITH_TESTS=ON -DPREFER_SYSTEM_ZSTD=ON -DUSE_JEMALLOC=OFF \ - -DCMAKE_BUILD_TYPE=$build_mode \ - -DCMAKE_PREFIX_PATH=$VCPKG_INSTALL_ROOT/uni-osx \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -fi diff --git a/test/tools_test.cpp b/test/tools_test.cpp index c5ec1a63..d15b5be9 100644 --- a/test/tools_test.cpp +++ b/test/tools_test.cpp @@ -345,6 +345,21 @@ struct new_process_group : public ::boost::process::detail::handler_base { }; #endif +void ignore_sigpipe() { +#ifdef __APPLE__ + static bool already_ignoring{false}; + + if (!already_ignoring) { + struct sigaction sa; + std::memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_IGN; + int res = ::sigaction(SIGPIPE, &sa, NULL); + assert(res == 0); + already_ignoring = true; + } +#endif +} + class subprocess { public: template @@ -352,6 +367,8 @@ class subprocess { : prog_{prog} { (append_arg(cmdline_, std::forward(args)), ...); + ignore_sigpipe(); + try { // std::cerr << "running: " << cmdline() << "\n"; c_ = bp::child(prog.string(), bp::args(cmdline_), bp::std_in.close(), @@ -755,8 +772,10 @@ std::ostream& operator<<(std::ostream& os, binary_mode m) { std::vector tools_test_modes{ binary_mode::standalone, +#ifdef DWARFS_HAVE_UNIVERSAL_BINARY binary_mode::universal_tool, binary_mode::universal_symlink, +#endif }; class tools_test : public ::testing::TestWithParam {}; @@ -1556,11 +1575,22 @@ TEST_P(manpage_test, manpage) { EXPECT_THAT(*out, ::testing::HasSubstr("COPYRIGHT")); } +namespace { + +std::vector manpage_test_modes{ + binary_mode::standalone, +#ifdef DWARFS_HAVE_UNIVERSAL_BINARY + binary_mode::universal_tool, +#endif +}; + +} // namespace + INSTANTIATE_TEST_SUITE_P( dwarfs, manpage_test, - ::testing::Combine( - ::testing::Values(binary_mode::standalone, binary_mode::universal_tool), - ::testing::Values("dwarfs", "mkdwarfs", "dwarfsck", "dwarfsextract"))); + ::testing::Combine(::testing::ValuesIn(manpage_test_modes), + ::testing::Values("dwarfs", "mkdwarfs", "dwarfsck", + "dwarfsextract"))); #endif TEST(tools_test, dwarfsextract_progress) {