From 3f1c6801a2d30552c06079c97d3e427854f7029a Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Thu, 2 May 2019 21:44:11 -0500 Subject: [PATCH] foundationdb: init 6.1.6pre4898_26fbbbf, cmake build This adds a new build of FoundationDB 6.1, using the new, much improved with CMake build system with fewer patches and rough edges. Signed-off-by: Austin Seipp --- pkgs/servers/foundationdb/cmake.nix | 129 ++++++++++++++++++ pkgs/servers/foundationdb/default.nix | 31 ++++- .../foundationdb/patches/clang-libcxx.patch | 52 +++++++ .../patches/suppress-clang-warnings.patch | 34 +++++ pkgs/servers/foundationdb/test-list.txt | 80 +++++++++++ pkgs/servers/foundationdb/vsmake.nix | 1 + pkgs/top-level/all-packages.nix | 4 +- pkgs/top-level/python-packages.nix | 1 + 8 files changed, 328 insertions(+), 4 deletions(-) create mode 100644 pkgs/servers/foundationdb/cmake.nix create mode 100644 pkgs/servers/foundationdb/patches/clang-libcxx.patch create mode 100644 pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch create mode 100644 pkgs/servers/foundationdb/test-list.txt diff --git a/pkgs/servers/foundationdb/cmake.nix b/pkgs/servers/foundationdb/cmake.nix new file mode 100644 index 00000000000..ae693a49027 --- /dev/null +++ b/pkgs/servers/foundationdb/cmake.nix @@ -0,0 +1,129 @@ +# This builder is for FoundationDB CMake build system. + +{ lib, fetchurl, fetchpatch, fetchFromGitHub +, cmake, ninja, boost, python3, openjdk, mono, libressl + +, gccStdenv, llvmPackages +, useClang ? true +, ... +}: + +let + stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv; + + tests = with builtins; + builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt); + + makeFdb = + { version + , branch + , sha256 + , rev ? "refs/tags/${version}" + , officialRelease ? true + , patches ? [] + }: stdenv.mkDerivation rec { + name = "foundationdb-${version}"; + inherit version; + + src = fetchFromGitHub { + owner = "apple"; + repo = "foundationdb"; + inherit rev sha256; + }; + + buildInputs = [ libressl boost ]; + nativeBuildInputs = [ cmake ninja python3 openjdk mono ] + ++ lib.optional useClang [ llvmPackages.lld ]; + + separateDebugInfo = true; + enableParallelBuilding = true; + dontFixCmake = true; + + cmakeFlags = + [ "-DCMAKE_BUILD_TYPE=Release" + "-DLIBRESSL_USE_STATIC_LIBS=FALSE" + + # CMake can't find these easily for some reason? + "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}" + "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so" + "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so" + "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so" + + # LTO brings up overall build time, but results in much smaller + # binaries for all users and the cache. + # + # TODO FIXME: bugs :( + (lib.optionalString (!useClang) "-DUSE_LTO=ON") + + # Gold helps alleviate the link time, especially when LTO is + # enabled. But even then, it still takes a majority of the time. + # Same with LLD when Clang is available. + (lib.optionalString useClang "-DUSE_LD=LLD") + (lib.optionalString (!useClang) "-DUSE_LD=GOLD") + ] + ++ lib.optional officialRelease [ "FDB_RELEASE=1" ]; + + inherit patches; + postPatch = '' + for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do + substituteInPlace $x --replace 'fdb_install' 'install' + done + ''; + + # the install phase for cmake is pretty wonky right now since it's not designed to + # coherently install packages as most linux distros expect -- it's designed to build + # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to + # cmake upstream to fix this, and if we do, i think most of this can go away. + postInstall = '' + mv $out/sbin/fdbserver $out/bin/fdbserver + rm -rf \ + $out/lib/systemd $out/Library $out/usr $out/sbin \ + $out/var $out/log $out/etc + + mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor + + rm -rf $out/lib/foundationdb/ + mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent + + mkdir $out/include/foundationdb && \ + mv $out/include/*.h $out/include/*.options $out/include/foundationdb + + # move results into multi outputs + mkdir -p $dev $lib + mv $out/include $dev/include + mv $out/lib $lib/lib + + # python bindings + # NB: use the original setup.py.in, so we can substitute VERSION correctly + cp ../LICENSE ./bindings/python + substitute ../bindings/python/setup.py.in ./bindings/python/setup.py \ + --replace 'VERSION' "${version}" + rm -f ./bindings/python/setup.py.* ./bindings/python/CMakeLists.txt + rm -f ./bindings/python/fdb/*.pth # remove useless files + rm -f ./bindings/python/*.rst ./bindings/python/*.mk + + cp -R ./bindings/python/ tmp-pythonsrc/ + tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/ + + # java bindings + mkdir -p $lib/share/java + mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar + + # include the tests + mkdir -p $out/share/test + (cd ../tests && for x in ${tests}; do + cp --parents $x $out/share/test + done) + ''; + + outputs = [ "out" "dev" "lib" "pythonsrc" ]; + + meta = with stdenv.lib; { + description = "Open source, distributed, transactional key-value store"; + homepage = https://www.foundationdb.org; + license = licenses.asl20; + platforms = [ "x86_64-linux" ]; + maintainers = with maintainers; [ thoughtpolice ]; + }; + }; +in makeFdb diff --git a/pkgs/servers/foundationdb/default.nix b/pkgs/servers/foundationdb/default.nix index b9b091f8476..790959f0a0d 100644 --- a/pkgs/servers/foundationdb/default.nix +++ b/pkgs/servers/foundationdb/default.nix @@ -1,12 +1,16 @@ -{ stdenv49 +{ stdenv, stdenv49, gcc9Stdenv, llvmPackages_8 , lib, fetchurl, fetchpatch, fetchFromGitHub -, which, findutils, m4, gawk -, python, openjdk, mono, libressl +, cmake, ninja, which, findutils, m4, gawk +, python, python3, openjdk, mono, libressl, boost }@args: let vsmakeBuild = import ./vsmake.nix args; + cmakeBuild = import ./cmake.nix (args // { + gccStdenv = gcc9Stdenv; + llvmPackages = llvmPackages_8; + }); python3-six-patch = fetchpatch { name = "update-python-six.patch"; @@ -21,6 +25,10 @@ let }; in with builtins; { + + # Older versions use the bespoke 'vsmake' build system + # ------------------------------------------------------ + foundationdb51 = vsmakeBuild rec { version = "5.1.7"; branch = "release-5.1"; @@ -56,4 +64,21 @@ in with builtins; { ./patches/ldflags-6.0.patch ]; }; + + # 6.1 and later versions should always use CMake + # ------------------------------------------------------ + + foundationdb61 = cmakeBuild rec { + version = "6.1.6pre4898_${substring 0 7 rev}"; + branch = "release-6.1"; + rev = "26fbbbf798971b2b9ecb882a8af766fa36734f53"; + sha256 = "1q1a1j8h0qlh67khcds0dg416myvjbp6gfm6s4sk8d60zfzny7wb"; + officialRelease = false; + + patches = [ + ./patches/clang-libcxx.patch + ./patches/suppress-clang-warnings.patch + ]; + }; + } diff --git a/pkgs/servers/foundationdb/patches/clang-libcxx.patch b/pkgs/servers/foundationdb/patches/clang-libcxx.patch new file mode 100644 index 00000000000..ebbacdf871e --- /dev/null +++ b/pkgs/servers/foundationdb/patches/clang-libcxx.patch @@ -0,0 +1,52 @@ +commit 7ed4745a092a203f92fc37ab5894e92117db0c94 +Author: Austin Seipp +Date: Sat May 4 15:23:35 2019 -0500 + + flow: fix a build failure with Clang/libcxx on Linux + + 11bd7d7da introduced a hack on Linux to work around a missing symbol in + libstdc++'s _pic library on Ubuntu. Unfortunately, this causes the build + to fail when using Clang, as it doesn't believe this symbol is part of + its headers in c++11 mode. + + Unfortunately there's no good way to distinguish libcxx from libstdc++ + with the preprocessor, so we merely gate it by only checking for clang, + iff we are on Linux. + + With this change, Clang 8.x can build FoundationDB on Linux using libcxx + as the standard C++ library. + + Signed-off-by: Austin Seipp + +diff --git a/flow/Platform.cpp b/flow/Platform.cpp +index 3d3f1ac0..9f21dfd4 100644 +--- a/flow/Platform.cpp ++++ b/flow/Platform.cpp +@@ -2841,13 +2841,26 @@ void setupSlowTaskProfiler() { + #endif + } + +-#ifdef __linux__ ++#if defined(__linux__) && !defined(__clang__) + // There's no good place to put this, so it's here. + // Ubuntu's packaging of libstdc++_pic offers different symbols than libstdc++. Go figure. + // Notably, it's missing a definition of std::istream::ignore(long), which causes compilation errors + // in the bindings. Thus, we provide weak versions of their definitions, so that if the + // linked-against libstdc++ is missing their definitions, we'll be able to use the provided + // ignore(long, int) version. ++// ++// Note that this hack is DISABLED when we use Clang. It is only needed when we statically link ++// to the _pic libraries, but only official FDB Linux binaries are built this way using GCC. If we ++// don't use the _pic libraries, then this hack is entirely unneeded -- likely the case when using ++// Clang on Linux. ++// ++// Doing this allows us to use LLVM's libc++ with Clang on Linux -- otherwise, providing ++// a weak symbol definition for an internal (non-public) class member fails (due to that member ++// being non-existant on libc++.) See upstream GitHub issue #1533 for more information. ++// ++// TODO FIXME: Obliterate this when the official build environment is upgraded beyond Ubuntu 14.04. ++// (This problem should be fixed in later LTS releases.) ++ + #include + namespace std { + typedef basic_istream> char_basic_istream; diff --git a/pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch b/pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch new file mode 100644 index 00000000000..9d1ae992efc --- /dev/null +++ b/pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch @@ -0,0 +1,34 @@ +commit 8076537a52bb026941f13f5542395aac69ef0825 +Author: Austin Seipp +Date: Sat May 4 17:34:51 2019 -0500 + + cmake: add workarounds for NixOS-specific deficiencies [NixOS] + + The NixOS debug builder hook adds '-Wa,--compress-debug-sections' to the + link flags (it actually adds it to the compiler flags, but the compiler + is used for linking, so...). This makes the compiler angry when -Werror + is passed, because it's unused at link-time (-Wa applies to the + assembler). Suppress this warning with -Wno-unused-command-line-argument + + NB: we *could* use -Wno-error=unused-command-line-argument, but that + still results in warnings anyway, just not fatal ones. We'd like to + remove them all for the sake of the build output. + + Signed-off-by: Austin Seipp + +diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake +index 03af9c10..7d059375 100644 +--- a/cmake/ConfigureCompiler.cmake ++++ b/cmake/ConfigureCompiler.cmake +@@ -119,6 +119,11 @@ else() + else() + add_compile_options(-Werror) + endif() ++ if (CLANG) ++ # aseipp: NixOS hack ++ add_compile_options(-Wno-unused-command-line-argument) ++ add_link_options(-Wno-unused-command-line-argument) ++ endif() + add_compile_options($<$:-Wno-pragmas>) + add_compile_options(-Wno-error=format + -Wunused-variable diff --git a/pkgs/servers/foundationdb/test-list.txt b/pkgs/servers/foundationdb/test-list.txt new file mode 100644 index 00000000000..3e9ce1428ad --- /dev/null +++ b/pkgs/servers/foundationdb/test-list.txt @@ -0,0 +1,80 @@ +fast/AtomicBackupCorrectness.txt +fast/AtomicBackupToDBCorrectness.txt +fast/AtomicOps.txt +fast/AtomicOpsApiCorrectness.txt +fast/BackupCorrectness.txt +fast/BackupCorrectnessClean.txt +fast/BackupToDBCorrectness.txt +fast/BackupToDBCorrectnessClean.txt +fast/CloggedSideband.txt +fast/ConstrainedRandomSelector.txt +fast/CycleAndLock.txt +fast/CycleTest.txt +fast/FuzzApiCorrectness.txt +fast/FuzzApiCorrectnessClean.txt +fast/IncrementTest.txt +fast/InventoryTestAlmostReadOnly.txt +fast/InventoryTestSomeWrites.txt +fast/KillRegionCycle.txt +fast/LongStackWriteDuringRead.txt +fast/LowLatency.txt +fast/MemoryLifetime.txt +fast/MoveKeysCycle.txt +fast/RandomSelector.txt +fast/RandomUnitTests.txt +fast/SelectorCorrectness.txt +fast/Sideband.txt +fast/SidebandWithStatus.txt +fast/SwizzledRollbackSideband.txt +fast/SystemRebootTestCycle.txt +fast/TaskBucketCorrectness.txt +fast/TimeKeeperCorrectness.txt +fast/Unreadable.txt +fast/VersionStamp.txt +fast/Watches.txt +fast/WriteDuringRead.txt +fast/WriteDuringReadClean.txt +rare/CheckRelocation.txt +rare/ClogUnclog.txt +rare/CloggedCycleWithKills.txt +rare/ConflictRangeCheck.txt +rare/ConflictRangeRYOWCheck.txt +rare/CycleRollbackClogged.txt +rare/CycleWithKills.txt +rare/FuzzTest.txt +rare/InventoryTestHeavyWrites.txt +rare/LargeApiCorrectness.txt +rare/LargeApiCorrectnessStatus.txt +rare/RYWDisable.txt +rare/RandomReadWriteTest.txt +rare/SwizzledLargeApiCorrectness.txt +slow/ApiCorrectness.txt +slow/ApiCorrectnessAtomicRestore.txt +slow/ApiCorrectnessSwitchover.txt +slow/ClogWithRollbacks.txt +slow/CloggedCycleTest.txt +slow/CloggedStorefront.txt +slow/CommitBug.txt +slow/ConfigureTest.txt +slow/CycleRollbackPlain.txt +slow/DDBalanceAndRemove.txt +slow/DDBalanceAndRemoveStatus.txt +slow/FastTriggeredWatches.txt +slow/LowLatencyWithFailures.txt +slow/MoveKeysClean.txt +slow/MoveKeysSideband.txt +slow/RyowCorrectness.txt +slow/Serializability.txt +slow/SharedBackupCorrectness.txt +slow/SharedBackupToDBCorrectness.txt +slow/StorefrontTest.txt +slow/SwizzledApiCorrectness.txt +slow/SwizzledCycleTest.txt +slow/SwizzledDdBalance.txt +slow/SwizzledRollbackTimeLapse.txt +slow/SwizzledRollbackTimeLapseIncrement.txt +slow/VersionStampBackupToDB.txt +slow/VersionStampSwitchover.txt +slow/WriteDuringReadAtomicRestore.txt +slow/WriteDuringReadSwitchover.txt +slow/ddbalance.txt diff --git a/pkgs/servers/foundationdb/vsmake.nix b/pkgs/servers/foundationdb/vsmake.nix index a834f04a086..9871afb0de6 100644 --- a/pkgs/servers/foundationdb/vsmake.nix +++ b/pkgs/servers/foundationdb/vsmake.nix @@ -5,6 +5,7 @@ , which, findutils, m4, gawk , python, openjdk, mono, libressl +, ... }: let diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f2dab6fdba3..c923ff2d9b4 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3026,7 +3026,9 @@ in inherit (fdbPackages) foundationdb51 foundationdb52 - foundationdb60; + foundationdb60 + foundationdb61 + ; foundationdb = foundationdb60; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index a32ce674fbf..a70f5309d4d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5528,6 +5528,7 @@ in { foundationdb51 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb51; }; foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; }; foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; }; + foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; }; libtorrentRasterbar = (toPythonModule (pkgs.libtorrentRasterbar.override { inherit python;