b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #215336 from Hjdskes/foundationdb

Browse source 
Foundationdb: package 7.1.26
This commit is contained in:
Weijia Wang 2023-04-26 15:00:28 +03:00 committed by GitHub
parent 7851415cf3 73ca83e965
commit 1d5ff2163e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 271 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json
View file

@ -5773,6 +5773,9 @@
"foundationdb61": [
"setuptools"
],
"foundationdb71": [
"setuptools"
],
"fountains": [
"setuptools"
],

91
pkgs/servers/foundationdb/cmake.nix
View file

@ -1,7 +1,8 @@
# This builder is for FoundationDB CMake build system.
{ lib, fetchFromGitHub
, cmake, ninja, boost, python3, openjdk, mono, libressl
, cmake, ninja, python3, openjdk, mono, pkg-config
, msgpack, toml11
, gccStdenv, llvmPackages
, useClang ? false
@ -11,16 +12,23 @@
let
stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv;
tests = with builtins;
builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);
tests = builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);
# Only even numbered versions compile on aarch64; odd numbered versions have avx enabled.
avxEnabled = version:
let
isOdd = n: lib.trivial.mod n 2 != 0;
patch = lib.toInt (lib.versions.patch version);
in isOdd patch;
makeFdb =
{ version
, branch # unused
, sha256
, rev ? "refs/tags/${version}"
, officialRelease ? true
, patches ? []
, boost
, ssl
}: stdenv.mkDerivation {
pname = "foundationdb";
inherit version;
@ -31,23 +39,28 @@ let
inherit rev sha256;
};
buildInputs = [ libressl boost ];
nativeBuildInputs = [ cmake ninja python3 openjdk mono ]
buildInputs = [ ssl boost ]
++ lib.optionals (lib.versionAtLeast version "7.1.0") [ msgpack toml11 ];
nativeBuildInputs = [ pkg-config cmake ninja python3 openjdk mono ]
++ lib.optionals useClang [ llvmPackages.lld ];
separateDebugInfo = true;
dontFixCmake = true;
cmakeFlags =
[ "-DCMAKE_BUILD_TYPE=Release"
(lib.optionalString officialRelease "-DFDB_RELEASE=TRUE")
[ (lib.optionalString officialRelease "-DFDB_RELEASE=TRUE")
# FIXME: why can't libressl be found automatically?
"-DLIBRESSL_USE_STATIC_LIBS=FALSE"
"-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"
"-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"
"-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"
"-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"
# Disable CMake warnings for project developers.
"-Wno-dev"
# CMake Error at fdbserver/CMakeLists.txt:332 (find_library):
# > Could not find lz4_STATIC_LIBRARIES using the following names: liblz4.a
"-DSSD_ROCKSDB_EXPERIMENTAL=FALSE"
# FoundationDB's CMake is hardcoded to pull in jemalloc as an external
# project at build time.
"-DUSE_JEMALLOC=FALSE"
# LTO brings up overall build time, but results in much smaller
# binaries for all users and the cache.
@ -58,40 +71,47 @@ let
# Same with LLD when Clang is available.
(lib.optionalString useClang "-DUSE_LD=LLD")
(lib.optionalString (!useClang) "-DUSE_LD=GOLD")
] ++ lib.optionals (lib.versionOlder version "7.0.0")
[ # FIXME: why can't libressl be found automatically?
"-DLIBRESSL_USE_STATIC_LIBS=FALSE"
"-DLIBRESSL_INCLUDE_DIR=${ssl.dev}"
"-DLIBRESSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"
"-DLIBRESSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"
"-DLIBRESSL_TLS_LIBRARY=${ssl.out}/lib/libtls.so"
] ++ lib.optionals (lib.versionAtLeast version "7.1.0" && lib.versionOlder version "7.2.0")
[ # FIXME: why can't openssl be found automatically?
"-DOPENSSL_USE_STATIC_LIBS=FALSE"
"-DOPENSSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"
"-DOPENSSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"
];
env.NIX_CFLAGS_COMPILE = toString [
# Needed with GCC 12
"-Wno-error=missing-template-keyword"
# Needed to compile on aarch64
(lib.optionalString stdenv.isAarch64 "-march=armv8-a+crc")
];
inherit patches;
# fix up the use of the very weird and custom 'fdb_install' command by just
# replacing it with cmake's ordinary version.
postPatch = ''
for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do
substituteInPlace $x --replace 'fdb_install' 'install'
done
'';
# the install phase for cmake is pretty wonky right now since it's not designed to
# coherently install packages as most linux distros expect -- it's designed to build
# packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to
# cmake upstream to fix this, and if we do, i think most of this can go away.
postInstall = ''
mv $out/sbin/fdbserver $out/bin/fdbserver
rm -rf \
$out/lib/systemd $out/Library $out/usr $out/sbin \
$out/var $out/log $out/etc
postInstall = lib.optionalString (lib.versionOlder version "7.0.0") ''
mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor
rm -rf $out/lib/foundationdb/
mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
rm -rf $out/Library
rm -rf $out/lib/foundationdb/
mkdir $out/include/foundationdb && \
mv $out/include/*.h $out/include/*.options $out/include/foundationdb
'' + lib.optionalString (lib.versionAtLeast version "7.0.0") ''
mv $out/sbin/fdbmonitor $out/bin/fdbmonitor
mkdir $out/libexec && mv $out/usr/lib/foundationdb/backup_agent/backup_agent $out/libexec/backup_agent
'' + ''
mv $out/sbin/fdbserver $out/bin/fdbserver
rm -rf $out/etc $out/lib/foundationdb $out/lib/systemd $out/log $out/sbin $out/usr $out/var
# move results into multi outputs
mkdir -p $dev $lib
@ -113,12 +133,6 @@ let
# java bindings
mkdir -p $lib/share/java
mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar
# include the tests
mkdir -p $out/share/test
(cd ../tests && for x in ${tests}; do
cp --parents $x $out/share/test
done)
'';
outputs = [ "out" "dev" "lib" "pythonsrc" ];
@ -127,7 +141,8 @@ let
description = "Open source, distributed, transactional key-value store";
homepage = "https://www.foundationdb.org";
license = licenses.asl20;
platforms = [ "x86_64-linux" ];
platforms = [ "x86_64-linux" ]
++ lib.optionals (lib.versionAtLeast version "7.1.0" && !(avxEnabled version)) [ "aarch64-linux" ];
maintainers = with maintainers; [ thoughtpolice lostnet ];
};
};

25
pkgs/servers/foundationdb/default.nix
View file

@ -2,16 +2,13 @@
, lib, fetchurl, fetchpatch, fetchFromGitHub
, cmake, ninja, which, findutils, m4, gawk
, python2, python3, openjdk, mono, libressl, boost168
, python2, python3, openjdk, mono, libressl, openssl, boost168, boost178
, pkg-config, msgpack, toml11
}@args:
let
vsmakeBuild = import ./vsmake.nix args;
cmakeBuild = import ./cmake.nix (args // {
gccStdenv = gccStdenv;
llvmPackages = llvmPackages;
boost = boost168;
});
cmakeBuild = import ./cmake.nix args;
python3-six-patch = fetchpatch {
name = "update-python-six.patch";
@ -79,8 +76,9 @@ in with builtins; {
foundationdb61 = cmakeBuild {
version = "6.1.13";
branch = "release-6.1";
sha256 = "10vd694dcnh2pp91mri1m80kfbwjanhiy50c53c5ncqfa6pwvk00";
boost = boost168;
ssl = libressl;
patches = [
./patches/clang-libcxx.patch
@ -90,4 +88,17 @@ in with builtins; {
];
};
foundationdb71 = cmakeBuild {
version = "7.1.26";
sha256 = "sha256-IVUFC2Z/nJAeKr/TtEiHAo+1HUeZuSZ2birwJtiYZx0=";
boost = boost178;
ssl = openssl;
patches = [
./patches/disable-flowbench.patch
./patches/don-t-run-tests-requiring-doctest.patch
./patches/don-t-use-static-boost-libs.patch
./patches/fix-open-with-O_CREAT.patch
];
};
}

15
pkgs/servers/foundationdb/patches/disable-flowbench.patch Normal file
View file

@ -0,0 +1,15 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 185b721eb..6752ff32d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -185,7 +185,6 @@ endif()
add_subdirectory(fdbbackup)
add_subdirectory(contrib)
add_subdirectory(tests)
-add_subdirectory(flowbench EXCLUDE_FROM_ALL)
if(WITH_PYTHON AND WITH_C_BINDING)
add_subdirectory(bindings)
endif()
--
2.38.1

128
pkgs/servers/foundationdb/patches/don-t-run-tests-requiring-doctest.patch Normal file
View file

@ -0,0 +1,128 @@
From 10c502fd36df24f1fdbdeff446982ff5247ba20e Mon Sep 17 00:00:00 2001
From: Jente Hidskes Ankarberg <jente@griffin.sh>
Date: Thu, 9 Feb 2023 12:40:21 +0100
Subject: [PATCH] Don't run tests requiring doctest
Doctest is unconditionally pulled in as an external project, which we can't do
---
bindings/c/CMakeLists.txt | 59 ---------------------------------------
1 file changed, 59 deletions(-)
diff --git bindings/c/CMakeLists.txt bindings/c/CMakeLists.txt
index b1a187b99..25b626819 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -84,7 +84,6 @@ if(NOT WIN32)
test/mako/mako.h
test/mako/utils.c
test/mako/utils.h)
- add_subdirectory(test/unit/third_party)
find_package(Threads REQUIRED)
set(UNIT_TEST_SRCS
test/unit/unit_tests.cpp
@@ -93,10 +92,6 @@ if(NOT WIN32)
set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
- set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS
- test/unit/disconnected_timeout_tests.cpp
- test/unit/fdb_api.cpp
- test/unit/fdb_api.hpp)
set(API_TESTER_SRCS
test/apitester/fdb_c_api_tester.cpp
@@ -128,11 +123,7 @@ if(NOT WIN32)
add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)
add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
add_library(mako OBJECT ${MAKO_SRCS})
- add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
- add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
- add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
- add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
add_library(fdb_c_api_tester OBJECT ${API_TESTER_SRCS})
else()
add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
@@ -140,11 +131,7 @@ if(NOT WIN32)
add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)
add_executable(fdb_c_client_memory_test test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
add_executable(mako ${MAKO_SRCS})
- add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
- add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
- add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
- add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
add_executable(fdb_c_api_tester ${API_TESTER_SRCS})
strip_debug_symbols(fdb_c_performance_test)
strip_debug_symbols(fdb_c_ryw_benchmark)
@@ -155,20 +142,7 @@ if(NOT WIN32)
target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
-
- add_dependencies(fdb_c_setup_tests doctest)
- add_dependencies(fdb_c_unit_tests doctest)
- add_dependencies(fdb_c_unit_tests_version_510 doctest)
- add_dependencies(disconnected_timeout_unit_tests doctest)
- target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
- target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
- target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
- target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
- target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
- target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient)
- target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)
- target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
if(USE_SANITIZER)
target_link_libraries(fdb_c_api_tester PRIVATE fdb_c toml11_target Threads::Threads fmt::fmt boost_asan)
@@ -203,46 +177,13 @@ endif()
DEPENDS fdb_c
COMMENT "Copy libfdb_c to use as external client for test")
add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so)
- add_dependencies(fdb_c_unit_tests external_client)
- add_dependencies(disconnected_timeout_unit_tests external_client)
add_dependencies(fdb_c_api_tester external_client)
- add_fdbclient_test(
- NAME fdb_c_setup_tests
- COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
- add_fdbclient_test(
- NAME fdb_c_unit_tests
- COMMAND $<TARGET_FILE:fdb_c_unit_tests>
- @CLUSTER_FILE@
- fdb)
- add_fdbclient_test(
- NAME fdb_c_unit_tests_version_510
- COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>
- @CLUSTER_FILE@
- fdb)
add_fdbclient_test(
NAME trace_partial_file_suffix_test
COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>
@CLUSTER_FILE@
fdb)
- add_fdbclient_test(
- NAME fdb_c_external_client_unit_tests
- COMMAND $<TARGET_FILE:fdb_c_unit_tests>
- @CLUSTER_FILE@
- fdb
- ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
- )
- add_unavailable_fdbclient_test(
- NAME disconnected_timeout_unit_tests
- COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
- @CLUSTER_FILE@
- )
- add_unavailable_fdbclient_test(
- NAME disconnected_timeout_external_client_unit_tests
- COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
- @CLUSTER_FILE@
- ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
- )
add_fdbclient_test(
NAME fdb_c_api_tests
DISABLE_LOG_DUMP
--
2.37.1 (Apple Git-137.1)

27
pkgs/servers/foundationdb/patches/don-t-use-static-boost-libs.patch Normal file
View file

@ -0,0 +1,27 @@
From 1a217164f8086137ce175da09329745d5ea63027 Mon Sep 17 00:00:00 2001
From: Jente Hidskes Ankarberg <jente@griffin.sh>
Date: Tue, 7 Feb 2023 17:17:16 +0100
Subject: Don't use static Boost libs
We cannot override this in our CMake flags, hence we have to patch it in the source.
---
cmake/CompileBoost.cmake | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git cmake/CompileBoost.cmake cmake/CompileBoost.cmake
index 3bc47c776..62b448421 100644
--- a/cmake/CompileBoost.cmake
+++ b/cmake/CompileBoost.cmake
@@ -85,7 +85,7 @@ if(USE_SANITIZER)
endif()
# since boost 1.72 boost installs cmake configs. We will enforce config mode
-set(Boost_USE_STATIC_LIBS ON)
+set(Boost_USE_STATIC_LIBS OFF)
# Clang and Gcc will have different name mangling to std::call_once, etc.
if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$")
--
2.37.1 (Apple Git-137.1)

25
pkgs/servers/foundationdb/patches/fix-open-with-O_CREAT.patch Normal file
View file

@ -0,0 +1,25 @@
From 533d064ce028a7ebd0ef3a845cb69f10ca04b09f Mon Sep 17 00:00:00 2001
From: Jente Hidskes Ankarberg <jente@griffin.sh>
Date: Fri, 17 Feb 2023 00:07:20 +0100
Subject: [PATCH 2/2] fix open with O_CREAT
---
fdbbackup/FileDecoder.actor.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git fdbbackup/FileDecoder.actor.cpp fdbbackup/FileDecoder.actor.cpp
index 71f693259..547147ea0 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -433,7 +433,7 @@ public:
platform::createDirectory(path);
}
}
- self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
+ self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (self->lfd == -1) {
TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
throw platform_error();
--
2.38.1

1
pkgs/top-level/all-packages.nix
View file

@ -7652,6 +7652,7 @@ with pkgs;
foundationdb52
foundationdb60
foundationdb61
foundationdb71
;
foundationdb = foundationdb61;

1
pkgs/top-level/python-packages.nix
View file

@ -3714,6 +3714,7 @@ self: super: with self; {
foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; };
foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; };
foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; };
foundationdb71 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb71; };
fountains = callPackage ../development/python-modules/fountains { };