From a21ee82a38fa6a37800bc963e5943a31a614450e Mon Sep 17 00:00:00 2001 From: Christian Bourjau Date: Sun, 30 Jul 2023 11:55:11 +0200 Subject: [PATCH] onnxruntime: 1.13.1 -> 1.15.1 Update to latest version and build on Darwin. Release notes: https://github.com/microsoft/onnxruntime/releases/tag/v1.15.1 --- .../libraries/onnxruntime/default.nix | 150 ++++++++++++------ .../python-modules/onnxruntime/default.nix | 6 +- pkgs/top-level/all-packages.nix | 2 +- 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/pkgs/development/libraries/onnxruntime/default.nix b/pkgs/development/libraries/onnxruntime/default.nix index 0d36ce7ae42..99e45c1013d 100644 --- a/pkgs/development/libraries/onnxruntime/default.nix +++ b/pkgs/development/libraries/onnxruntime/default.nix @@ -1,59 +1,98 @@ { stdenv , lib , fetchFromGitHub +, fetchFromGitLab , fetchpatch , fetchurl -, pkg-config -, cmake -, python3Packages -, libpng -, zlib -, eigen -, protobuf -, howard-hinnant-date -, nlohmann_json -, boost -, oneDNN_2 +, Foundation , abseil-cpp -, gtest -, pythonSupport ? false +, cmake +, libpng +, nlohmann_json , nsync -, flatbuffers +, pkg-config +, python3Packages +, re2 +, zlib +, microsoft-gsl +, iconv +, gtest +, protobuf3_21 +, pythonSupport ? true }: -# Python Support -# -# When enabling Python support a wheel is made and stored in a `dist` output. -# This wheel is then installed in a separate derivation. -assert pythonSupport -> lib.versionOlder protobuf.version "3.20"; +let + howard-hinnant-date = fetchFromGitHub { + owner = "HowardHinnant"; + repo = "date"; + rev = "v2.4.1"; + sha256 = "sha256-BYL7wxsYRI45l8C3VwxYIIocn5TzJnBtU0UZ9pHwwZw="; + }; + eigen = fetchFromGitLab { + owner = "libeigen"; + repo = "eigen"; + rev = "d10b27fe37736d2944630ecd7557cefa95cf87c9"; + sha256 = "sha256-Lmco0s9gIm9sIw7lCr5Iewye3RmrHEE4HLfyzRkQCm0="; + }; + + mp11 = fetchFromGitHub { + owner = "boostorg"; + repo = "mp11"; + rev = "boost-1.79.0"; + sha256 = "sha256-ZxgPDLvpISrjpEHKpLGBowRKGfSwTf6TBfJD18yw+LM="; + }; + + safeint = fetchFromGitHub { + owner = "dcleblanc"; + repo = "safeint"; + rev = "ff15c6ada150a5018c5ef2172401cb4529eac9c0"; + sha256 = "sha256-PK1ce4C0uCR4TzLFg+elZdSk5DdPCRhhwT3LvEwWnPU="; + }; + + pytorch_cpuinfo = fetchFromGitHub { + owner = "pytorch"; + repo = "cpuinfo"; + # There are no tags in the repository + rev = "5916273f79a21551890fd3d56fc5375a78d1598d"; + sha256 = "sha256-nXBnloVTuB+AVX59VDU/Wc+Dsx94o92YQuHp3jowx2A="; + }; + + flatbuffers = fetchFromGitHub { + owner = "google"; + repo = "flatbuffers"; + rev = "v1.12.0"; + sha256 = "sha256-L1B5Y/c897Jg9fGwT2J3+vaXsZ+lfXnskp8Gto1p/Tg="; + }; + + gtest' = gtest.overrideAttrs (oldAttrs: rec { + version = "1.13.0"; + src = fetchFromGitHub { + owner = "google"; + repo = "googletest"; + rev = "v${version}"; + hash = "sha256-LVLEn+e7c8013pwiLzJiiIObyrlbBHYaioO/SWbItPQ="; + }; + }); +in stdenv.mkDerivation rec { pname = "onnxruntime"; - version = "1.13.1"; + version = "1.15.1"; src = fetchFromGitHub { owner = "microsoft"; repo = "onnxruntime"; rev = "v${version}"; - sha256 = "sha256-paaeq6QeiOzwiibbz0GkYZxEI/V80lvYNYTm6AuyAXQ="; + sha256 = "sha256-SnHo2sVACc++fog7Tg6f2LK/Sv/EskFzN7RZS7D113s="; fetchSubmodules = true; }; - patches = [ - # Use dnnl from nixpkgs instead of submodules - (fetchpatch { - name = "system-dnnl.patch"; - url = "https://aur.archlinux.org/cgit/aur.git/plain/system-dnnl.diff?h=python-onnxruntime&id=9c392fb542979981fe0026e0fe3cc361a5f00a36"; - sha256 = "sha256-+kedzJHLFU1vMbKO9cn8fr+9A5+IxIuiqzOfR2AfJ0k="; - }) - ]; - nativeBuildInputs = [ cmake pkg-config python3Packages.python - gtest + protobuf3_21 ] ++ lib.optionals pythonSupport (with python3Packages; [ setuptools wheel @@ -64,18 +103,26 @@ stdenv.mkDerivation rec { buildInputs = [ libpng zlib - howard-hinnant-date nlohmann_json - boost - oneDNN_2 - protobuf - ] ++ lib.optionals pythonSupport [ nsync + re2 + microsoft-gsl + ] ++ lib.optionals pythonSupport [ python3Packages.numpy python3Packages.pybind11 python3Packages.packaging + ] ++ lib.optionals stdenv.isDarwin [ + Foundation + iconv ]; + nativeCheckInputs = lib.optionals pythonSupport (with python3Packages; [ + gtest' + pytest + sympy + onnx + ]); + # TODO: build server, and move .so's to lib output # Python's wheel is stored in a separate dist output outputs = [ "out" "dev" ] ++ lib.optionals pythonSupport [ "dist" ]; @@ -85,15 +132,23 @@ stdenv.mkDerivation rec { cmakeDir = "../cmake"; cmakeFlags = [ - "-Donnxruntime_PREFER_SYSTEM_LIB=ON" - "-Donnxruntime_BUILD_SHARED_LIB=ON" - "-Donnxruntime_ENABLE_LTO=ON" - "-Donnxruntime_BUILD_UNIT_TESTS=ON" - "-Donnxruntime_USE_PREINSTALLED_EIGEN=ON" - "-Donnxruntime_USE_MPI=ON" - "-Deigen_SOURCE_PATH=${eigen.src}" + "-DCMAKE_BUILD_TYPE=RELEASE" + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + "-DFETCHCONTENT_QUIET=OFF" "-DFETCHCONTENT_SOURCE_DIR_ABSEIL_CPP=${abseil-cpp.src}" - "-Donnxruntime_USE_DNNL=YES" + "-DFETCHCONTENT_SOURCE_DIR_DATE=${howard-hinnant-date}" + "-DFETCHCONTENT_SOURCE_DIR_EIGEN=${eigen}" + "-DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=${flatbuffers}" + "-DFETCHCONTENT_SOURCE_DIR_GOOGLE_NSYNC=${nsync.src}" + "-DFETCHCONTENT_SOURCE_DIR_MP11=${mp11}" + "-DFETCHCONTENT_SOURCE_DIR_ONNX=${python3Packages.onnx.src}" + "-DFETCHCONTENT_SOURCE_DIR_PYTORCH_CPUINFO=${pytorch_cpuinfo}" + "-DFETCHCONTENT_SOURCE_DIR_SAFEINT=${safeint}" + "-DFETCHCONTENT_TRY_FIND_PACKAGE_MODE=ALWAYS" + "-Donnxruntime_BUILD_SHARED_LIB=ON" + "-Donnxruntime_BUILD_UNIT_TESTS=ON" + "-Donnxruntime_ENABLE_LTO=ON" + "-Donnxruntime_USE_FULL_PROTOBUF=OFF" ] ++ lib.optionals pythonSupport [ "-Donnxruntime_ENABLE_PYTHON=ON" ]; @@ -103,6 +158,9 @@ stdenv.mkDerivation rec { postPatch = '' substituteInPlace cmake/libonnxruntime.pc.cmake.in \ --replace '$'{prefix}/@CMAKE_INSTALL_ @CMAKE_INSTALL_ + '' + lib.optionalString (stdenv.hostPlatform.system == "aarch64-linux") '' + # https://github.com/NixOS/nixpkgs/pull/226734#issuecomment-1663028691 + rm -v onnxruntime/test/optimizer/nhwc_transformer_test.cc ''; postBuild = lib.optionalString pythonSupport '' @@ -118,7 +176,7 @@ stdenv.mkDerivation rec { ''; passthru = { - inherit protobuf; + protobuf = protobuf3_21; tests = lib.optionalAttrs pythonSupport { python = python3Packages.onnxruntime; }; @@ -140,6 +198,6 @@ stdenv.mkDerivation rec { # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md#architectures platforms = platforms.unix; license = licenses.mit; - maintainers = with maintainers; [ jonringer puffnfresh ck3d ]; + maintainers = with maintainers; [ jonringer puffnfresh ck3d cbourjau ]; }; } diff --git a/pkgs/development/python-modules/onnxruntime/default.nix b/pkgs/development/python-modules/onnxruntime/default.nix index af369b0d82d..8fea2e6d911 100644 --- a/pkgs/development/python-modules/onnxruntime/default.nix +++ b/pkgs/development/python-modules/onnxruntime/default.nix @@ -1,4 +1,5 @@ { lib +, stdenv , buildPythonPackage , autoPatchelfHook , pythonRelaxDepsHook @@ -7,6 +8,7 @@ , numpy , packaging , oneDNN +, re2 }: @@ -34,8 +36,9 @@ buildPythonPackage { ''; nativeBuildInputs = [ - autoPatchelfHook pythonRelaxDepsHook + ] ++ lib.optionals stdenv.isLinux [ + autoPatchelfHook ]; # This project requires fairly large dependencies such as sympy which we really don't always need. @@ -48,6 +51,7 @@ buildPythonPackage { # Libraries are not linked correctly. buildInputs = [ oneDNN + re2 onnxruntime.protobuf ]; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b5f9e673616..0109d702efa 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5956,7 +5956,7 @@ with pkgs; online-judge-tools = with python3.pkgs; toPythonApplication online-judge-tools; onnxruntime = callPackage ../development/libraries/onnxruntime { - protobuf = protobuf3_19; + inherit (darwin.apple_sdk.frameworks) Foundation; }; xkbd = callPackage ../applications/misc/xkbd { };