Merge pull request #158484 from markuskowa/fix-blaslapack

blas/lapack: increase flexibility of wrappers

doc/using/overlays.chapter.md

@@ -77,7 +77,7 @@ In Nixpkgs, we have multiple implementations of the BLAS/LAPACK numerical linear
 
     The Nixpkgs attribute is `openblas` for ILP64 (integer width = 64 bits) and `openblasCompat` for LP64 (integer width = 32 bits).  `openblasCompat` is the default.
 
--   [LAPACK reference](http://www.netlib.org/lapack/) (also provides BLAS)
+-   [LAPACK reference](http://www.netlib.org/lapack/) (also provides BLAS and CBLAS)
 
     The Nixpkgs attribute is `lapack-reference`.
 
@@ -117,7 +117,23 @@ $ LD_LIBRARY_PATH=$(nix-build -A mkl)/lib${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH n
 
 Intel MKL requires an `openmp` implementation when running with multiple processors. By default, `mkl` will use Intel's `iomp` implementation if no other is specified, but this is a runtime-only dependency and binary compatible with the LLVM implementation. To use that one instead, Intel recommends users set it with `LD_PRELOAD`. Note that `mkl` is only available on `x86_64-linux` and `x86_64-darwin`. Moreover, Hydra is not building and distributing pre-compiled binaries using it.
 
-For BLAS/LAPACK switching to work correctly, all packages must depend on `blas` or `lapack`. This ensures that only one BLAS/LAPACK library is used at one time. There are two versions of BLAS/LAPACK currently in the wild, `LP64` (integer size = 32 bits) and `ILP64` (integer size = 64 bits). Some software needs special flags or patches to work with `ILP64`. You can check if `ILP64` is used in Nixpkgs with `blas.isILP64` and `lapack.isILP64`. Some software does NOT work with `ILP64`, and derivations need to specify an assertion to prevent this. You can prevent `ILP64` from being used with the following:
+To override `blas` and `lapack` with its reference implementations (i.e. for development purposes), one can use the following overlay:
+
+```nix
+self: super:
+
+{
+  blas = super.blas.override {
+    blasProvider = self.lapack-reference;
+  };
+
+  lapack = super.lapack.override {
+    lapackProvider = self.lapack-reference;
+  };
+}
+```
+
+For BLAS/LAPACK switching to work correctly, all packages must depend on `blas` or `lapack`. This ensures that only one BLAS/LAPACK library is used at one time. There are two versions of BLAS/LAPACK currently in the wild, `LP64` (integer size = 32 bits) and `ILP64` (integer size = 64 bits). The attributes `blas` and `lapack` are `LP64` by default. Their `ILP64` version are provided through the attributes `blas-ilp64` and `lapack-ilp64`. Some software needs special flags or patches to work with `ILP64`. You can check if `ILP64` is used in Nixpkgs with `blas.isILP64` and `lapack.isILP64`. Some software does NOT work with `ILP64`, and derivations need to specify an assertion to prevent this. You can prevent `ILP64` from being used with the following:
 
 ```nix
 { stdenv, blas, lapack, ... }:

pkgs/build-support/alternatives/blas/default.nix

@@ -1,7 +1,7 @@
 { lib, stdenv
-, lapack-reference, openblasCompat, openblas
+, lapack-reference, openblas
 , isILP64 ? false
-, blasProvider ? if isILP64 then openblas else openblasCompat }:
+, blasProvider ? openblas }:
 
 let
   blasFortranSymbols = [
@@ -32,10 +32,13 @@ let
 
 
   blasImplementation = lib.getName blasProvider;
+  blasProvider' = if blasImplementation == "mkl"
+                  then blasProvider
+                  else blasProvider.override { blas64 = isILP64; };
 
 in
 
-assert isILP64 -> (blasImplementation == "openblas" && blasProvider.blas64) || blasImplementation == "mkl";
+assert isILP64 -> blasImplementation == "mkl" || blasProvider'.blas64;
 
 stdenv.mkDerivation {
   pname = "blas";
@@ -43,13 +46,13 @@ stdenv.mkDerivation {
 
   outputs = [ "out" "dev" ];
 
-  meta = (blasProvider.meta or {}) // {
+  meta = (blasProvider'.meta or {}) // {
     description = "${lib.getName blasProvider} with just the BLAS C and FORTRAN ABI";
   };
 
   passthru = {
     inherit isILP64;
-    provider = blasProvider;
+    provider = blasProvider';
     implementation = blasImplementation;
   };
 
@@ -62,10 +65,10 @@ stdenv.mkDerivation {
   installPhase = (''
   mkdir -p $out/lib $dev/include $dev/lib/pkgconfig
 
-  libblas="${lib.getLib blasProvider}/lib/libblas${canonicalExtension}"
+  libblas="${lib.getLib blasProvider'}/lib/libblas${canonicalExtension}"
 
   if ! [ -e "$libblas" ]; then
-    echo "$libblas does not exist, ${blasProvider.name} does not provide libblas."
+    echo "$libblas does not exist, ${blasProvider'.name} does not provide libblas."
     exit 1
   fi
 
@@ -79,11 +82,11 @@ stdenv.mkDerivation {
 
 '' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then ''
   patchelf --set-soname libblas${canonicalExtension} $out/lib/libblas${canonicalExtension}
-  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libblas${canonicalExtension}):${lib.getLib blasProvider}/lib" $out/lib/libblas${canonicalExtension}
+  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libblas${canonicalExtension}):${lib.getLib blasProvider'}/lib" $out/lib/libblas${canonicalExtension}
 '' else if stdenv.hostPlatform.isDarwin then ''
   install_name_tool \
     -id $out/lib/libblas${canonicalExtension} \
-    -add_rpath ${lib.getLib blasProvider}/lib \
+    -add_rpath ${lib.getLib blasProvider'}/lib \
     $out/lib/libblas${canonicalExtension}
 '' else "") + ''
 
@@ -99,10 +102,10 @@ Libs: -L$out/lib -lblas
 Cflags: -I$dev/include
 EOF
 
-  libcblas="${lib.getLib blasProvider}/lib/libcblas${canonicalExtension}"
+  libcblas="${lib.getLib blasProvider'}/lib/libcblas${canonicalExtension}"
 
   if ! [ -e "$libcblas" ]; then
-    echo "$libcblas does not exist, ${blasProvider.name} does not provide libcblas."
+    echo "$libcblas does not exist, ${blasProvider'.name} does not provide libcblas."
     exit 1
   fi
 
@@ -111,11 +114,11 @@ EOF
 
 '' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then ''
   patchelf --set-soname libcblas${canonicalExtension} $out/lib/libcblas${canonicalExtension}
-  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libcblas${canonicalExtension}):${lib.getLib blasProvider}/lib" $out/lib/libcblas${canonicalExtension}
+  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libcblas${canonicalExtension}):${lib.getLib blasProvider'}/lib" $out/lib/libcblas${canonicalExtension}
 '' else if stdenv.hostPlatform.isDarwin then ''
   install_name_tool \
     -id $out/lib/libcblas${canonicalExtension} \
-    -add_rpath ${lib.getLib blasProvider}/lib \
+    -add_rpath ${lib.getLib blasProvider'}/lib \
     $out/lib/libcblas${canonicalExtension}
 '' else "") + ''
   if [ "$out/lib/libcblas${canonicalExtension}" != "$out/lib/libcblas${stdenv.hostPlatform.extensions.sharedLibrary}" ]; then
@@ -135,6 +138,6 @@ EOF
   mkdir -p $out/nix-support
   echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook
   ln -s $out/lib/libblas${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary}
-  ln -sf ${blasProvider}/include/* $dev/include
+  ln -sf ${blasProvider'}/include/* $dev/include
 '');
 }

pkgs/build-support/alternatives/lapack/default.nix

@@ -1,7 +1,7 @@
 { lib, stdenv
-, lapack-reference, openblasCompat, openblas
+, lapack-reference, openblas
 , isILP64 ? false
-, lapackProvider ? if isILP64 then openblas else openblasCompat }:
+, lapackProvider ? openblas }:
 
 let
 
@@ -11,10 +11,13 @@ let
                        else stdenv.hostPlatform.extensions.sharedLibrary;
 
   lapackImplementation = lib.getName lapackProvider;
+  lapackProvider' = if lapackImplementation == "mkl"
+                  then lapackProvider
+                  else lapackProvider.override { blas64 = isILP64; };
 
 in
 
-assert isILP64 -> (lapackImplementation == "openblas" && lapackProvider.blas64) || lapackImplementation == "mkl";
+assert isILP64 -> lapackImplementation == "mkl" || lapackProvider'.blas64;
 
 stdenv.mkDerivation {
   pname = "lapack";
@@ -22,13 +25,13 @@ stdenv.mkDerivation {
 
   outputs = [ "out" "dev" ];
 
-  meta = (lapackProvider.meta or {}) // {
-    description = "${lib.getName lapackProvider} with just the LAPACK C and FORTRAN ABI";
+  meta = (lapackProvider'.meta or {}) // {
+    description = "${lib.getName lapackProvider'} with just the LAPACK C and FORTRAN ABI";
   };
 
   passthru = {
     inherit isILP64;
-    provider = lapackProvider;
+    provider = lapackProvider';
     implementation = lapackImplementation;
   };
 
@@ -41,10 +44,10 @@ stdenv.mkDerivation {
   installPhase = (''
   mkdir -p $out/lib $dev/include $dev/lib/pkgconfig
 
-  liblapack="${lib.getLib lapackProvider}/lib/liblapack${canonicalExtension}"
+  liblapack="${lib.getLib lapackProvider'}/lib/liblapack${canonicalExtension}"
 
   if ! [ -e "$liblapack" ]; then
-    echo "$liblapack does not exist, ${lapackProvider.name} does not provide liblapack."
+    echo "$liblapack does not exist, ${lapackProvider'.name} does not provide liblapack."
     exit 1
   fi
 
@@ -53,7 +56,7 @@ stdenv.mkDerivation {
 
 '' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then ''
   patchelf --set-soname liblapack${canonicalExtension} $out/lib/liblapack${canonicalExtension}
-  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapack${canonicalExtension}):${lapackProvider}/lib" $out/lib/liblapack${canonicalExtension}
+  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapack${canonicalExtension}):${lapackProvider'}/lib" $out/lib/liblapack${canonicalExtension}
 '' else "") + ''
 
   if [ "$out/lib/liblapack${canonicalExtension}" != "$out/lib/liblapack${stdenv.hostPlatform.extensions.sharedLibrary}" ]; then
@@ -70,10 +73,10 @@ Cflags: -I$dev/include
 Libs: -L$out/lib -llapack
 EOF
 
-  liblapacke="${lib.getLib lapackProvider}/lib/liblapacke${canonicalExtension}"
+  liblapacke="${lib.getLib lapackProvider'}/lib/liblapacke${canonicalExtension}"
 
   if ! [ -e "$liblapacke" ]; then
-    echo "$liblapacke does not exist, ${lapackProvider.name} does not provide liblapacke."
+    echo "$liblapacke does not exist, ${lapackProvider'.name} does not provide liblapacke."
     exit 1
   fi
 
@@ -82,7 +85,7 @@ EOF
 
 '' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then ''
   patchelf --set-soname liblapacke${canonicalExtension} $out/lib/liblapacke${canonicalExtension}
-  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapacke${canonicalExtension}):${lib.getLib lapackProvider}/lib" $out/lib/liblapacke${canonicalExtension}
+  patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapacke${canonicalExtension}):${lib.getLib lapackProvider'}/lib" $out/lib/liblapacke${canonicalExtension}
 '' else "") + ''
 
   if [ -f "$out/lib/liblapacke.so.3" ]; then
@@ -102,6 +105,6 @@ EOF
   mkdir -p $out/nix-support
   echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook
   ln -s $out/lib/liblapack${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary}
-  ln -sf ${lapackProvider}/include/* $dev/include
+  ln -sf ${lapackProvider'}/include/* $dev/include
 '');
 }

pkgs/development/libraries/science/math/amd-blis/default.nix

@@ -64,7 +64,7 @@ in stdenv.mkDerivation rec {
     description = "BLAS-compatible library optimized for AMD CPUs";
     homepage = "https://developer.amd.com/amd-aocl/blas-library/";
     license = licenses.bsd3;
-    maintainers = [ ];
+    maintainers = [ maintainers.markuskowa ];
     platforms = [ "x86_64-linux" ];
   };
 }

pkgs/development/libraries/science/math/amd-libflame/default.nix

@@ -6,8 +6,12 @@
 , amd-blis
 
 , withOpenMP ? true
+, blas64 ? false
 }:
 
+# right now only LP64 is supported
+assert !blas64;
+
 stdenv.mkDerivation rec {
   pname = "amd-libflame";
   version = "3.0";
@@ -26,6 +30,8 @@ stdenv.mkDerivation rec {
     ./add-lapacke.diff
   ];
 
+  passthru = { inherit blas64; };
+
   nativeBuildInputs = [ gfortran python3 ];
 
   buildInputs = [ amd-blis ];

pkgs/development/libraries/science/math/blas/default.nix

@@ -1,4 +1,7 @@
-{ lib, stdenv, fetchurl, gfortran }:
+{ lib, stdenv, fetchurl, cmake, gfortran
+# Wether to build with ILP64 interface
+, blas64 ? false
+}:
 
 stdenv.mkDerivation rec {
   pname = "blas";
@@ -9,50 +12,19 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-LjYNmcm9yEB6YYiMQKqFP7QhlCDruCZNtIbLiGBGirM=";
   };
 
-  nativeBuildInputs = [ gfortran ];
+  passthru = { inherit blas64; };
 
-  configurePhase = ''
-    echo >make.inc  "SHELL = ${stdenv.shell}"
-    echo >>make.inc "PLAT = _LINUX"
-    echo >>make.inc "FORTRAN = gfortran"
-    echo >>make.inc "OPTS = -O2 -fPIC"
-    echo >>make.inc "DRVOPTS = $$(OPTS)"
-    echo >>make.inc "NOOPT = -O0 -fPIC"
-    echo >>make.inc "LOADER = gfortran"
-    echo >>make.inc "LOADOPTS ="
-    echo >>make.inc "AR = gfortran"
-    echo >>make.inc "ARFLAGS = -shared -o"
-    echo >>make.inc "RANLIB = echo"
-    echo >>make.inc "BLASLIB = libblas.so.${version}"
-  '';
+  nativeBuildInputs = [ cmake gfortran ];
 
-  buildPhase = ''
-    make
-    echo >>make.inc "ARFLAGS = "
-    echo >>make.inc "BLASLIB = libblas.a"
-    echo >>make.inc "AR = ar rcs"
-    echo >>make.inc "RANLIB = ranlib"
-    make
-  '';
+  cmakeFlags = [ "-DBUILD_SHARED_LIBS=ON" ]
+    ++ lib.optional blas64 "-DBUILD_INDEX64=ON";
 
-  installPhase =
-    # FreeBSD's stdenv doesn't use Coreutils.
-    let dashD = if stdenv.isFreeBSD then "" else "-D"; in
-    (lib.optionalString stdenv.isFreeBSD "mkdir -p $out/lib ;")
-    + ''
-    install ${dashD} -m755 libblas.a "$out/lib/libblas.a"
-    install ${dashD} -m755 libblas.so.${version} "$out/lib/libblas.so.${version}"
-    ln -s libblas.so.${version} "$out/lib/libblas.so.3"
-    ln -s libblas.so.${version} "$out/lib/libblas.so"
-    # Write pkg-config alias.
-    # See also openblas/default.nix
-    mkdir $out/lib/pkgconfig
-    cat <<EOF > $out/lib/pkgconfig/blas.pc
-Name: blas
-Version: ${version}
-Description: blas provided by the BLAS package.
-Libs: -L$out/lib -lblas
-EOF
+  postInstall = let
+    canonicalExtension = if stdenv.hostPlatform.isLinux
+                       then "${stdenv.hostPlatform.extensions.sharedLibrary}.${lib.versions.major version}"
+                       else stdenv.hostPlatform.extensions.sharedLibrary;
+  in lib.optionalString blas64 ''
+    ln -s $out/lib/libblas64${canonicalExtension} $out/lib/libblas${canonicalExtension}
   '';
 
   preFixup = lib.optionalString stdenv.isDarwin ''
@@ -62,10 +34,11 @@ EOF
     done
   '';
 
-  meta = {
+  meta = with lib; {
     description = "Basic Linear Algebra Subprograms";
-    license = lib.licenses.publicDomain;
+    license = licenses.publicDomain;
+    maintainers = [ maintainers.markuskowa ];
     homepage = "http://www.netlib.org/blas/";
-    platforms = lib.platforms.unix;
+    platforms = platforms.unix;
   };
 }

pkgs/development/libraries/science/math/liblapack/default.nix

@@ -5,6 +5,8 @@
 , gfortran
 , cmake
 , shared ? true
+# Compile with ILP64 interface
+, blas64 ? false
 }:
 
 stdenv.mkDerivation rec {
@@ -36,7 +38,19 @@ stdenv.mkDerivation rec {
     "-DLAPACKE=ON"
     "-DCBLAS=ON"
     "-DBUILD_TESTING=ON"
-  ] ++ lib.optional shared "-DBUILD_SHARED_LIBS=ON";
+  ] ++ lib.optional shared "-DBUILD_SHARED_LIBS=ON"
+    ++ lib.optional blas64 "-DBUILD_INDEX64=ON";
+
+  passthru = { inherit blas64; };
+
+  postInstall =  let
+    canonicalExtension = if stdenv.hostPlatform.isLinux
+                       then "${stdenv.hostPlatform.extensions.sharedLibrary}.${lib.versions.major version}"
+                       else stdenv.hostPlatform.extensions.sharedLibrary;
+  in lib.optionalString blas64 ''
+    ln -s $out/lib/liblapack64${canonicalExtension} $out/lib/liblapack${canonicalExtension}
+    ln -s $out/lib/liblapacke64${canonicalExtension} $out/lib/liblapacke${canonicalExtension}
+  '';
 
   doCheck = true;
 
@@ -63,7 +77,7 @@ stdenv.mkDerivation rec {
   meta = with lib; {
     description = "Linear Algebra PACKage";
     homepage = "http://www.netlib.org/lapack/";
-    maintainers = with maintainers; [ ];
+    maintainers = with maintainers; [ markuskowa ];
     license = licenses.bsd3;
     platforms = platforms.all;
   };

pkgs/top-level/all-packages.nix

@@ -31989,6 +31989,8 @@ with pkgs;
 
   blas = callPackage ../build-support/alternatives/blas { };
 
+  blas-ilp64 = blas.override { isILP64 = true; };
+
   blas-reference = callPackage ../development/libraries/science/math/blas { };
 
   brial = callPackage ../development/libraries/science/math/brial { };
@@ -32011,6 +32013,8 @@ with pkgs;
 
   lapack = callPackage ../build-support/alternatives/lapack { };
 
+  lapack-ilp64 = lapack.override { isILP64 = true; };
+
   lapack-reference = callPackage ../development/libraries/science/math/liblapack { };
   liblapack = lapack-reference;