From d909c0c1f1ba4e5fad7339c7d7bb4386add3441f Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 11:06:54 -0300 Subject: [PATCH 1/7] graalvm-ce: add lib output Add dynamic libraries (for now only glibc) to $lib output, so we can use them with e.g.: ``` native-image -H:CLibraryPath=${graalvm11-ce.lib}/lib ... ``` Reducing the finaly closure size of a GraalVM package since it will not pull the whole GraalVM as a direct dependency. --- .../graalvm/community-edition/repository.nix | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/pkgs/development/compilers/graalvm/community-edition/repository.nix b/pkgs/development/compilers/graalvm/community-edition/repository.nix index 50a3b53c579..615926b52ee 100644 --- a/pkgs/development/compilers/graalvm/community-edition/repository.nix +++ b/pkgs/development/compilers/graalvm/community-edition/repository.nix @@ -130,37 +130,51 @@ let unpack_jar ''${arr[4]} ''; - installPhase = { - "8-linux-amd64" = '' + outputs = [ "out" "lib" ]; + + installPhase = let + nativePRNGWorkaround = path: '' # BUG workaround http://mail.openjdk.java.net/pipermail/graal-dev/2017-December/005141.html - substituteInPlace $out/jre/lib/security/java.security \ + substituteInPlace ${path} \ --replace file:/dev/random file:/dev/./urandom \ --replace NativePRNGBlocking SHA1PRNG - + ''; + copyClibrariesToOut = basepath: '' # provide libraries needed for static compilation for f in ${glibc}/lib/* ${glibc.static}/lib/* ${zlib.static}/lib/*; do - ln -s $f $out/jre/lib/svm/clibraries/${platform}/$(basename $f) + ln -s $f ${basepath}/${platform}/$(basename $f) done + ''; + + copyClibrariesToLib = '' + # add those libraries to $lib output too, so we can use them with + # `native-image -H:CLibraryPath=''${graalvm11-ce.lib}/lib ...` and reduce + # closure size by not depending on GraalVM $out (that is much bigger) + mkdir -p $lib/lib + for f in ${glibc}/lib/*; do + ln -s $f $lib/lib/$(basename $f) + done + ''; + in { + "8-linux-amd64" = '' + ${nativePRNGWorkaround "$out/jre/lib/security/java.security"} + + ${copyClibrariesToOut "$out/jre/lib/svm/clibraries"} + + ${copyClibrariesToLib} # allow using external truffle-api.jar and languages not included in the distrubution rm $out/jre/lib/jvmci/parentClassLoader.classpath ''; "11-linux-amd64" = '' - # BUG workaround http://mail.openjdk.java.net/pipermail/graal-dev/2017-December/005141.html - substituteInPlace $out/conf/security/java.security \ - --replace file:/dev/random file:/dev/./urandom \ - --replace NativePRNGBlocking SHA1PRNG + ${nativePRNGWorkaround "$out/conf/security/java.security"} - # provide libraries needed for static compilation - for f in ${glibc}/lib/* ${glibc.static}/lib/* ${zlib.static}/lib/*; do - ln -s $f $out/lib/svm/clibraries/${platform}/$(basename $f) - done + ${copyClibrariesToOut "$out/lib/svm/clibraries"} + + ${copyClibrariesToLib} ''; "11-darwin-amd64" = '' - # BUG workaround http://mail.openjdk.java.net/pipermail/graal-dev/2017-December/005141.html - substituteInPlace $out/conf/security/java.security \ - --replace file:/dev/random file:/dev/./urandom \ - --replace NativePRNGBlocking SHA1PRNG + ${nativePRNGWorkaround "$out/conf/security/java.security"} ''; }.${javaVersionPlatform} + '' # jni.h expects jni_md.h to be in the header search path. From e7b3740842a3f82c9235d0dc48867653d2a73c96 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 11:08:52 -0300 Subject: [PATCH 2/7] babashka: reduce closure size Before: ``` $ nix path-info -S ./result -h /nix/store/6jsb0zpq531w9iymyzaik7lksppgw31k-babashka-0.6.1 2.0G ``` After: ``` $ nix path-info -S ./result -h /nix/store/fay0p3hiiic4kd4fk0xyczl1ff8sjcp1-babashka-0.6.1 116.6M ``` --- pkgs/development/interpreters/clojure/babashka.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/interpreters/clojure/babashka.nix b/pkgs/development/interpreters/clojure/babashka.nix index b167a4e9b12..df75a233fff 100644 --- a/pkgs/development/interpreters/clojure/babashka.nix +++ b/pkgs/development/interpreters/clojure/babashka.nix @@ -23,6 +23,7 @@ stdenv.mkDerivation rec { # https://github.com/babashka/babashka/blob/v0.6.1/script/compile#L41-L52 args=("-jar" "$BABASHKA_JAR" + "-H:CLibraryPath=${graalvm11-ce.lib}/lib" # Required to build babashka on darwin. Do not remove. "${lib.optionalString stdenv.isDarwin "-H:-CheckToolchain"}" "-H:Name=$BABASHKA_BINARY" From bc40a84b554a85f5bdfd782c487a3536301c2dfd Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 11:21:19 -0300 Subject: [PATCH 3/7] clj-kondo: refactor and reduce closure size Use the new `native-image.properties` to simplify build: https://github.com/clj-kondo/clj-kondo/pull/1373 Also link to `graalvm11-ce.lib` to reduce closure size: Before: ``` $ nix path-info -S ./result -h /nix/store/bc46ki5sj5abbdfsighvvvlk03qnzrp5-clj-kondo-2021.09.25 1.9G ``` After: ``` $ nix path-info -S ./result -h /nix/store/6mzk1f3j2j7lzzg378jl1g86p4pkz43z-clj-kondo-2021.09.25 74.1M ``` --- pkgs/development/tools/clj-kondo/default.nix | 38 +++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/pkgs/development/tools/clj-kondo/default.nix b/pkgs/development/tools/clj-kondo/default.nix index 45348b562c1..c4b90813a33 100644 --- a/pkgs/development/tools/clj-kondo/default.nix +++ b/pkgs/development/tools/clj-kondo/default.nix @@ -4,12 +4,6 @@ stdenv.mkDerivation rec { pname = "clj-kondo"; version = "2021.09.25"; - reflectionJson = fetchurl { - name = "reflection.json"; - url = "https://raw.githubusercontent.com/clj-kondo/${pname}/v${version}/reflection.json"; - sha256 = "sha256-C4QYk5lLienCHKnWXXZPcKmsCTMtIIkXOkvCrZfyIhA="; - }; - src = fetchurl { url = "https://github.com/clj-kondo/${pname}/releases/download/v${version}/${pname}-${version}-standalone.jar"; sha256 = "sha256-kS6bwsYH/cbjJlIeiDAy6QsAw+D1uHp26d4NBLfStjg="; @@ -20,21 +14,21 @@ stdenv.mkDerivation rec { buildInputs = [ graalvm11-ce ]; buildPhase = '' - native-image \ - -jar ${src} \ - -H:Name=clj-kondo \ - ${lib.optionalString stdenv.isDarwin ''-H:-CheckToolchain''} \ - -H:+ReportExceptionStackTraces \ - -J-Dclojure.spec.skip-macros=true \ - -J-Dclojure.compiler.direct-linking=true \ - "-H:IncludeResources=clj_kondo/impl/cache/built_in/.*" \ - -H:ReflectionConfigurationFiles=${reflectionJson} \ - --initialize-at-build-time \ - -H:Log=registerResource: \ - --verbose \ - --no-fallback \ - --no-server \ - "-J-Xmx3g" + runHook preBuild + + # https://github.com/clj-kondo/clj-kondo/blob/v2021.09.25/script/compile#L17-L21 + args=("-jar" "$src" + "-H:CLibraryPath=${graalvm11-ce.lib}/lib" + # Required to build babashka on darwin. Do not remove. + "${lib.optionalString stdenv.isDarwin "-H:-CheckToolchain"}" + "-H:+ReportExceptionStackTraces" + "--verbose" + "--no-fallback" + "-J-Xmx3g") + + native-image ''${args[@]} + + runHook postBuild ''; installPhase = '' @@ -47,6 +41,6 @@ stdenv.mkDerivation rec { homepage = "https://github.com/clj-kondo/clj-kondo"; license = licenses.epl10; platforms = graalvm11-ce.meta.platforms; - maintainers = with maintainers; [ jlesquembre bandresen ]; + maintainers = with maintainers; [ jlesquembre bandresen thiagokokada ]; }; } From 72971ccccb8063e018480c0ad60a824472a3b08d Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 11:41:47 -0300 Subject: [PATCH 4/7] clojure-lsp: reduce closure size Before: ``` $ nix path-info -S ./result -h /nix/store/z8802pkmg77wq10kz851fvrjib0pm75f-clojure-lsp-2021.09.30-15.28.01 2.0G ``` After: ``` $ nix path-info -S ./result -h /nix/store/1q6r7z9ga2hnar1gj5gbd6s0b3h5xhp8-clojure-lsp-2021.09.30-15.28.01 154.4M ``` --- pkgs/development/tools/misc/clojure-lsp/default.nix | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkgs/development/tools/misc/clojure-lsp/default.nix b/pkgs/development/tools/misc/clojure-lsp/default.nix index 160c1b560ee..f27a91ace4c 100644 --- a/pkgs/development/tools/misc/clojure-lsp/default.nix +++ b/pkgs/development/tools/misc/clojure-lsp/default.nix @@ -25,7 +25,18 @@ stdenv.mkDerivation rec { buildPhase = with lib; '' runHook preBuild - bash ./graalvm/native-unix-compile.sh + # https://github.com/clojure-lsp/clojure-lsp/blob/2021.09.30-15.28.01/graalvm/native-unix-compile.sh#L19-L24 + args=("-jar" "$CLOJURE_LSP_JAR" + "-H:CLibraryPath=${graalvm11-ce.lib}/lib" + # Required to build babashka on darwin. Do not remove. + "${lib.optionalString stdenv.isDarwin "-H:-CheckToolchain"}" + "-H:+ReportExceptionStackTraces" + "--verbose" + "--no-fallback" + "--native-image-info" + "$CLOJURE_LSP_XMX") + + native-image ''${args[@]} runHook postBuild ''; From e4a1d3292219f0d1d140da5eb912fef1dfcbc630 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 11:47:59 -0300 Subject: [PATCH 5/7] zprint: reduce closure size Before: ``` nix path-info -S ./result -h /nix/store/06i7frz4c4plpcmws5l57rqfd0nm20lm-zprint-1.1.2 1.9G ``` After: ``` $ nix path-info -S ./result -h /nix/store/irhavk9z81cgz3jwqxm7v7svmxdpqp9k-zprint-1.1.2 93.4M ``` --- pkgs/development/tools/zprint/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/tools/zprint/default.nix b/pkgs/development/tools/zprint/default.nix index 266ac83b80d..f00d9c29a48 100644 --- a/pkgs/development/tools/zprint/default.nix +++ b/pkgs/development/tools/zprint/default.nix @@ -24,6 +24,7 @@ stdenv.mkDerivation rec { -H:Name=${pname} \ -H:EnableURLProtocols=https,http \ -H:+ReportExceptionStackTraces \ + -H:CLibraryPath=${graalvm11-ce.lib}/lib \ ${lib.optionalString stdenv.isDarwin ''-H:-CheckToolchain''} \ --report-unsupported-elements-at-runtime \ --initialize-at-build-time \ From 07be670cf94fa896832689c3f59f73b49ef1ad13 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 12:08:27 -0300 Subject: [PATCH 6/7] graalvm-ce: create empty lib directory on macOS --- .../compilers/graalvm/community-edition/repository.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/compilers/graalvm/community-edition/repository.nix b/pkgs/development/compilers/graalvm/community-edition/repository.nix index 615926b52ee..b532fe8cb5d 100644 --- a/pkgs/development/compilers/graalvm/community-edition/repository.nix +++ b/pkgs/development/compilers/graalvm/community-edition/repository.nix @@ -174,6 +174,8 @@ let ${copyClibrariesToLib} ''; "11-darwin-amd64" = '' + # create empty $lib/lib to avoid breaking builds + mkdir -p $lib/lib ${nativePRNGWorkaround "$out/conf/security/java.security"} ''; }.${javaVersionPlatform} + '' From d672332c26371a50c76f52e379795eb974ff6f65 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 2 Oct 2021 12:50:44 -0300 Subject: [PATCH 7/7] clojure-lsp: remove "-H:CheckToolchain" on macOS --- pkgs/development/tools/misc/clojure-lsp/default.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/development/tools/misc/clojure-lsp/default.nix b/pkgs/development/tools/misc/clojure-lsp/default.nix index f27a91ace4c..30028375420 100644 --- a/pkgs/development/tools/misc/clojure-lsp/default.nix +++ b/pkgs/development/tools/misc/clojure-lsp/default.nix @@ -28,8 +28,6 @@ stdenv.mkDerivation rec { # https://github.com/clojure-lsp/clojure-lsp/blob/2021.09.30-15.28.01/graalvm/native-unix-compile.sh#L19-L24 args=("-jar" "$CLOJURE_LSP_JAR" "-H:CLibraryPath=${graalvm11-ce.lib}/lib" - # Required to build babashka on darwin. Do not remove. - "${lib.optionalString stdenv.isDarwin "-H:-CheckToolchain"}" "-H:+ReportExceptionStackTraces" "--verbose" "--no-fallback"