From 05c12ee78cf5c9a7a01fcdb841825db4b6a771d9 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 8 Feb 2022 08:59:23 -0500 Subject: [PATCH 1/3] spark: init 3.2.1 and test on aarch64-linux --- nixos/tests/all-tests.nix | 2 +- .../networking/cluster/spark/default.nix | 53 +++++++++++-------- pkgs/top-level/all-packages.nix | 4 +- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 342e8f461b5..dcdc79c9bb1 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -481,7 +481,7 @@ in sonarr = handleTest ./sonarr.nix {}; sourcehut = handleTest ./sourcehut.nix {}; spacecookie = handleTest ./spacecookie.nix {}; - spark = handleTestOn ["x86_64-linux"] ./spark {}; + spark = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./spark {}; sslh = handleTest ./sslh.nix {}; sssd = handleTestOn ["x86_64-linux"] ./sssd.nix {}; sssd-ldap = handleTestOn ["x86_64-linux"] ./sssd-ldap.nix {}; diff --git a/pkgs/applications/networking/cluster/spark/default.nix b/pkgs/applications/networking/cluster/spark/default.nix index 7876eb1c752..3429a1d703a 100644 --- a/pkgs/applications/networking/cluster/spark/default.nix +++ b/pkgs/applications/networking/cluster/spark/default.nix @@ -1,13 +1,26 @@ -{ lib, stdenv, fetchzip, makeWrapper, jdk8, python3Packages, extraPythonPackages ? [], coreutils, hadoop -, RSupport? true, R +{ lib +, stdenv +, fetchzip +, makeWrapper +, jdk8 +, python3Packages +, extraPythonPackages ? [ ] +, coreutils +, hadoop +, RSupport ? true +, R }: with lib; let - spark = { pname, version, src }: + spark = { pname, version, sha256 }: stdenv.mkDerivation rec { - inherit pname version src; + inherit pname version; + src = fetchzip { + url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz"; + sha256 = sha256; + }; nativeBuildInputs = [ makeWrapper ]; buildInputs = [ jdk8 python3Packages.python ] ++ extraPythonPackages @@ -45,31 +58,29 @@ let ''; meta = { - description = "Apache Spark is a fast and general engine for large-scale data processing"; - homepage = "https://spark.apache.org/"; - license = lib.licenses.asl20; - platforms = lib.platforms.all; - maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ]; + description = "Apache Spark is a fast and general engine for large-scale data processing"; + homepage = "https://spark.apache.org/"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ]; repositories.git = "git://git.apache.org/spark.git"; }; }; -in { - spark3 = spark rec { +in +{ + spark3_2_1 = spark rec { + pname = "spark"; + version = "3.2.1"; + sha256 = "0kxdqczwmj6pray0h8h1qhygni9m82jzznw5fbv9hrxrkq1v182d"; + }; + spark3_1_2 = spark rec { pname = "spark"; version = "3.1.2"; - - src = fetchzip { - url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz"; - sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf"; - }; + sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf"; }; spark2 = spark rec { pname = "spark"; version = "2.4.8"; - - src = fetchzip { - url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz"; - sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv"; - }; + sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv"; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 590582487eb..76fd1ba9675 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14095,8 +14095,10 @@ with pkgs; self = pkgsi686Linux.callPackage ../development/interpreters/self { }; inherit (callPackages ../applications/networking/cluster/spark { }) - spark3 + spark3_2_1 + spark3_1_2 spark2; + spark3 = spark3_1_2; spark = spark3; sparkleshare = callPackage ../applications/version-management/sparkleshare { }; From 246794d5c48de67ede26c3ca9bc122a56f097877 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 9 Mar 2022 23:35:54 -0500 Subject: [PATCH 2/3] spark: bring version naming in line with hadoop --- pkgs/applications/networking/cluster/spark/default.nix | 6 +++--- pkgs/top-level/all-packages.nix | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkgs/applications/networking/cluster/spark/default.nix b/pkgs/applications/networking/cluster/spark/default.nix index 3429a1d703a..7770f98afe5 100644 --- a/pkgs/applications/networking/cluster/spark/default.nix +++ b/pkgs/applications/networking/cluster/spark/default.nix @@ -68,17 +68,17 @@ let }; in { - spark3_2_1 = spark rec { + spark_3_2 = spark rec { pname = "spark"; version = "3.2.1"; sha256 = "0kxdqczwmj6pray0h8h1qhygni9m82jzznw5fbv9hrxrkq1v182d"; }; - spark3_1_2 = spark rec { + spark_3_1 = spark rec { pname = "spark"; version = "3.1.2"; sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf"; }; - spark2 = spark rec { + spark_2_4 = spark rec { pname = "spark"; version = "2.4.8"; sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 76fd1ba9675..c2c324307f8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14095,10 +14095,11 @@ with pkgs; self = pkgsi686Linux.callPackage ../development/interpreters/self { }; inherit (callPackages ../applications/networking/cluster/spark { }) - spark3_2_1 - spark3_1_2 - spark2; - spark3 = spark3_1_2; + spark_3_2 + spark_3_1 + spark_2_4; + spark3 = spark_3_2; + spark2 = spark_2_4; spark = spark3; sparkleshare = callPackage ../applications/version-management/sparkleshare { }; From ee1ff0797b6f16e5a6fe7caf2ec50907c6de99d2 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 9 Mar 2022 23:48:06 -0500 Subject: [PATCH 3/3] spark: update release notes for 22.05 --- .../from_md/release-notes/rl-2205.section.xml | 40 +++++++++++++++++++ .../manual/release-notes/rl-2205.section.md | 9 +++++ 2 files changed, 49 insertions(+) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 4f4a5a3394e..223f16e2ca7 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -1303,6 +1303,46 @@ desktop environments as needed. + + + The hadoop package has added support for + aarch64-linux and + aarch64-darwin as of 3.3.1 + (#158613). + + + + + The R package now builds again on + aarch64-darwin + (#158992). + + + + + The spark3 package has been updated from + 3.1.2 to 3.2.1 + (#160075): + + + + + Testing has been enabled for + aarch64-linux in addition to + x86_64-linux. + + + + + The spark3 package is now usable on + aarch64-darwin as a result of + #158613 + and + #158992. + + + + diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index c4281561f16..37432e7abd0 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -471,4 +471,13 @@ In addition to numerous new and upgraded packages, this release has the followin - The polkit service, available at `security.polkit.enable`, is now disabled by default. It will automatically be enabled through services and desktop environments as needed. +- The `hadoop` package has added support for `aarch64-linux` and `aarch64-darwin` as of 3.3.1 ([#158613](https://github.com/NixOS/nixpkgs/pull/158613)). + +- The `R` package now builds again on `aarch64-darwin` ([#158992](https://github.com/NixOS/nixpkgs/pull/158992)). + +- The `spark3` package has been updated from 3.1.2 to 3.2.1 ([#160075](https://github.com/NixOS/nixpkgs/pull/160075)): + + - Testing has been enabled for `aarch64-linux` in addition to `x86_64-linux`. + - The `spark3` package is now usable on `aarch64-darwin` as a result of [#158613](https://github.com/NixOS/nixpkgs/pull/158613) and [#158992](https://github.com/NixOS/nixpkgs/pull/158992). +