92 lines
3.1 KiB
Nix
92 lines
3.1 KiB
Nix
{ lib
|
|
, stdenv
|
|
, fetchzip
|
|
, makeWrapper
|
|
, jdk8
|
|
, python3Packages
|
|
, extraPythonPackages ? [ ]
|
|
, coreutils
|
|
, hadoopSupport ? true
|
|
, hadoop
|
|
, RSupport ? true
|
|
, R
|
|
}:
|
|
|
|
let
|
|
spark = { pname, version, hash, extraMeta ? {} }:
|
|
stdenv.mkDerivation rec {
|
|
inherit pname version;
|
|
jdk = if hadoopSupport then hadoop.jdk else jdk8;
|
|
src = fetchzip {
|
|
url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
|
|
inherit hash;
|
|
};
|
|
nativeBuildInputs = [ makeWrapper ];
|
|
buildInputs = [ jdk python3Packages.python ]
|
|
++ extraPythonPackages
|
|
++ lib.optional RSupport R;
|
|
|
|
untarDir = "${pname}-${version}";
|
|
installPhase = ''
|
|
mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java}
|
|
mv * $out/lib/${untarDir}
|
|
|
|
cp $out/lib/${untarDir}/conf/log4j.properties{.template,} || \
|
|
cp $out/lib/${untarDir}/conf/log4j2.properties{.template,}
|
|
|
|
cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
|
|
export JAVA_HOME="${jdk}"
|
|
export SPARK_HOME="$out/lib/${untarDir}"
|
|
'' + lib.optionalString hadoopSupport ''
|
|
export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath)
|
|
'' + ''
|
|
export PYSPARK_PYTHON="${python3Packages.python}/bin/${python3Packages.python.executable}"
|
|
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
|
|
${lib.optionalString RSupport ''
|
|
export SPARKR_R_SHELL="${R}/bin/R"
|
|
export PATH="\$PATH:${R}/bin"''}
|
|
EOF
|
|
|
|
for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
|
|
makeWrapper "$n" "$out/bin/$(basename $n)"
|
|
substituteInPlace "$n" --replace dirname ${coreutils.out}/bin/dirname
|
|
done
|
|
for n in $(find $out/lib/${untarDir}/sbin -type f); do
|
|
# Spark deprecated scripts with "slave" in the name.
|
|
# This line adds forward compatibility with the nixos spark module for
|
|
# older versions of spark that don't have the new "worker" scripts.
|
|
ln -s "$n" $(echo "$n" | sed -r 's/slave(s?).sh$/worker\1.sh/g') || true
|
|
done
|
|
ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
|
|
'';
|
|
|
|
meta = {
|
|
description = "Apache Spark is a fast and general engine for large-scale data processing";
|
|
homepage = "https://spark.apache.org/";
|
|
sourceProvenance = with lib.sourceTypes; [ binaryBytecode ];
|
|
license = lib.licenses.asl20;
|
|
platforms = lib.platforms.all;
|
|
maintainers = with lib.maintainers; [ thoughtpolice offline kamilchm illustris ];
|
|
} // extraMeta;
|
|
};
|
|
in
|
|
{
|
|
spark_3_4 = spark rec {
|
|
pname = "spark";
|
|
version = "3.4.0";
|
|
hash = "sha256-0y80dRYzb6Ceu6MlGQHtpMdzOob/TBg6kf8dtF6KyCk=";
|
|
};
|
|
spark_3_3 = spark rec {
|
|
pname = "spark";
|
|
version = "3.3.2";
|
|
hash = "sha256-AeKe2QN+mhUJgZRSIgbi/DttAWlDgwC1kl9p7syEvbo=";
|
|
extraMeta.knownVulnerabilities = [ "CVE-2023-22946" ];
|
|
};
|
|
spark_3_2 = spark rec {
|
|
pname = "spark";
|
|
version = "3.2.4";
|
|
hash = "sha256-xL4W+dTWbvmmncq3/8iXmhp24rp5SftvoRfkTyxCI8E=";
|
|
extraMeta.knownVulnerabilities = [ "CVE-2023-22946" ];
|
|
};
|
|
}
|