diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 66f2ac7ae26..41b99c59054 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -569,6 +569,25 @@ because Python 2 is being retired from nixpkgs. + + + Services in the hadoop module previously + set openFirewall to true by default. This + has now been changed to false. Node definitions for multi-node + clusters would need openFirewall = true; to + be added to to hadoop services when upgrading from NixOS + 21.11. + + + + + services.hadoop.yarn.nodemanager now uses + cgroup-based CPU limit enforcement by default. Additionally, + the option useCGroups was added to + nodemanagers as an easy way to switch back to the old + behavior. + + The wafHook hook now honors @@ -1173,6 +1192,33 @@ using the PyPy interpreter were added. + + + Some improvements have been made to the + hadoop module: + + + + + A gatewayRole option has been added, + for deploying hadoop cluster configuration files to a node + that does not have any active services + + + + + Support for older versions of hadoop have been added to + the module + + + + + Overriding and extending site XML files has been made + easier + + + + If you are using Wayland you can choose to use the Ozone diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 01893993955..3c4652ae5c4 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -244,6 +244,14 @@ In addition to numerous new and upgraded packages, this release has the followin - The MoinMoin wiki engine (`services.moinmoin`) has been removed, because Python 2 is being retired from nixpkgs. +- Services in the `hadoop` module previously set `openFirewall` to true by default. + This has now been changed to false. Node definitions for multi-node clusters would need + `openFirewall = true;` to be added to to hadoop services when upgrading from NixOS 21.11. + +- `services.hadoop.yarn.nodemanager` now uses cgroup-based CPU limit enforcement by default. + Additionally, the option `useCGroups` was added to nodemanagers as an easy way to switch + back to the old behavior. + - The `wafHook` hook now honors `NIX_BUILD_CORES` when `enableParallelBuilding` is not set explicitly. Packages can restore the old behaviour by setting `enableParallelBuilding=false`. - `pkgs.claws-mail-gtk2`, representing Claws Mail's older release version three, was removed in order to get rid of Python 2. @@ -436,6 +444,11 @@ In addition to numerous new and upgraded packages, this release has the followin - The `writers.writePyPy2`/`writers.writePyPy3` and corresponding `writers.writePyPy2Bin`/`writers.writePyPy3Bin` convenience functions to create executable Python 2/3 scripts using the PyPy interpreter were added. +- Some improvements have been made to the `hadoop` module: + - A `gatewayRole` option has been added, for deploying hadoop cluster configuration files to a node that does not have any active services + - Support for older versions of hadoop have been added to the module + - Overriding and extending site XML files has been made easier + - If you are using Wayland you can choose to use the Ozone Wayland support in Chrome and several Electron apps by setting the environment variable `NIXOS_OZONE_WL=1` (for example via diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix index 0caec5cfc20..e3c26a0d550 100644 --- a/nixos/modules/services/cluster/hadoop/conf.nix +++ b/nixos/modules/services/cluster/hadoop/conf.nix @@ -1,6 +1,6 @@ { cfg, pkgs, lib }: let - propertyXml = name: value: '' + propertyXml = name: value: lib.optionalString (value != null) '' ${name} ${builtins.toString value} @@ -29,16 +29,16 @@ let export HADOOP_LOG_DIR=/tmp/hadoop/$USER ''; in -pkgs.runCommand "hadoop-conf" {} '' +pkgs.runCommand "hadoop-conf" {} (with cfg; '' mkdir -p $out/ - cp ${siteXml "core-site.xml" cfg.coreSite}/* $out/ - cp ${siteXml "hdfs-site.xml" cfg.hdfsSite}/* $out/ - cp ${siteXml "mapred-site.xml" cfg.mapredSite}/* $out/ - cp ${siteXml "yarn-site.xml" cfg.yarnSite}/* $out/ - cp ${siteXml "httpfs-site.xml" cfg.httpfsSite}/* $out/ - cp ${cfgFile "container-executor.cfg" cfg.containerExecutorCfg}/* $out/ + cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/ + cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/ + cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/ + cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/ + cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/ + cp ${cfgFile "container-executor.cfg" containerExecutorCfg}/* $out/ cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/ cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/ - cp ${cfg.log4jProperties} $out/log4j.properties - ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") cfg.extraConfDirs} -'' + cp ${log4jProperties} $out/log4j.properties + ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") extraConfDirs} +'') diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix index a1a95fe31ca..a4fdea81037 100644 --- a/nixos/modules/services/cluster/hadoop/default.nix +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -21,24 +21,50 @@ with lib; ''; }; + coreSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to core-site.xml based on module options + ''; + }; - hdfsSite = mkOption { + hdfsSiteDefault = mkOption { default = { "dfs.namenode.rpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-address" = "0.0.0.0:9870"; + "dfs.namenode.servicerpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-bind-host" = "0.0.0.0"; }; type = types.attrsOf types.anything; + description = '' + Default options for hdfs-site.xml + ''; + }; + hdfsSite = mkOption { + default = {}; + type = types.attrsOf types.anything; example = literalExpression '' { "dfs.nameservices" = "namenode1"; } ''; description = '' - Hadoop hdfs-site.xml definition + Additional options and overrides for hdfs-site.xml ''; }; + hdfsSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to hdfs-site.xml based on module options + ''; + }; - mapredSite = mkOption { + mapredSiteDefault = mkOption { default = { "mapreduce.framework.name" = "yarn"; "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; @@ -54,18 +80,25 @@ with lib; } ''; type = types.attrsOf types.anything; + description = '' + Default options for mapred-site.xml + ''; + }; + mapredSite = mkOption { + default = {}; + type = types.attrsOf types.anything; example = literalExpression '' - options.services.hadoop.mapredSite.default // { + { "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC"; } ''; description = '' - Hadoop mapred-site.xml definition + Additional options and overrides for mapred-site.xml ''; }; - yarnSite = mkOption { + yarnSiteDefault = mkOption { default = { "yarn.nodemanager.admin-env" = "PATH=$PATH"; "yarn.nodemanager.aux-services" = "mapreduce_shuffle"; @@ -77,19 +110,34 @@ with lib; "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor"; "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager"; "yarn.resourcemanager.bind-host" = "0.0.0.0"; - "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"; + "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; }; type = types.attrsOf types.anything; + description = '' + Default options for yarn-site.xml + ''; + }; + yarnSite = mkOption { + default = {}; + type = types.attrsOf types.anything; example = literalExpression '' - options.services.hadoop.yarnSite.default // { + { "yarn.resourcemanager.hostname" = "''${config.networking.hostName}"; } ''; description = '' - Hadoop yarn-site.xml definition + Additional options and overrides for yarn-site.xml ''; }; + yarnSiteInternal = mkOption { + default = {}; + type = types.attrsOf types.anything; + internal = true; + description = '' + Internal option to add configs to yarn-site.xml based on module options + ''; + }; httpfsSite = mkOption { default = { }; @@ -123,6 +171,7 @@ with lib; "yarn.nodemanager.linux-container-executor.group"="hadoop"; "min.user.id"=1000; "feature.terminal.enabled"=1; + "feature.mount-cgroup.enabled" = 1; }; type = types.attrsOf types.anything; example = literalExpression '' @@ -148,6 +197,8 @@ with lib; description = "Directories containing additional config files to be added to HADOOP_CONF_DIR"; }; + gatewayRole.enable = mkEnableOption "gateway role for deploying hadoop configs"; + package = mkOption { type = types.package; default = pkgs.hadoop; @@ -157,20 +208,16 @@ with lib; }; - config = mkMerge [ - (mkIf (builtins.hasAttr "yarn" config.users.users || - builtins.hasAttr "hdfs" config.users.users || - builtins.hasAttr "httpfs" config.users.users) { - users.groups.hadoop = { - gid = config.ids.gids.hadoop; - }; - environment = { - systemPackages = [ cfg.package ]; - etc."hadoop-conf".source = let - hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; - in "${hadoopConf}"; - }; - }) - - ]; + config = mkIf cfg.gatewayRole.enable { + users.groups.hadoop = { + gid = config.ids.gids.hadoop; + }; + environment = { + systemPackages = [ cfg.package ]; + etc."hadoop-conf".source = let + hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; + in "${hadoopConf}"; + variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/"; + }; + }; } diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix index be667aa82d8..325a002ad32 100644 --- a/nixos/modules/services/cluster/hadoop/hdfs.nix +++ b/nixos/modules/services/cluster/hadoop/hdfs.nix @@ -1,191 +1,191 @@ -{ config, lib, pkgs, ...}: +{ config, lib, pkgs, ... }: with lib; let cfg = config.services.hadoop; + + # Config files for hadoop services hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; - restartIfChanged = mkOption { - type = types.bool; - description = '' - Automatically restart the service on config change. - This can be set to false to defer restarts on clusters running critical applications. - Please consider the security implications of inadvertently running an older version, - and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option. - ''; - default = false; - }; + + # Generator for HDFS service options + hadoopServiceOption = { serviceName, firewallOption ? true, extraOpts ? null }: { + enable = mkEnableOption serviceName; + restartIfChanged = mkOption { + type = types.bool; + description = '' + Automatically restart the service on config change. + This can be set to false to defer restarts on clusters running critical applications. + Please consider the security implications of inadvertently running an older version, + and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option. + ''; + default = false; + }; + extraFlags = mkOption{ + type = with types; listOf str; + default = []; + description = "Extra command line flags to pass to ${serviceName}"; + example = [ + "-Dcom.sun.management.jmxremote" + "-Dcom.sun.management.jmxremote.port=8010" + ]; + }; + extraEnv = mkOption{ + type = with types; attrsOf str; + default = {}; + description = "Extra environment variables for ${serviceName}"; + }; + } // (optionalAttrs firewallOption { + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open firewall ports for ${serviceName}."; + }; + }) // (optionalAttrs (extraOpts != null) extraOpts); + + # Generator for HDFS service configs + hadoopServiceConfig = + { name + , serviceOptions ? cfg.hdfs."${toLower name}" + , description ? "Hadoop HDFS ${name}" + , User ? "hdfs" + , allowedTCPPorts ? [ ] + , preStart ? "" + , environment ? { } + , extraConfig ? { } + }: ( + + mkIf serviceOptions.enable ( mkMerge [{ + systemd.services."hdfs-${toLower name}" = { + inherit description preStart; + environment = environment // serviceOptions.extraEnv; + wantedBy = [ "multi-user.target" ]; + inherit (serviceOptions) restartIfChanged; + serviceConfig = { + inherit User; + SyslogIdentifier = "hdfs-${toLower name}"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} ${toLower name} ${escapeShellArgs serviceOptions.extraFlags}"; + Restart = "always"; + }; + }; + + services.hadoop.gatewayRole.enable = true; + + networking.firewall.allowedTCPPorts = mkIf + ((builtins.hasAttr "openFirewall" serviceOptions) && serviceOptions.openFirewall) + allowedTCPPorts; + } extraConfig]) + ); + in { options.services.hadoop.hdfs = { - namenode = { - enable = mkEnableOption "Whether to run the HDFS NameNode"; + + namenode = hadoopServiceOption { serviceName = "HDFS NameNode"; } // { formatOnInit = mkOption { type = types.bool; default = false; description = '' - Format HDFS namenode on first start. This is useful for quickly spinning up ephemeral HDFS clusters with a single namenode. - For HA clusters, initialization involves multiple steps across multiple nodes. Follow [this guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html) - to initialize an HA cluster manually. - ''; - }; - inherit restartIfChanged; - openFirewall = mkOption { - type = types.bool; - default = true; - description = '' - Open firewall ports for namenode + Format HDFS namenode on first start. This is useful for quickly spinning up + ephemeral HDFS clusters with a single namenode. + For HA clusters, initialization involves multiple steps across multiple nodes. + Follow this guide to initialize an HA cluster manually: + ''; }; }; - datanode = { - enable = mkEnableOption "Whether to run the HDFS DataNode"; - inherit restartIfChanged; - openFirewall = mkOption { - type = types.bool; - default = true; - description = '' - Open firewall ports for datanode - ''; + + datanode = hadoopServiceOption { serviceName = "HDFS DataNode"; } // { + dataDirs = mkOption { + default = null; + description = "Tier and path definitions for datanode storage."; + type = with types; nullOr (listOf (submodule { + options = { + type = mkOption { + type = enum [ "SSD" "DISK" "ARCHIVE" "RAM_DISK" ]; + description = '' + Storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS storage policies. + ''; + }; + path = mkOption { + type = path; + example = [ "/var/lib/hadoop/hdfs/dn" ]; + description = "Determines where on the local filesystem a data node should store its blocks."; + }; + }; + })); }; }; - journalnode = { - enable = mkEnableOption "Whether to run the HDFS JournalNode"; - inherit restartIfChanged; - openFirewall = mkOption { - type = types.bool; - default = true; - description = '' - Open firewall ports for journalnode - ''; - }; + + journalnode = hadoopServiceOption { serviceName = "HDFS JournalNode"; }; + + zkfc = hadoopServiceOption { + serviceName = "HDFS ZooKeeper failover controller"; + firewallOption = false; }; - zkfc = { - enable = mkEnableOption "Whether to run the HDFS ZooKeeper failover controller"; - inherit restartIfChanged; - }; - httpfs = { - enable = mkEnableOption "Whether to run the HDFS HTTPfs server"; + + httpfs = hadoopServiceOption { serviceName = "HDFS JournalNode"; } // { tempPath = mkOption { type = types.path; default = "/tmp/hadoop/httpfs"; - description = '' - HTTPFS_TEMP path used by HTTPFS - ''; - }; - inherit restartIfChanged; - openFirewall = mkOption { - type = types.bool; - default = true; - description = '' - Open firewall ports for HTTPFS - ''; + description = "HTTPFS_TEMP path used by HTTPFS"; }; }; + }; config = mkMerge [ - (mkIf cfg.hdfs.namenode.enable { - systemd.services.hdfs-namenode = { - description = "Hadoop HDFS NameNode"; - wantedBy = [ "multi-user.target" ]; - inherit (cfg.hdfs.namenode) restartIfChanged; - - preStart = (mkIf cfg.hdfs.namenode.formatOnInit '' - ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true - ''); - - serviceConfig = { - User = "hdfs"; - SyslogIdentifier = "hdfs-namenode"; - ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode"; - Restart = "always"; - }; - }; - - networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.namenode.openFirewall [ + (hadoopServiceConfig { + name = "NameNode"; + allowedTCPPorts = [ 9870 # namenode.http-address 8020 # namenode.rpc-address - 8022 # namenode. servicerpc-address - ]); + 8022 # namenode.servicerpc-address + 8019 # dfs.ha.zkfc.port + ]; + preStart = (mkIf cfg.hdfs.namenode.formatOnInit + "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true" + ); }) - (mkIf cfg.hdfs.datanode.enable { - systemd.services.hdfs-datanode = { - description = "Hadoop HDFS DataNode"; - wantedBy = [ "multi-user.target" ]; - inherit (cfg.hdfs.datanode) restartIfChanged; - serviceConfig = { - User = "hdfs"; - SyslogIdentifier = "hdfs-datanode"; - ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode"; - Restart = "always"; - }; - }; - - networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [ + (hadoopServiceConfig { + name = "DataNode"; + # port numbers for datanode changed between hadoop 2 and 3 + allowedTCPPorts = if versionAtLeast cfg.package.version "3" then [ 9864 # datanode.http.address 9866 # datanode.address 9867 # datanode.ipc.address - ]); + ] else [ + 50075 # datanode.http.address + 50010 # datanode.address + 50020 # datanode.ipc.address + ]; + extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = let d = cfg.hdfs.datanode.dataDirs; in + if (d!= null) then (concatMapStringsSep "," (x: "["+x.type+"]file://"+x.path) cfg.hdfs.datanode.dataDirs) else d; }) - (mkIf cfg.hdfs.journalnode.enable { - systemd.services.hdfs-journalnode = { - description = "Hadoop HDFS JournalNode"; - wantedBy = [ "multi-user.target" ]; - inherit (cfg.hdfs.journalnode) restartIfChanged; - serviceConfig = { - User = "hdfs"; - SyslogIdentifier = "hdfs-journalnode"; - ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} journalnode"; - Restart = "always"; - }; - }; - - networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.journalnode.openFirewall [ + (hadoopServiceConfig { + name = "JournalNode"; + allowedTCPPorts = [ 8480 # dfs.journalnode.http-address 8485 # dfs.journalnode.rpc-address - ]); + ]; }) - (mkIf cfg.hdfs.zkfc.enable { - systemd.services.hdfs-zkfc = { - description = "Hadoop HDFS ZooKeeper failover controller"; - wantedBy = [ "multi-user.target" ]; - inherit (cfg.hdfs.zkfc) restartIfChanged; - serviceConfig = { - User = "hdfs"; - SyslogIdentifier = "hdfs-zkfc"; - ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} zkfc"; - Restart = "always"; - }; - }; + (hadoopServiceConfig { + name = "zkfc"; + description = "Hadoop HDFS ZooKeeper failover controller"; }) - (mkIf cfg.hdfs.httpfs.enable { - systemd.services.hdfs-httpfs = { - description = "Hadoop httpfs"; - wantedBy = [ "multi-user.target" ]; - inherit (cfg.hdfs.httpfs) restartIfChanged; - environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath; - - preStart = '' - mkdir -p $HTTPFS_TEMP - ''; - - serviceConfig = { - User = "httpfs"; - SyslogIdentifier = "hdfs-httpfs"; - ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} httpfs"; - Restart = "always"; - }; - }; - networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.httpfs.openFirewall [ + (hadoopServiceConfig { + name = "HTTPFS"; + environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath; + preStart = "mkdir -p $HTTPFS_TEMP"; + User = "httpfs"; + allowedTCPPorts = [ 14000 # httpfs.http.port - ]); + ]; }) - (mkIf ( - cfg.hdfs.namenode.enable || cfg.hdfs.datanode.enable || cfg.hdfs.journalnode.enable || cfg.hdfs.zkfc.enable - ) { + + (mkIf cfg.gatewayRole.enable { users.users.hdfs = { description = "Hadoop HDFS user"; group = "hadoop"; @@ -199,5 +199,6 @@ in isSystemUser = true; }; }) + ]; } diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index 37c26ea10f7..74e16bdec68 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -13,23 +13,77 @@ let ''; default = false; }; + extraFlags = mkOption{ + type = with types; listOf str; + default = []; + description = "Extra command line flags to pass to the service"; + example = [ + "-Dcom.sun.management.jmxremote" + "-Dcom.sun.management.jmxremote.port=8010" + ]; + }; + extraEnv = mkOption{ + type = with types; attrsOf str; + default = {}; + description = "Extra environment variables"; + }; in { options.services.hadoop.yarn = { resourcemanager = { - enable = mkEnableOption "Whether to run the Hadoop YARN ResourceManager"; - inherit restartIfChanged; + enable = mkEnableOption "Hadoop YARN ResourceManager"; + inherit restartIfChanged extraFlags extraEnv; + openFirewall = mkOption { type = types.bool; - default = true; + default = false; description = '' Open firewall ports for resourcemanager ''; }; }; nodemanager = { - enable = mkEnableOption "Whether to run the Hadoop YARN NodeManager"; - inherit restartIfChanged; + enable = mkEnableOption "Hadoop YARN NodeManager"; + inherit restartIfChanged extraFlags extraEnv; + + resource = { + cpuVCores = mkOption { + description = "Number of vcores that can be allocated for containers."; + type = with types; nullOr ints.positive; + default = null; + }; + maximumAllocationVCores = mkOption { + description = "The maximum virtual CPU cores any container can be allocated."; + type = with types; nullOr ints.positive; + default = null; + }; + memoryMB = mkOption { + description = "Amount of physical memory, in MB, that can be allocated for containers."; + type = with types; nullOr ints.positive; + default = null; + }; + maximumAllocationMB = mkOption { + description = "The maximum physical memory any container can be allocated."; + type = with types; nullOr ints.positive; + default = null; + }; + }; + + useCGroups = mkOption { + type = types.bool; + default = true; + description = '' + Use cgroups to enforce resource limits on containers + ''; + }; + + localDir = mkOption { + description = "List of directories to store localized files in."; + type = with types; nullOr (listOf path); + example = [ "/var/lib/hadoop/yarn/nm" ]; + default = null; + }; + addBinBash = mkOption { type = types.bool; default = true; @@ -39,7 +93,7 @@ in }; openFirewall = mkOption { type = types.bool; - default = true; + default = false; description = '' Open firewall ports for nodemanager. Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened. @@ -49,10 +103,7 @@ in }; config = mkMerge [ - (mkIf ( - cfg.yarn.resourcemanager.enable || cfg.yarn.nodemanager.enable - ) { - + (mkIf cfg.gatewayRole.enable { users.users.yarn = { description = "Hadoop YARN user"; group = "hadoop"; @@ -65,15 +116,19 @@ in description = "Hadoop YARN ResourceManager"; wantedBy = [ "multi-user.target" ]; inherit (cfg.yarn.resourcemanager) restartIfChanged; + environment = cfg.yarn.resourcemanager.extraEnv; serviceConfig = { User = "yarn"; SyslogIdentifier = "yarn-resourcemanager"; ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + - " resourcemanager"; + " resourcemanager ${escapeShellArgs cfg.yarn.resourcemanager.extraFlags}"; Restart = "always"; }; }; + + services.hadoop.gatewayRole.enable = true; + networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [ 8088 # resourcemanager.webapp.address 8030 # resourcemanager.scheduler.address @@ -94,6 +149,7 @@ in description = "Hadoop YARN NodeManager"; wantedBy = [ "multi-user.target" ]; inherit (cfg.yarn.nodemanager) restartIfChanged; + environment = cfg.yarn.nodemanager.extraEnv; preStart = '' # create log dir @@ -101,8 +157,9 @@ in chown yarn:hadoop /var/log/hadoop/yarn/nodemanager # set up setuid container executor binary + umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true rm -rf /run/wrappers/yarn-nodemanager/ || true - mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop} + mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu} cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor @@ -114,11 +171,26 @@ in SyslogIdentifier = "yarn-nodemanager"; PermissionsStartOnly = true; ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + - " nodemanager"; + " nodemanager ${escapeShellArgs cfg.yarn.nodemanager.extraFlags}"; Restart = "always"; }; }; + services.hadoop.gatewayRole.enable = true; + + services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; { + "yarn.nodemanager.local-dirs" = localDir; + "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores; + "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB; + "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; + "yarn.nodemanager.resource.memory-mb" = resource.memoryMB; + } // mkIf useCGroups { + "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; + "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; + "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; + "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup"; + }; + networking.firewall.allowedTCPPortRanges = [ (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;}) ]; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 6f4c6213595..b2b35119ce3 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -189,9 +189,9 @@ in grocy = handleTest ./grocy.nix {}; grub = handleTest ./grub.nix {}; gvisor = handleTest ./gvisor.nix {}; - hadoop.all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hadoop.nix {}; - hadoop.hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hdfs.nix {}; - hadoop.yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/yarn.nix {}; + hadoop = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop; }; + hadoop_3_2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop_3_2; }; + hadoop2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop2; }; haka = handleTest ./haka.nix {}; haproxy = handleTest ./haproxy.nix {}; hardened = handleTest ./hardened.nix {}; diff --git a/nixos/tests/hadoop/default.nix b/nixos/tests/hadoop/default.nix new file mode 100644 index 00000000000..d2a97cbeffb --- /dev/null +++ b/nixos/tests/hadoop/default.nix @@ -0,0 +1,7 @@ +{ handleTestOn, package, ... }: + +{ + all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; }; + hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; }; + yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; }; +} diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix index 48737debab5..b132f4fa58b 100644 --- a/nixos/tests/hadoop/hadoop.nix +++ b/nixos/tests/hadoop/hadoop.nix @@ -1,121 +1,148 @@ # This test is very comprehensive. It tests whether all hadoop services work well with each other. # Run this when updating the Hadoop package or making significant changes to the hadoop module. # For a more basic test, see hdfs.nix and yarn.nix -import ../make-test-python.nix ({pkgs, ...}: { +import ../make-test-python.nix ({ package, ... }: { + name = "hadoop-combined"; - nodes = let - package = pkgs.hadoop; - coreSite = { - "fs.defaultFS" = "hdfs://ns1"; - }; - hdfsSite = { - "dfs.namenode.rpc-bind-host" = "0.0.0.0"; - "dfs.namenode.http-bind-host" = "0.0.0.0"; - "dfs.namenode.servicerpc-bind-host" = "0.0.0.0"; + nodes = + let + coreSite = { + "fs.defaultFS" = "hdfs://ns1"; + }; + hdfsSite = { + # HA Quorum Journal Manager configuration + "dfs.nameservices" = "ns1"; + "dfs.ha.namenodes.ns1" = "nn1,nn2"; + "dfs.namenode.shared.edits.dir.ns1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1"; + "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020"; + "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020"; + "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022"; + "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022"; + "dfs.namenode.http-address.ns1.nn1" = "nn1:9870"; + "dfs.namenode.http-address.ns1.nn2" = "nn2:9870"; - # HA Quorum Journal Manager configuration - "dfs.nameservices" = "ns1"; - "dfs.ha.namenodes.ns1" = "nn1,nn2"; - "dfs.namenode.shared.edits.dir.ns1.nn1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1"; - "dfs.namenode.shared.edits.dir.ns1.nn2" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1"; - "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020"; - "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020"; - "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022"; - "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022"; - "dfs.namenode.http-address.ns1.nn1" = "nn1:9870"; - "dfs.namenode.http-address.ns1.nn2" = "nn2:9870"; + # Automatic failover configuration + "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"; + "dfs.ha.automatic-failover.enabled.ns1" = "true"; + "dfs.ha.fencing.methods" = "shell(true)"; + "ha.zookeeper.quorum" = "zk1:2181"; + }; + yarnSite = { + "yarn.resourcemanager.zk-address" = "zk1:2181"; + "yarn.resourcemanager.ha.enabled" = "true"; + "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2"; + "yarn.resourcemanager.hostname.rm1" = "rm1"; + "yarn.resourcemanager.hostname.rm2" = "rm2"; + "yarn.resourcemanager.ha.automatic-failover.enabled" = "true"; + "yarn.resourcemanager.cluster-id" = "cluster1"; + # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in + # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70 + # that causes AM containers to fail otherwise. + "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088"; + "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088"; + }; + in + { + zk1 = { ... }: { + services.zookeeper.enable = true; + networking.firewall.allowedTCPPorts = [ 2181 ]; + }; - # Automatic failover configuration - "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"; - "dfs.ha.automatic-failover.enabled.ns1" = "true"; - "dfs.ha.fencing.methods" = "shell(true)"; - "ha.zookeeper.quorum" = "zk1:2181"; - }; - yarnSiteHA = { - "yarn.resourcemanager.zk-address" = "zk1:2181"; - "yarn.resourcemanager.ha.enabled" = "true"; - "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2"; - "yarn.resourcemanager.hostname.rm1" = "rm1"; - "yarn.resourcemanager.hostname.rm2" = "rm2"; - "yarn.resourcemanager.ha.automatic-failover.enabled" = "true"; - "yarn.resourcemanager.cluster-id" = "cluster1"; - # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in - # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70 - # that causes AM containers to fail otherwise. - "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088"; - "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088"; - }; - in { - zk1 = { ... }: { - services.zookeeper.enable = true; - networking.firewall.allowedTCPPorts = [ 2181 ]; - }; + # HDFS cluster + nn1 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.namenode = { + enable = true; + openFirewall = true; + }; + hdfs.zkfc.enable = true; + }; + }; + nn2 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.namenode = { + enable = true; + openFirewall = true; + }; + hdfs.zkfc.enable = true; + }; + }; - # HDFS cluster - nn1 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.namenode.enable = true; - hdfs.zkfc.enable = true; + jn1 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.journalnode = { + enable = true; + openFirewall = true; + }; + }; }; - }; - nn2 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.namenode.enable = true; - hdfs.zkfc.enable = true; + jn2 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.journalnode = { + enable = true; + openFirewall = true; + }; + }; + }; + jn3 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.journalnode = { + enable = true; + openFirewall = true; + }; + }; }; - }; - jn1 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.journalnode.enable = true; + dn1 = { ... }: { + services.hadoop = { + inherit package coreSite hdfsSite; + hdfs.datanode = { + enable = true; + openFirewall = true; + }; + }; }; - }; - jn2 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.journalnode.enable = true; - }; - }; - jn3 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.journalnode.enable = true; - }; - }; - dn1 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - hdfs.datanode.enable = true; + # YARN cluster + rm1 = { options, ... }: { + services.hadoop = { + inherit package coreSite hdfsSite yarnSite; + yarn.resourcemanager = { + enable = true; + openFirewall = true; + }; + }; }; - }; - - # YARN cluster - rm1 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA; - yarn.resourcemanager.enable = true; + rm2 = { options, ... }: { + services.hadoop = { + inherit package coreSite hdfsSite yarnSite; + yarn.resourcemanager = { + enable = true; + openFirewall = true; + }; + }; }; - }; - rm2 = {pkgs, options, ...}: { - services.hadoop = { - inherit package coreSite hdfsSite; - yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA; - yarn.resourcemanager.enable = true; + nm1 = { options, ... }: { + virtualisation.memorySize = 2048; + services.hadoop = { + inherit package coreSite hdfsSite yarnSite; + yarn.nodemanager = { + enable = true; + openFirewall = true; + }; + }; }; - }; - nm1 = {pkgs, options, ...}: { - virtualisation.memorySize = 2048; - services.hadoop = { - inherit package coreSite hdfsSite; - yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA; - yarn.nodemanager.enable = true; + client = { options, ... }: { + services.hadoop = { + gatewayRole.enable = true; + inherit package coreSite hdfsSite yarnSite; + }; }; - }; }; testScript = '' @@ -173,26 +200,26 @@ import ../make-test-python.nix ({pkgs, ...}: { # DN should have started by now, but confirm anyway dn1.wait_for_unit("hdfs-datanode") # Print states of namenodes - dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") + client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") # Wait for cluster to exit safemode - dn1.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait") - dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") + client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait") + client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") # test R/W - dn1.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile") - assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile") + client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile") + assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile") # Test NN failover nn1.succeed("systemctl stop hdfs-namenode") - assert "active" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState") - dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") - assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile") + assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState") + client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") + assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile") nn1.succeed("systemctl start hdfs-namenode") nn1.wait_for_open_port(9870) nn1.wait_for_open_port(8022) nn1.wait_for_open_port(8020) - assert "standby" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState") - dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") + assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState") + client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat") #### YARN tests #### @@ -208,21 +235,21 @@ import ../make-test-python.nix ({pkgs, ...}: { nm1.wait_for_unit("yarn-nodemanager") nm1.wait_for_open_port(8042) nm1.wait_for_open_port(8040) - nm1.wait_until_succeeds("yarn node -list | grep Nodes:1") - nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") - nm1.succeed("sudo -u yarn yarn node -list | systemd-cat") + client.wait_until_succeeds("yarn node -list | grep Nodes:1") + client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") + client.succeed("sudo -u yarn yarn node -list | systemd-cat") # Test RM failover rm1.succeed("systemctl stop yarn-resourcemanager") - assert "standby" not in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState") - nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") + assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState") + client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") rm1.succeed("systemctl start yarn-resourcemanager") rm1.wait_for_unit("yarn-resourcemanager") rm1.wait_for_open_port(8088) - assert "standby" in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState") - nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") + assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState") + client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") - assert "Estimated value of Pi is" in nm1.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10") - assert "SUCCEEDED" in nm1.succeed("yarn application -list -appStates FINISHED") + assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10") + assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED") ''; }) diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix index b63cbf48032..9415500463d 100644 --- a/nixos/tests/hadoop/hdfs.nix +++ b/nixos/tests/hadoop/hdfs.nix @@ -1,32 +1,46 @@ # Test a minimal HDFS cluster with no HA -import ../make-test-python.nix ({...}: { - nodes = { - namenode = {pkgs, ...}: { +import ../make-test-python.nix ({ package, lib, ... }: +with lib; +{ + name = "hadoop-hdfs"; + + nodes = let + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + "hadoop.proxyuser.httpfs.groups" = "*"; + "hadoop.proxyuser.httpfs.hosts" = "*"; + }; + in { + namenode = { pkgs, ... }: { services.hadoop = { - package = pkgs.hadoop; + inherit package; hdfs = { namenode = { enable = true; + openFirewall = true; formatOnInit = true; }; - httpfs.enable = true; - }; - coreSite = { - "fs.defaultFS" = "hdfs://namenode:8020"; - "hadoop.proxyuser.httpfs.groups" = "*"; - "hadoop.proxyuser.httpfs.hosts" = "*"; + httpfs = { + # The NixOS hadoop module only support webHDFS on 3.3 and newer + enable = mkIf (versionAtLeast package.version "3.3") true; + openFirewall = true; + }; }; + inherit coreSite; }; }; - datanode = {pkgs, ...}: { + datanode = { pkgs, ... }: { services.hadoop = { - package = pkgs.hadoop; - hdfs.datanode.enable = true; - coreSite = { - "fs.defaultFS" = "hdfs://namenode:8020"; - "hadoop.proxyuser.httpfs.groups" = "*"; - "hadoop.proxyuser.httpfs.hosts" = "*"; + inherit package; + hdfs.datanode = { + enable = true; + openFirewall = true; + dataDirs = [{ + type = "DISK"; + path = "/tmp/dn1"; + }]; }; + inherit coreSite; }; }; }; @@ -37,21 +51,32 @@ import ../make-test-python.nix ({...}: { namenode.wait_for_unit("hdfs-namenode") namenode.wait_for_unit("network.target") namenode.wait_for_open_port(8020) + namenode.succeed("ss -tulpne | systemd-cat") + namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat") namenode.wait_for_open_port(9870) datanode.wait_for_unit("hdfs-datanode") datanode.wait_for_unit("network.target") + '' + ( if versionAtLeast package.version "3" then '' datanode.wait_for_open_port(9864) datanode.wait_for_open_port(9866) datanode.wait_for_open_port(9867) - namenode.succeed("curl -f http://namenode:9870") datanode.succeed("curl -f http://datanode:9864") + '' else '' + datanode.wait_for_open_port(50075) + datanode.wait_for_open_port(50010) + datanode.wait_for_open_port(50020) + + datanode.succeed("curl -f http://datanode:50075") + '' ) + '' + namenode.succeed("curl -f http://namenode:9870") datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait") datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile") assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile") + '' + optionalString ( versionAtLeast package.version "3.3" ) '' namenode.wait_for_unit("hdfs-httpfs") namenode.wait_for_open_port(14000) assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1") diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix index 09bdb35791c..1bf8e3831f6 100644 --- a/nixos/tests/hadoop/yarn.nix +++ b/nixos/tests/hadoop/yarn.nix @@ -1,22 +1,30 @@ # This only tests if YARN is able to start its services -import ../make-test-python.nix ({...}: { - nodes = { - resourcemanager = {pkgs, ...}: { - services.hadoop.package = pkgs.hadoop; - services.hadoop.yarn.resourcemanager.enable = true; - services.hadoop.yarnSite = { - "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"; - }; - }; - nodemanager = {pkgs, ...}: { - services.hadoop.package = pkgs.hadoop; - services.hadoop.yarn.nodemanager.enable = true; - services.hadoop.yarnSite = { - "yarn.resourcemanager.hostname" = "resourcemanager"; - "yarn.nodemanager.log-dirs" = "/tmp/userlogs"; - }; - }; +import ../make-test-python.nix ({ package, ... }: { + name = "hadoop-yarn"; + nodes = { + resourcemanager = { ... }: { + services.hadoop = { + inherit package; + yarn.resourcemanager = { + enable = true; + openFirewall = true; + }; + }; + }; + nodemanager = { options, lib, ... }: { + services.hadoop = { + inherit package; + yarn.nodemanager = { + enable = true; + openFirewall = true; + }; + yarnSite = options.services.hadoop.yarnSite.default // { + "yarn.resourcemanager.hostname" = "resourcemanager"; + "yarn.nodemanager.log-dirs" = "/tmp/userlogs"; + }; + }; + }; }; testScript = '' diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix index 6a48cc8ada8..adb46540cba 100644 --- a/pkgs/applications/networking/cluster/hadoop/default.nix +++ b/pkgs/applications/networking/cluster/hadoop/default.nix @@ -15,6 +15,8 @@ , zlib , zstd , openssl +, glibc +, nixosTests }: with lib; @@ -22,7 +24,7 @@ with lib; assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; let - common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "" }: + common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "", tests }: stdenv.mkDerivation rec { inherit pname version jdk libPatches untarDir openssl; src = fetchurl { @@ -38,7 +40,10 @@ let installPhase = '' mkdir -p $out/{lib/${untarDir}/conf,bin,lib} mv * $out/lib/${untarDir} - + '' + optionalString stdenv.isLinux '' + # All versions need container-executor, but some versions can't use autoPatchelf because of broken SSL versions + patchelf --set-interpreter ${glibc.out}/lib64/ld-linux-x86-64.so.2 $out/lib/${untarDir}/bin/container-executor + '' + '' for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do makeWrapper "$n" "$out/bin/$(basename $n)"\ --set-default JAVA_HOME ${jdk.home}\ @@ -49,6 +54,8 @@ let done '' + libPatches; + passthru = { inherit tests; }; + meta = { homepage = "https://hadoop.apache.org/"; description = "Framework for distributed processing of large data sets across clusters of computers"; @@ -73,30 +80,34 @@ in { # Different version of hadoop support different java runtime versions # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions - hadoop_3_3 = - common - (rec { - pname = "hadoop"; - version = "3.3.1"; - untarDir = "${pname}-${version}"; - sha256 = rec { - x86_64-linux = "1b3v16ihysqaxw8za1r5jlnphy8dwhivdx2d0z64309w57ihlxxd"; - x86_64-darwin = x86_64-linux; - aarch64-linux = "00ln18vpi07jq2slk3kplyhcj8ad41n0yl880q5cihilk7daclxz"; - aarch64-darwin = aarch64-linux; - }; - - inherit openssl; - nativeLibs = [ stdenv.cc.cc.lib protobuf3_7 zlib snappy ]; - libPatches = '' - ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2 - ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/ - ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/ - ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/ - ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/ - '' + optionalString stdenv.isLinux "patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0"; - jdk = jdk11_headless; - }); + hadoop_3_3 = common rec { + pname = "hadoop"; + version = "3.3.1"; + untarDir = "${pname}-${version}"; + sha256 = rec { + x86_64-linux = "1b3v16ihysqaxw8za1r5jlnphy8dwhivdx2d0z64309w57ihlxxd"; + x86_64-darwin = x86_64-linux; + aarch64-linux = "00ln18vpi07jq2slk3kplyhcj8ad41n0yl880q5cihilk7daclxz"; + aarch64-darwin = aarch64-linux; + }; + jdk = jdk11_headless; + inherit openssl; + # TODO: Package and add Intel Storage Acceleration Library + nativeLibs = [ stdenv.cc.cc.lib protobuf3_7 zlib snappy ]; + libPatches = '' + ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2 + ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/ + ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/ + ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/ + ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/ + '' + optionalString stdenv.isLinux '' + # libjvm.so for Java >=11 + patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0 + # Java 8 has libjvm.so at a different path + patchelf --add-rpath ${jdk.home}/jre/lib/amd64/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0 + ''; + tests = nixosTests.hadoop; + }; hadoop_3_2 = common rec { pname = "hadoop"; version = "3.2.2"; @@ -104,11 +115,13 @@ in jdk = jdk8_headless; # not using native libs because of broken openssl_1_0_2 dependency # can be manually overriden + tests = nixosTests.hadoop_3_2; }; hadoop2 = common rec { pname = "hadoop"; version = "2.10.1"; sha256.x86_64-linux = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97"; jdk = jdk8_headless; + tests = nixosTests.hadoop2; }; }