b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

nixos/hadoop: use CGroups to enforce container limits by default

Browse source
This commit is contained in:
illustris 2022-02-28 20:41:43 +05:30
parent 716b0dfaaf
commit bef71d7c53
3 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
nixos/modules/services/cluster/hadoop/default.nix
View file

@ -79,6 +79,10 @@ with lib;
"yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
"yarn.resourcemanager.bind-host" = "0.0.0.0";
"yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
"yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
"yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
"yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
"yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup";
};
type = types.attrsOf types.anything;
example = literalExpression ''
@ -124,6 +128,7 @@ with lib;
"yarn.nodemanager.linux-container-executor.group"="hadoop";
"min.user.id"=1000;
"feature.terminal.enabled"=1;
"feature.mount-cgroup.enabled" = 1;
};
type = types.attrsOf types.anything;
example = literalExpression ''

3
nixos/modules/services/cluster/hadoop/yarn.nix
View file

@ -101,8 +101,9 @@ in
chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
# set up setuid container executor binary
umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
rm -rf /run/wrappers/yarn-nodemanager/ || true
mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop}
mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor

4
nixos/tests/hadoop/yarn.nix
View file

@ -12,14 +12,14 @@ import ../make-test-python.nix ({ package, ... }: {
};
};
};
nodemanager = { ... }: {
nodemanager = { options, lib, ... }: {
services.hadoop = {
inherit package;
yarn.nodemanager = {
enable = true;
openFirewall = true;
};
yarnSite = {
yarnSite = options.services.hadoop.yarnSite.default // {
"yarn.resourcemanager.hostname" = "resourcemanager";
"yarn.nodemanager.log-dirs" = "/tmp/userlogs";
};