b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

nixos/hadoop: use CGroups to enforce container limits by default

Browse source
This commit is contained in:
illustris 2022-02-28 20:41:43 +05:30
parent 716b0dfaaf
commit bef71d7c53
3 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
nixos/modules/services/cluster/hadoop/default.nix
View file

@ -79,6 +79,10 @@ with lib;
"yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager"; "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
"yarn.resourcemanager.bind-host" = "0.0.0.0"; "yarn.resourcemanager.bind-host" = "0.0.0.0";
"yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
"yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
"yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
"yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
"yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup";
}; };
type = types.attrsOf types.anything; type = types.attrsOf types.anything;
example = literalExpression '' example = literalExpression ''
@ -124,6 +128,7 @@ with lib;
"yarn.nodemanager.linux-container-executor.group"="hadoop"; "yarn.nodemanager.linux-container-executor.group"="hadoop";
"min.user.id"=1000; "min.user.id"=1000;
"feature.terminal.enabled"=1; "feature.terminal.enabled"=1;
"feature.mount-cgroup.enabled" = 1;
}; };
type = types.attrsOf types.anything; type = types.attrsOf types.anything;
example = literalExpression '' example = literalExpression ''

3
nixos/modules/services/cluster/hadoop/yarn.nix
View file

@ -101,8 +101,9 @@ in
chown yarn:hadoop /var/log/hadoop/yarn/nodemanager chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
# set up setuid container executor binary # set up setuid container executor binary
umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
rm -rf /run/wrappers/yarn-nodemanager/ || true rm -rf /run/wrappers/yarn-nodemanager/ || true
mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop} mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor

4
nixos/tests/hadoop/yarn.nix
View file

@ -12,14 +12,14 @@ import ../make-test-python.nix ({ package, ... }: {
}; };
}; };
}; };
nodemanager = { ... }: { nodemanager = { options, lib, ... }: {
services.hadoop = { services.hadoop = {
inherit package; inherit package;
yarn.nodemanager = { yarn.nodemanager = {
enable = true; enable = true;
openFirewall = true; openFirewall = true;
}; };
yarnSite = { yarnSite = options.services.hadoop.yarnSite.default // {
"yarn.resourcemanager.hostname" = "resourcemanager"; "yarn.resourcemanager.hostname" = "resourcemanager";
"yarn.nodemanager.log-dirs" = "/tmp/userlogs"; "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
}; };