From ebafd551d74a6bf3d315876c401635395c5d84e9 Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Mon, 22 May 2023 15:05:16 +0300 Subject: [PATCH] nixos/hercules-ci-agent: sync module with upstream --- .../hercules-ci-agent/common.nix | 203 +++--------------- .../hercules-ci-agent/default.nix | 8 +- .../hercules-ci-agent/settings.nix | 153 +++++++++++++ 3 files changed, 188 insertions(+), 176 deletions(-) create mode 100644 nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix index 663f3df775c..ea9b5ffbf43 100644 --- a/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix @@ -10,171 +10,18 @@ let inherit (lib) filterAttrs - literalMD literalExpression mkIf mkOption mkRemovedOptionModule mkRenamedOptionModule types + ; - cfg = - config.services.hercules-ci-agent; + cfg = config.services.hercules-ci-agent; - format = pkgs.formats.toml { }; - - settingsModule = { config, ... }: { - freeformType = format.type; - options = { - apiBaseUrl = mkOption { - description = lib.mdDoc '' - API base URL that the agent will connect to. - - When using Hercules CI Enterprise, set this to the URL where your - Hercules CI server is reachable. - ''; - type = types.str; - default = "https://hercules-ci.com"; - }; - baseDirectory = mkOption { - type = types.path; - default = "/var/lib/hercules-ci-agent"; - description = lib.mdDoc '' - State directory (secrets, work directory, etc) for agent - ''; - }; - concurrentTasks = mkOption { - description = lib.mdDoc '' - Number of tasks to perform simultaneously. - - A task is a single derivation build, an evaluation or an effect run. - At minimum, you need 2 concurrent tasks for `x86_64-linux` - in your cluster, to allow for import from derivation. - - `concurrentTasks` can be around the CPU core count or lower if memory is - the bottleneck. - - The optimal value depends on the resource consumption characteristics of your workload, - including memory usage and in-task parallelism. This is typically determined empirically. - - When scaling, it is generally better to have a double-size machine than two machines, - because each split of resources causes inefficiencies; particularly with regards - to build latency because of extra downloads. - ''; - type = types.either types.ints.positive (types.enum [ "auto" ]); - default = "auto"; - }; - labels = mkOption { - description = lib.mdDoc '' - A key-value map of user data. - - This data will be available to organization members in the dashboard and API. - - The values can be of any TOML type that corresponds to a JSON type, but arrays - can not contain tables/objects due to limitations of the TOML library. Values - involving arrays of non-primitive types may not be representable currently. - ''; - type = format.type; - defaultText = literalExpression '' - { - agent.source = "..."; # One of "nixpkgs", "flake", "override" - lib.version = "..."; - pkgs.version = "..."; - } - ''; - }; - workDirectory = mkOption { - description = lib.mdDoc '' - The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation. - ''; - type = types.path; - default = config.baseDirectory + "/work"; - defaultText = literalExpression ''baseDirectory + "/work"''; - }; - staticSecretsDirectory = mkOption { - description = lib.mdDoc '' - This is the default directory to look for statically configured secrets like `cluster-join-token.key`. - - See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration. - ''; - type = types.path; - default = config.baseDirectory + "/secrets"; - defaultText = literalExpression ''baseDirectory + "/secrets"''; - }; - clusterJoinTokenPath = mkOption { - description = lib.mdDoc '' - Location of the cluster-join-token.key file. - - You can retrieve the contents of the file when creating a new agent via - . - - As this value is confidential, it should not be in the store, but - installed using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The contents of the file are used for authentication between the agent and the API. - ''; - type = types.path; - default = config.staticSecretsDirectory + "/cluster-join-token.key"; - defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"''; - }; - binaryCachesPath = mkOption { - description = lib.mdDoc '' - Path to a JSON file containing binary cache secret keys. - - As these values are confidential, they should not be in the store, but - copied over using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The format is described on . - ''; - type = types.path; - default = config.staticSecretsDirectory + "/binary-caches.json"; - defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"''; - }; - secretsJsonPath = mkOption { - description = lib.mdDoc '' - Path to a JSON file containing secrets for effects. - - As these values are confidential, they should not be in the store, but - copied over using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The format is described on . - ''; - type = types.path; - default = config.staticSecretsDirectory + "/secrets.json"; - defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"''; - }; - }; - }; - - # TODO (roberth, >=2022) remove - checkNix = - if !cfg.checkNix - then "" - else if lib.versionAtLeast config.nix.package.version "2.3.10" - then "" - else - pkgs.stdenv.mkDerivation { - name = "hercules-ci-check-system-nix-src"; - inherit (config.nix.package) src patches; - dontConfigure = true; - buildPhase = '' - echo "Checking in-memory pathInfoCache expiry" - if ! grep 'PathInfoCacheValue' src/libstore/store-api.hh >/dev/null; then - cat 1>&2 < + builtins.match ".*(^|\n)[ \t]*trusted-users[ \t]*=.*" config.nix.extraOptions == null; + message = '' + hercules-ci-agent: Please do not set `trusted-users` in `nix.extraOptions`. + + The hercules-ci-agent module by default relies on `nix.settings.trusted-users` + to be effectful, but a line like `trusted-users = ...` in `nix.extraOptions` + will override the value set in `nix.settings.trusted-users`. + + Instead of setting `trusted-users` in the `nix.extraOptions` string, you should + set an option with additive semantics, such as + - the NixOS option `nix.settings.trusted-users`, or + - the Nix option in the `extraOptions` string, `extra-trusted-users` + ''; + } + ]; + nix.extraOptions = '' # A store path that was missing at first may well have finished building, # even shortly after the previous lookup. This *also* applies to the daemon. narinfo-cache-negative-ttl = 0 @@ -251,14 +109,9 @@ in services.hercules-ci-agent = { tomlFile = format.generate "hercules-ci-agent.toml" cfg.settings; - - settings.labels = { - agent.source = - if options.services.hercules-ci-agent.package.highestPrio == (lib.modules.mkOptionDefault { }).priority - then "nixpkgs" - else lib.mkOptionDefault "override"; - pkgs.version = pkgs.lib.version; - lib.version = lib.version; + settings.config._module.args = { + packageOption = options.services.hercules-ci-agent.package; + inherit pkgs; }; }; }; diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix index e28493ce776..ad26b5316dd 100644 --- a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix @@ -36,8 +36,14 @@ in Restart = "on-failure"; RestartSec = 120; - LimitSTACK = 256 * 1024 * 1024; + # If a worker goes OOM, don't kill the main process. It needs to + # report the failure and it's unlikely to be part of the problem. OOMPolicy = "continue"; + + # Work around excessive stack use by libstdc++ regex + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86164 + # A 256 MiB stack allows between 400 KiB and 1.5 MiB file to be matched by ".*". + LimitSTACK = 256 * 1024 * 1024; }; }; diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix new file mode 100644 index 00000000000..8eb902313ee --- /dev/null +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix @@ -0,0 +1,153 @@ +# Not a module +{ pkgs, lib }: +let + inherit (lib) + types + literalExpression + mkOption + ; + + format = pkgs.formats.toml { }; + + settingsModule = { config, packageOption, pkgs, ... }: { + freeformType = format.type; + options = { + apiBaseUrl = mkOption { + description = lib.mdDoc '' + API base URL that the agent will connect to. + + When using Hercules CI Enterprise, set this to the URL where your + Hercules CI server is reachable. + ''; + type = types.str; + default = "https://hercules-ci.com"; + }; + baseDirectory = mkOption { + type = types.path; + default = "/var/lib/hercules-ci-agent"; + description = lib.mdDoc '' + State directory (secrets, work directory, etc) for agent + ''; + }; + concurrentTasks = mkOption { + description = lib.mdDoc '' + Number of tasks to perform simultaneously. + + A task is a single derivation build, an evaluation or an effect run. + At minimum, you need 2 concurrent tasks for `x86_64-linux` + in your cluster, to allow for import from derivation. + + `concurrentTasks` can be around the CPU core count or lower if memory is + the bottleneck. + + The optimal value depends on the resource consumption characteristics of your workload, + including memory usage and in-task parallelism. This is typically determined empirically. + + When scaling, it is generally better to have a double-size machine than two machines, + because each split of resources causes inefficiencies; particularly with regards + to build latency because of extra downloads. + ''; + type = types.either types.ints.positive (types.enum [ "auto" ]); + default = "auto"; + defaultText = lib.literalMD '' + `"auto"`, meaning equal to the number of CPU cores. + ''; + }; + labels = mkOption { + description = lib.mdDoc '' + A key-value map of user data. + + This data will be available to organization members in the dashboard and API. + + The values can be of any TOML type that corresponds to a JSON type, but arrays + can not contain tables/objects due to limitations of the TOML library. Values + involving arrays of non-primitive types may not be representable currently. + ''; + type = format.type; + defaultText = literalExpression '' + { + agent.source = "..."; # One of "nixpkgs", "flake", "override" + lib.version = "..."; + pkgs.version = "..."; + } + ''; + }; + workDirectory = mkOption { + description = lib.mdDoc '' + The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation. + ''; + type = types.path; + default = config.baseDirectory + "/work"; + defaultText = literalExpression ''baseDirectory + "/work"''; + }; + staticSecretsDirectory = mkOption { + description = lib.mdDoc '' + This is the default directory to look for statically configured secrets like `cluster-join-token.key`. + + See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration. + ''; + type = types.path; + default = config.baseDirectory + "/secrets"; + defaultText = literalExpression ''baseDirectory + "/secrets"''; + }; + clusterJoinTokenPath = mkOption { + description = lib.mdDoc '' + Location of the cluster-join-token.key file. + + You can retrieve the contents of the file when creating a new agent via + . + + As this value is confidential, it should not be in the store, but + installed using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The contents of the file are used for authentication between the agent and the API. + ''; + type = types.path; + default = config.staticSecretsDirectory + "/cluster-join-token.key"; + defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"''; + }; + binaryCachesPath = mkOption { + description = lib.mdDoc '' + Path to a JSON file containing binary cache secret keys. + + As these values are confidential, they should not be in the store, but + copied over using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The format is described on . + ''; + type = types.path; + default = config.staticSecretsDirectory + "/binary-caches.json"; + defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"''; + }; + secretsJsonPath = mkOption { + description = lib.mdDoc '' + Path to a JSON file containing secrets for effects. + + As these values are confidential, they should not be in the store, but + copied over using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The format is described on . + ''; + type = types.path; + default = config.staticSecretsDirectory + "/secrets.json"; + defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"''; + }; + }; + config = { + labels = { + agent.source = + if packageOption.highestPrio == (lib.modules.mkOptionDefault { }).priority + then "nixpkgs" + else lib.mkOptionDefault "override"; + pkgs.version = pkgs.lib.version; + lib.version = lib.version; + }; + }; + }; +in +{ + inherit format settingsModule; +}