hydra: 2020-02-06 -> 2020-03-{24,27}

Upgrades Hydra to the latest master/flake branch. To perform this
upgrade, it's needed to do a non-trivial db-migration which provides a
massive performance-improvement[1].

The basic ideas behind multi-step upgrades of services between NixOS versions
have been gathered already[2]. For further context it's recommended to
read this first.

Basically, the following steps are needed:

* Upgrade to a non-breaking version of Hydra with the db-changes
  (columns are still nullable here). If `system.stateVersion` is set to
  something older than 20.03, the package will be selected
  automatically, otherwise `pkgs.hydra-migration` needs to be used.

* Run `hydra-backfill-ids` on the server.

* Deploy either `pkgs.hydra-unstable` (for Hydra master) or
  `pkgs.hydra-flakes` (for flakes-support) to activate the optimization.

The steps are also documented in the release-notes and in the module
using `warnings`.

`pkgs.hydra` has been removed as latest Hydra doesn't compile with
`pkgs.nixStable` and to ensure a graceful migration using the newly
introduced packages.

To verify the approach, a simple vm-test has been added which verifies
the migration steps.

[1] https://github.com/NixOS/hydra/pull/711
[2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471
This commit is contained in:
Maximilian Bosch 2020-03-27 22:09:13 +01:00
parent ea5c800175
commit bd5324c4fc
No known key found for this signature in database
GPG key ID: 091DBF4D1FC46B8E
9 changed files with 444 additions and 231 deletions

View file

@ -697,6 +697,66 @@ auth required pam_succeed_if.so uid >= 1000 quiet
</para>
</warning>
</listitem>
<listitem>
<para>
<package>Hydra</package> has gained a massive performance improvement due to
<link xlink:href="https://github.com/NixOS/hydra/pull/710">some database schema
changes</link> by adding several IDs and better indexing. However, it's necessary
to upgrade Hydra in multiple steps:
<itemizedlist>
<listitem>
<para>
At first, an older version of Hydra needs to be deployed which adds those
(nullable) columns. When having set <link linkend="opt-system.stateVersion">stateVersion
</link> to a value older than <literal>20.03</literal>, this package will be selected
by default from the module when upgrading. Otherwise, the package can be deployed using
the following config:
<programlisting>{ pkgs, ... }: {
<link linkend="opt-services.hydra.package">services.hydra.package</link> = pkgs.hydra-migration;
}</programlisting>
</para>
</listitem>
<listitem>
<para>
Automatically fill the newly added ID columns on the server by running the following
command:
<screen>
<prompt>$ </prompt>hydra-backfill-ids
</screen>
<warning>
<para>Please note that this process can take a while depending on your database-size!</para>
</warning>
</para>
</listitem>
<listitem>
<para>
Deploy a newer version of Hydra to activate the DB optimizations. You can choose from
either <package>hydra-unstable</package> (latest <literal>master</literal> compiled
against <package>nixUnstable</package>) and <package>hydra-flakes</package> (latest
version with flake-support).
<warning>
<para>
If your <link linkend="opt-system.stateVersion">stateVersion</link> is set to
<literal>20.03</literal> or greater, <package>hydra-unstable</package> will be used
automatically! This will break your setup if you didn't run the migration.
</para>
</warning>
Please note that Hydra is currently not available with <package>nixStable</package>
as this doesn't compile anymore.
</para>
</listitem>
</itemizedlist>
<warning>
<para>
<package>pkgs.hydra</package> has been removed to ensure a graceful database-migration
using the dedicated package-attributes. If you still have <package>pkgs.hydra</package>
defined in e.g. an overlay, an assertion error will be thrown. To circumvent this,
you need to set <xref linkend="opt-services.hydra.package" /> to <package>pkgs.hydra</package>
explicitly and make sure you know what you're doing!
</para>
</warning>
</para>
</listitem>
</itemizedlist>
</section>

View file

@ -37,6 +37,8 @@ let
haveLocalDB = cfg.dbi == localDB;
inherit (config.system) stateVersion;
in
{
@ -63,8 +65,7 @@ in
};
package = mkOption {
type = types.path;
default = pkgs.hydra;
type = types.package;
defaultText = "pkgs.hydra";
description = "The Hydra package.";
};
@ -194,6 +195,34 @@ in
config = mkIf cfg.enable {
warnings = optional (cfg.package.migration or false) ''
You're currently deploying an older version of Hydra which is needed to
make some required database changes[1]. As soon as this is done, it's recommended
to run `hydra-backfill-ids` and set `services.hydra.package` to either `pkgs.hydra-unstable`
or `pkgs.hydra-flakes` after that.
[1] https://github.com/NixOS/hydra/pull/711
'';
services.hydra.package = with pkgs;
mkDefault (
if pkgs ? hydra
then throw ''
The Hydra package doesn't exist anymore in `nixpkgs`! It probably exists
due to an overlay. To upgrade Hydra, you need to take two steps as some
bigger changes in the database schema were implemented recently[1]. You first
need to deploy `pkgs.hydra-migration`, run `hydra-backfill-ids` on the server
and then deploy either `pkgs.hydra-unstable` or `pkgs.hydra-flakes`.
If you want to use `pkgs.hydra` from your overlay, please set `services.hydra.package`
explicitly to `pkgs.hydra` and make sure you know what you're doing.
[1] https://github.com/NixOS/hydra/pull/711
''
else if versionOlder stateVersion "20.03" then hydra-migration
else hydra-unstable
);
users.groups.hydra = {
gid = config.ids.gids.hydra;
};

View file

@ -127,6 +127,7 @@ in
home-assistant = handleTest ./home-assistant.nix {};
hound = handleTest ./hound.nix {};
hydra = handleTest ./hydra {};
hydra-db-migration = handleTest ./hydra/db-migration.nix {};
i3wm = handleTest ./i3wm.nix {};
icingaweb2 = handleTest ./icingaweb2.nix {};
iftop = handleTest ./iftop.nix {};

View file

@ -0,0 +1,47 @@
{ system, ... }:
{
baseConfig = { pkgs, ... }: let
trivialJob = pkgs.writeTextDir "trivial.nix" ''
{ trivial = builtins.derivation {
name = "trivial";
system = "${system}";
builder = "/bin/sh";
allowSubstitutes = false;
preferLocalBuild = true;
args = ["-c" "echo success > $out; exit 0"];
};
}
'';
createTrivialProject = pkgs.stdenv.mkDerivation {
name = "create-trivial-project";
dontUnpack = true;
buildInputs = [ pkgs.makeWrapper ];
installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
postFixup = ''
wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
'';
};
in {
virtualisation.memorySize = 2048;
time.timeZone = "UTC";
environment.systemPackages = [ createTrivialProject pkgs.jq ];
services.hydra = {
enable = true;
# Hydra needs those settings to start up, so we add something not harmfull.
hydraURL = "example.com";
notificationSender = "example@example.com";
extraConfig = ''
email_notification = 1
'';
};
services.postfix.enable = true;
nix = {
buildMachines = [{
hostName = "localhost";
systems = [ system ];
}];
binaryCaches = [];
};
};
}

View file

@ -0,0 +1,86 @@
{ system ? builtins.currentSystem, ... }:
let inherit (import ./common.nix { inherit system; }) baseConfig; in
{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: {
name = "hydra-db-migration";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ ma27 ];
};
nodes = {
original = { pkgs, lib, ... }: {
imports = [ baseConfig ];
# An older version of Hydra before the db change
# for testing purposes.
services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: {
inherit (old) pname;
version = "2020-02-06";
src = pkgs.fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
};
});
};
migration_phase1 = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra.package = pkgs.hydra-migration;
};
finished = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra.package = pkgs.hydra-unstable;
};
};
testScript = { nodes, ... }: let
next = nodes.migration_phase1.config.system.build.toplevel;
finished = nodes.finished.config.system.build.toplevel;
in ''
original.start()
original.wait_for_unit("multi-user.target")
original.wait_for_unit("postgresql.service")
original.wait_for_unit("hydra-init.service")
original.require_unit_state("hydra-queue-runner.service")
original.require_unit_state("hydra-evaluator.service")
original.require_unit_state("hydra-notify.service")
original.succeed("hydra-create-user admin --role admin --password admin")
original.wait_for_open_port(3000)
original.succeed("create-trivial-project.sh")
original.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id" not in out
original.succeed(
"${next}/bin/switch-to-configuration test >&2"
)
original.wait_for_unit("hydra-init.service")
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id|integer|||" in out
original.succeed("hydra-backfill-ids")
original.succeed(
"${finished}/bin/switch-to-configuration test >&2"
)
original.wait_for_unit("hydra-init.service")
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id|integer||not null|" in out
original.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
original.shutdown()
'';
});
}

View file

@ -3,102 +3,57 @@
, pkgs ? import ../../.. { inherit system config; }
}:
with import ../../lib/testing-python.nix { inherit system pkgs; };
with pkgs.lib;
let
trivialJob = pkgs.writeTextDir "trivial.nix" ''
{ trivial = builtins.derivation {
name = "trivial";
system = "${system}";
builder = "/bin/sh";
allowSubstitutes = false;
preferLocalBuild = true;
args = ["-c" "echo success > $out; exit 0"];
};
}
'';
inherit (import ./common.nix { inherit system; }) baseConfig;
createTrivialProject = pkgs.stdenv.mkDerivation {
name = "create-trivial-project";
dontUnpack = true;
buildInputs = [ pkgs.makeWrapper ];
installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
postFixup = ''
wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
hydraPkgs = {
inherit (pkgs) hydra-migration hydra-unstable hydra-flakes;
};
makeHydraTest = with pkgs.lib; name: package: makeTest {
name = "hydra-${name}";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ pstn lewo ma27 ];
};
machine = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra = { inherit package; };
};
testScript = ''
# let the system boot up
machine.wait_for_unit("multi-user.target")
# test whether the database is running
machine.wait_for_unit("postgresql.service")
# test whether the actual hydra daemons are running
machine.wait_for_unit("hydra-init.service")
machine.require_unit_state("hydra-queue-runner.service")
machine.require_unit_state("hydra-evaluator.service")
machine.require_unit_state("hydra-notify.service")
machine.succeed("hydra-create-user admin --role admin --password admin")
# create a project with a trivial job
machine.wait_for_open_port(3000)
# make sure the build as been successfully built
machine.succeed("create-trivial-project.sh")
machine.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
machine.wait_until_succeeds(
'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
)
'';
};
callTest = f: f { inherit system pkgs; };
hydraPkgs = {
inherit (pkgs) nixStable nixUnstable nixFlakes;
};
tests = pkgs.lib.flip pkgs.lib.mapAttrs hydraPkgs (name: nix:
callTest (import ../make-test-python.nix ({ pkgs, lib, ... }:
{
name = "hydra-with-${name}";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ pstn lewo ma27 ];
};
machine = { pkgs, ... }:
{
virtualisation.memorySize = 1024;
time.timeZone = "UTC";
environment.systemPackages = [ createTrivialProject pkgs.jq ];
services.hydra = {
enable = true;
#Hydra needs those settings to start up, so we add something not harmfull.
hydraURL = "example.com";
notificationSender = "example@example.com";
package = pkgs.hydra.override { inherit nix; };
extraConfig = ''
email_notification = 1
'';
};
services.postfix.enable = true;
nix = {
buildMachines = [{
hostName = "localhost";
systems = [ system ];
}];
binaryCaches = [];
};
};
testScript = ''
# let the system boot up
machine.wait_for_unit("multi-user.target")
# test whether the database is running
machine.wait_for_unit("postgresql.service")
# test whether the actual hydra daemons are running
machine.wait_for_unit("hydra-init.service")
machine.require_unit_state("hydra-queue-runner.service")
machine.require_unit_state("hydra-evaluator.service")
machine.require_unit_state("hydra-notify.service")
machine.succeed("hydra-create-user admin --role admin --password admin")
# create a project with a trivial job
machine.wait_for_open_port(3000)
# make sure the build as been successfully built
machine.succeed("create-trivial-project.sh")
machine.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
machine.wait_until_succeeds(
'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
)
'';
})));
in
tests
mapAttrs makeHydraTest hydraPkgs

View file

@ -0,0 +1,135 @@
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook, src ? null, version ? null
, migration ? false
}:
with stdenv;
if lib.versions.major nix.version == "1"
then throw "This Hydra version doesn't support Nix 1.x"
else
let
perlDeps = buildEnv {
name = "hydra-perl-deps";
paths = with perlPackages; lib.closePropagation
[ ModulePluggable
CatalystActionREST
CatalystAuthenticationStoreDBIxClass
CatalystDevel
CatalystDispatchTypeRegex
CatalystPluginAccessLog
CatalystPluginAuthorizationRoles
CatalystPluginCaptcha
CatalystPluginSessionStateCookie
CatalystPluginSessionStoreFastMmap
CatalystPluginStackTrace
CatalystPluginUnicodeEncoding
CatalystTraitForRequestProxyBase
CatalystViewDownload
CatalystViewJSON
CatalystViewTT
CatalystXScriptServerStarman
CatalystXRoleApplicator
CryptRandPasswd
DBDPg
DBDSQLite
DataDump
DateTime
DigestSHA1
EmailMIME
EmailSender
FileSlurp
IOCompress
IPCRun
JSON
JSONAny
JSONXS
LWP
LWPProtocolHttps
NetAmazonS3
NetPrometheus
NetStatsd
PadWalker
Readonly
SQLSplitStatement
SetScalar
Starman
SysHostnameLong
TermSizeAny
TestMore
TextDiff
TextTable
XMLSimple
nix
nix.perl-bindings
git
boehmgc
];
};
in stdenv.mkDerivation rec {
pname = "hydra";
inherit stdenv src version;
buildInputs =
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx
gitAndTools.top-git mercurial /*darcs*/ subversion bazaar openssl bzip2 libxslt
perlDeps perl nix
postgresql # for running the tests
nlohmann_json
boost
];
hydraPath = lib.makeBinPath (
[ sqlite subversion openssh nix coreutils findutils pixz
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial /*darcs*/ gnused bazaar
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
nativeBuildInputs = [ autoreconfHook pkgconfig ];
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
NIX_CFLAGS_COMPILE = "-pthread";
shellHook = ''
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH
PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
'';
enableParallelBuilding = true;
preCheck = ''
patchShebangs .
export LOGNAME=''${LOGNAME:-foo}
'';
postInstall = ''
mkdir -p $out/nix-support
for i in $out/bin/*; do
read -n 4 chars < $i
if [[ $chars =~ ELF ]]; then continue; fi
wrapProgram $i \
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
--prefix PATH ':' $out/bin:$hydraPath \
--set HYDRA_RELEASE ${version} \
--set HYDRA_HOME $out/libexec/hydra \
--set NIX_RELEASE ${nix.name or "unknown"}
done
''; # */
dontStrip = true;
passthru = { inherit perlDeps migration; };
meta = with stdenv.lib; {
description = "Nix-based continuous build system";
license = licenses.gpl3;
platforms = platforms.linux;
maintainers = with maintainers; [ ma27 ];
};
}

View file

@ -1,143 +1,42 @@
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook
}:
{ fetchFromGitHub, nixStable, nixUnstable, callPackage, nixFlakes }:
with stdenv;
if lib.versions.major nix.version == "1"
then throw "This Hydra version doesn't support Nix 1.x"
else
let
perlDeps = buildEnv {
name = "hydra-perl-deps";
paths = with perlPackages; lib.closePropagation
[ ModulePluggable
CatalystActionREST
CatalystAuthenticationStoreDBIxClass
CatalystDevel
CatalystDispatchTypeRegex
CatalystPluginAccessLog
CatalystPluginAuthorizationRoles
CatalystPluginCaptcha
CatalystPluginSessionStateCookie
CatalystPluginSessionStoreFastMmap
CatalystPluginStackTrace
CatalystPluginUnicodeEncoding
CatalystTraitForRequestProxyBase
CatalystViewDownload
CatalystViewJSON
CatalystViewTT
CatalystXScriptServerStarman
CatalystXRoleApplicator
CryptRandPasswd
DBDPg
DBDSQLite
DataDump
DateTime
DigestSHA1
EmailMIME
EmailSender
FileSlurp
IOCompress
IPCRun
JSON
JSONAny
JSONXS
LWP
LWPProtocolHttps
NetAmazonS3
NetPrometheus
NetStatsd
PadWalker
Readonly
SQLSplitStatement
SetScalar
Starman
SysHostnameLong
TermSizeAny
TestMore
TextDiff
TextTable
XMLSimple
nix
nix.perl-bindings
git
boehmgc
];
};
in stdenv.mkDerivation rec {
pname = "hydra";
version = "2020-02-06";
inherit stdenv;
src = fetchFromGitHub {
owner = "NixOS";
repo = pname;
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
{
# Package for phase-1 of the db migration for Hydra.
# https://github.com/NixOS/hydra/pull/711
hydra-migration = callPackage ./common.nix {
version = "2020-02-10";
src = fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "add4f610ce6f206fb44702b5a894d877b3a30e3a";
sha256 = "1d8hdgjx2ys0zmixi2ydmimdq7ml20h1ji4amwawcyw59kssh6l3";
};
nix = nixStable;
migration = true;
};
buildInputs =
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx
gitAndTools.top-git mercurial darcs subversion bazaar openssl bzip2 libxslt
guile # optional, for Guile + Guix support
perlDeps perl nix
postgresql # for running the tests
nlohmann_json
boost
];
# Hydra from latest master (or flakes) branch. Contains breaking changes,
# so when having an older version, `pkgs.hydra-migration` should be deployed first.
hydraPath = lib.makeBinPath (
[ sqlite subversion openssh nix coreutils findutils pixz
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial darcs gnused bazaar
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
hydra-unstable = callPackage ./common.nix {
version = "2020-03-24";
src = fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "12cc46cdb36321acd4c982429a86eb0f8f3cc969";
sha256 = "10ipxzdxr47c8w5jg69mbax2ykc7lb5fs9bbdd3iai9wzyfz17ln";
};
nix = nixUnstable;
};
nativeBuildInputs = [ autoreconfHook pkgconfig ];
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
NIX_CFLAGS_COMPILE = "-pthread";
shellHook = ''
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH
PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
'';
enableParallelBuilding = true;
preCheck = ''
patchShebangs .
export LOGNAME=''${LOGNAME:-foo}
'';
postInstall = ''
mkdir -p $out/nix-support
for i in $out/bin/*; do
read -n 4 chars < $i
if [[ $chars =~ ELF ]]; then continue; fi
wrapProgram $i \
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
--prefix PATH ':' $out/bin:$hydraPath \
--set HYDRA_RELEASE ${version} \
--set HYDRA_HOME $out/libexec/hydra \
--set NIX_RELEASE ${nix.name or "unknown"}
done
''; # */
dontStrip = true;
passthru.perlDeps = perlDeps;
meta = with stdenv.lib; {
description = "Nix-based continuous build system";
license = licenses.gpl3;
platforms = platforms.linux;
maintainers = with maintainers; [ ma27 ];
hydra-flakes = callPackage ./common.nix {
version = "2020-03-27";
src = fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "a7540b141d085a7e78c21fda8e8c05907c659b34";
sha256 = "08fs7593w5zs8vh4c66gvrxk6s840pp6hj8nwf51wsa27kg5a943";
};
nix = nixFlakes;
};
}

View file

@ -12249,7 +12249,8 @@ in
hwloc = callPackage ../development/libraries/hwloc {};
hydra = callPackage ../development/tools/misc/hydra { };
inherit (callPackage ../development/tools/misc/hydra { })
hydra-migration hydra-unstable hydra-flakes;
hydra-cli = callPackage ../development/tools/misc/hydra-cli { };