From 523c701c0badbad5cadca2f847ef2f42846f4570 Mon Sep 17 00:00:00 2001 From: Vladimir Serov Date: Sat, 27 Nov 2021 01:23:32 +0300 Subject: [PATCH 01/20] stdenv: move overriden stdenv in closure Before that, base stdenv passed non-makeOverridable version of itself inside. This cause it to be lost on package-name.stdenv. --- pkgs/stdenv/generic/default.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index d7fb1b0ba06..49ebc67f854 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -1,6 +1,6 @@ -let lib = import ../../../lib; in lib.makeOverridable ( +let lib = import ../../../lib; stdenv-overridable = lib.makeOverridable ( -{ name ? "stdenv", preHook ? "", initialPath +argsStdenv@{ name ? "stdenv", preHook ? "", initialPath , # If we don't have a C compiler, we might either have `cc = null` or `cc = # throw ...`, but if we do have a C compiler we should definiely have `cc != @@ -81,8 +81,10 @@ let defaultBuildInputs = extraBuildInputs; + stdenv = (stdenv-overridable argsStdenv); + # The stdenv that we are producing. - stdenv = + in derivation ( lib.optionalAttrs (allowedRequisites != null) { allowedRequisites = allowedRequisites @@ -172,6 +174,5 @@ let # "lift" packages like curl from the final stdenv for Linux to # all-packages.nix for that platform (meaning that it has a line # like curl = if stdenv ? curl then stdenv.curl else ...). - // extraAttrs; - -in stdenv) + // extraAttrs +); in stdenv-overridable From 706988fd4937048d30c52f3251965445e133fa76 Mon Sep 17 00:00:00 2001 From: Vladimir Serov Date: Sat, 27 Nov 2021 02:05:36 +0300 Subject: [PATCH 02/20] CODEOWNERS: added cab404 --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7dc14a61cba..8fba792f5eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -32,7 +32,7 @@ /pkgs/top-level/stage.nix @nbp @Ericson2314 @matthewbauer /pkgs/top-level/splice.nix @Ericson2314 @matthewbauer /pkgs/top-level/release-cross.nix @Ericson2314 @matthewbauer -/pkgs/stdenv/generic @Ericson2314 @matthewbauer +/pkgs/stdenv/generic @Ericson2314 @matthewbauer @cab404 /pkgs/stdenv/cross @Ericson2314 @matthewbauer /pkgs/build-support/cc-wrapper @Ericson2314 @orivej /pkgs/build-support/bintools-wrapper @Ericson2314 @orivej From bc9dc56583b257111ac38648e647d50e1039c081 Mon Sep 17 00:00:00 2001 From: Nikolay Korotkiy Date: Sun, 5 Dec 2021 21:27:05 +0300 Subject: [PATCH 03/20] =?UTF-8?q?crow-translate:=202.8.7=20=E2=86=92=202.9?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../applications/misc/crow-translate/default.nix | 8 ++++---- .../dont-fetch-external-libs.patch | 8 ++++---- .../crow-translate/fix-qttranslations-path.patch | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkgs/applications/misc/crow-translate/default.nix b/pkgs/applications/misc/crow-translate/default.nix index c7309449731..71142727a83 100644 --- a/pkgs/applications/misc/crow-translate/default.nix +++ b/pkgs/applications/misc/crow-translate/default.nix @@ -34,8 +34,8 @@ let qonlinetranslator = fetchFromGitHub { owner = "crow-translate"; repo = "QOnlineTranslator"; - rev = "df89083d2f680a8f856b1df00b8846f995cf1fae"; - sha256 = "sha256-I64KGInnYd/QdI5kANJERsF95wMvRlr8kgQhUqXXN/0="; + rev = "1.5.2"; + sha256 = "sha256-iGi25aKwff2hNNx6o4kHZV8gVbEQcMgpTTvop3CoLjM="; }; circleflags = fetchFromGitHub { owner = "HatScripts"; @@ -52,13 +52,13 @@ let in mkDerivation rec { pname = "crow-translate"; - version = "2.8.7"; + version = "2.9.1"; src = fetchFromGitHub { owner = "crow-translate"; repo = pname; rev = version; - sha256 = "sha256-0bq9itbFyzdOhdNuUtdCYLTCIhc91MM+YRhJgXC5PPw="; + sha256 = "sha256-7Zb6PZO8eLeGPEZD37ja+LZydIQdsgy5gMAMtlS4k5Y="; }; patches = [ diff --git a/pkgs/applications/misc/crow-translate/dont-fetch-external-libs.patch b/pkgs/applications/misc/crow-translate/dont-fetch-external-libs.patch index 44c859a1ea0..1eb1f464e55 100644 --- a/pkgs/applications/misc/crow-translate/dont-fetch-external-libs.patch +++ b/pkgs/applications/misc/crow-translate/dont-fetch-external-libs.patch @@ -1,8 +1,8 @@ diff --git i/CMakeLists.txt w/CMakeLists.txt -index faa9417..059b899 100644 +index 375b17c..106efa9 100644 --- i/CMakeLists.txt +++ w/CMakeLists.txt -@@ -101,13 +101,11 @@ qt5_add_translation(QM_FILES +@@ -114,13 +114,11 @@ qt5_add_translation(QM_FILES ) configure_file(src/cmake.h.in cmake.h) @@ -19,7 +19,7 @@ index faa9417..059b899 100644 src/addlanguagedialog.cpp src/addlanguagedialog.ui diff --git i/cmake/ExternalLibraries.cmake w/cmake/ExternalLibraries.cmake -index e2501e1..e15ce6c 100644 +index c92e745..f265f03 100644 --- i/cmake/ExternalLibraries.cmake +++ w/cmake/ExternalLibraries.cmake @@ -2,34 +2,28 @@ include(FetchContent) @@ -46,7 +46,7 @@ index e2501e1..e15ce6c 100644 FetchContent_Declare(QOnlineTranslator - GIT_REPOSITORY https://github.com/crow-translate/QOnlineTranslator -- GIT_TAG df89083d2f680a8f856b1df00b8846f995cf1fae +- GIT_TAG 1.5.2 + SOURCE_DIR @qonlinetranslator@ ) diff --git a/pkgs/applications/misc/crow-translate/fix-qttranslations-path.patch b/pkgs/applications/misc/crow-translate/fix-qttranslations-path.patch index 322fd5e0484..882e4090267 100644 --- a/pkgs/applications/misc/crow-translate/fix-qttranslations-path.patch +++ b/pkgs/applications/misc/crow-translate/fix-qttranslations-path.patch @@ -1,13 +1,13 @@ diff --git i/src/settings/appsettings.cpp w/src/settings/appsettings.cpp -index aa8b357..15e4f74 100644 +index a73371b..b9d66ca 100644 --- i/src/settings/appsettings.cpp +++ w/src/settings/appsettings.cpp -@@ -81,7 +81,7 @@ void AppSettings::applyLanguage(QLocale::Language lang) - QLocale::setDefault(locale); - - s_appTranslator.load(locale, QStringLiteral(PROJECT_NAME), QStringLiteral("_"), QStandardPaths::locate(QStandardPaths::AppDataLocation, QStringLiteral("translations"), QStandardPaths::LocateDirectory)); -- s_qtTranslator.load(locale, QStringLiteral("qtbase"), QStringLiteral("_"), QLibraryInfo::location(QLibraryInfo::TranslationsPath)); -+ s_qtTranslator.load(locale, QStringLiteral("qtbase"), QStringLiteral("_"), QLatin1String("@qttranslations@/translations")); +@@ -75,7 +75,7 @@ void AppSettings::applyLocale(const QLocale &locale) + const QLocale newLocale = locale == defaultLocale() ? QLocale::system() : locale; + QLocale::setDefault(newLocale); + s_appTranslator.load(newLocale, QStringLiteral(PROJECT_NAME), QStringLiteral("_"), QStandardPaths::locate(QStandardPaths::AppDataLocation, QStringLiteral("translations"), QStandardPaths::LocateDirectory)); +- s_qtTranslator.load(newLocale, QStringLiteral("qt"), QStringLiteral("_"), QLibraryInfo::location(QLibraryInfo::TranslationsPath)); ++ s_qtTranslator.load(newLocale, QStringLiteral("qt"), QStringLiteral("_"), QLatin1String("@qttranslations@/translations")); } - QLocale::Language AppSettings::defaultLanguage() + QLocale AppSettings::defaultLocale() From eab69fa28d2e8c54883bc5ada55784930bcba7c4 Mon Sep 17 00:00:00 2001 From: 0x4A6F <0x4A6F@users.noreply.github.com> Date: Mon, 6 Dec 2021 10:26:58 +0100 Subject: [PATCH 04/20] innernet: 1.5.1 -> 1.5.2 --- pkgs/tools/networking/innernet/default.nix | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pkgs/tools/networking/innernet/default.nix b/pkgs/tools/networking/innernet/default.nix index 608a7379ef9..5baaf2e1220 100644 --- a/pkgs/tools/networking/innernet/default.nix +++ b/pkgs/tools/networking/innernet/default.nix @@ -1,16 +1,27 @@ -{ lib, stdenv, rustPlatform, fetchFromGitHub, llvmPackages, sqlite, installShellFiles, Security, libiconv }: +{ lib +, stdenv +, rustPlatform +, fetchFromGitHub +, llvmPackages +, sqlite +, installShellFiles +, Security +, libiconv +, innernet +, testVersion +}: rustPlatform.buildRustPackage rec { pname = "innernet"; - version = "1.5.1"; + version = "1.5.2"; src = fetchFromGitHub { owner = "tonarino"; repo = "innernet"; rev = "v${version}"; - sha256 = "1ja7khvc4cy317ckglnlf11wfmin62ihic061phdp6rmfv95cza0"; + sha256 = "141zjfl125m5lrimam1dbpk40dqfq4vnaz42sbiq1v1avyg684fq"; }; - cargoSha256 = "18xpwav48xv7xm7r3w9qplmv2i18cg09pkahyvs5l4akdjgxyw10"; + cargoSha256 = "0559d0ayysvqs4k46fhgd4r8wa89abgx6rvhlh0gnlnga8vacpw5"; nativeBuildInputs = with llvmPackages; [ llvm @@ -28,21 +39,10 @@ rustPlatform.buildRustPackage rec { installShellCompletion doc/innernet-server.completions.{bash,fish,zsh} ''; - doInstallCheck = true; - installCheckPhase = '' - if [[ "$("$out/bin/${pname}"-server --version)" == "${pname}-server ${version}" ]]; then - echo '${pname}-server smoke check passed' - else - echo '${pname}-server smoke check failed' - return 1 - fi - if [[ "$("$out/bin/${pname}" --version)" == "${pname} ${version}" ]]; then - echo '${pname} smoke check passed' - else - echo '${pname} smoke check failed' - return 1 - fi - ''; + passthru.tests = { + serverVersion = testVersion { package = innernet; command = "innernet-server --version"; }; + version = testVersion { package = innernet; command = "innernet --version"; }; + }; meta = with lib; { description = "A private network system that uses WireGuard under the hood"; From baa438cfc54c3e9ce48e5d0c937f914d84801763 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Mon, 6 Dec 2021 10:01:19 +0000 Subject: [PATCH 05/20] code-minimap: 0.6.1 -> 0.6.2 --- pkgs/tools/misc/code-minimap/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/misc/code-minimap/default.nix b/pkgs/tools/misc/code-minimap/default.nix index cc2d979ca75..6723ad98f48 100644 --- a/pkgs/tools/misc/code-minimap/default.nix +++ b/pkgs/tools/misc/code-minimap/default.nix @@ -7,16 +7,16 @@ rustPlatform.buildRustPackage rec { pname = "code-minimap"; - version = "0.6.1"; + version = "0.6.2"; src = fetchFromGitHub { owner = "wfxr"; repo = pname; rev = "v${version}"; - sha256 = "sha256-eCHmMtndcQJqKmjxhkcLvjMUXApkSnH+7qyG7PDfcwo="; + sha256 = "sha256-nUEmlKqCskPEQCOS2NC6jF4yVDarJeb3p+BKZq/2qvw="; }; - cargoSha256 = "sha256-wKCANWznOJMlQ8T2q39NNNRmgPYMpbkJhXpxojusNsE="; + cargoSha256 = "sha256-yjjoQYYWK9/9fOP5ICnhpuF/07SyCszB9GCDr0GJ0v0="; buildInputs = lib.optional stdenv.isDarwin libiconv; From d74cbf0073ee8678170051f79024623f94c6eb62 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Tue, 7 Dec 2021 04:39:51 +0000 Subject: [PATCH 06/20] qownnotes: 21.11.4 -> 21.12.1 --- pkgs/applications/office/qownnotes/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/office/qownnotes/default.nix b/pkgs/applications/office/qownnotes/default.nix index dfaec869022..851b367ac45 100644 --- a/pkgs/applications/office/qownnotes/default.nix +++ b/pkgs/applications/office/qownnotes/default.nix @@ -5,13 +5,13 @@ mkDerivation rec { pname = "qownnotes"; - version = "21.11.4"; + version = "21.12.1"; src = fetchurl { url = "https://download.tuxfamily.org/${pname}/src/${pname}-${version}.tar.xz"; # Fetch the checksum of current version with curl: # curl https://download.tuxfamily.org/qownnotes/src/qownnotes-.tar.xz.sha256 - sha256 = "3eb025873160cecd4fa35ae5079c150d4aa5dd3152fd58c5e216b592af43e8dc"; + sha256 = "sha256-gTxt3U2/x3VMWYU9cb5FPRvqezdn2K6VGYwEKBi5xFk="; }; nativeBuildInputs = [ qmake qttools ]; From 8c595361d1b8ea6ccf738b7cf377723df5d8fe5d Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Tue, 7 Dec 2021 23:30:52 +0000 Subject: [PATCH 07/20] micromamba: fix libyamlcpp dependency --- pkgs/tools/package-management/micromamba/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/tools/package-management/micromamba/default.nix b/pkgs/tools/package-management/micromamba/default.nix index ad325cd4f41..9f5fbac787d 100644 --- a/pkgs/tools/package-management/micromamba/default.nix +++ b/pkgs/tools/package-management/micromamba/default.nix @@ -38,6 +38,13 @@ let rev = "yaml-cpp-${version}"; sha256 = "0ykkxzxcwwiv8l8r697gyqh1nl582krpvi7m7l6b40ijnk4pw30s"; }; + + patches = [ + (fetchpatch { + url = "https://github.com/jbeder/yaml-cpp/commit/4f48727b365962e31451cd91027bd797bc7d2ee7.patch"; + sha256 = "sha256-jarZAh7NgwL3xXzxijDiAQmC/EC2WYfNMkYHEIQBPhM="; + }) + ]; }); in stdenv.mkDerivation rec { From e1c5ce0b52dd5e7e39eb7d870341036ec525be75 Mon Sep 17 00:00:00 2001 From: Johannes Schleifenbaum Date: Wed, 8 Dec 2021 08:54:52 +0100 Subject: [PATCH 08/20] haruna: 0.7.2 -> 0.7.3 --- pkgs/applications/video/haruna/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/video/haruna/default.nix b/pkgs/applications/video/haruna/default.nix index a9e87861365..5d204e8297a 100644 --- a/pkgs/applications/video/haruna/default.nix +++ b/pkgs/applications/video/haruna/default.nix @@ -27,13 +27,13 @@ mkDerivation rec { pname = "haruna"; - version = "0.7.2"; + version = "0.7.3"; src = fetchFromGitLab { owner = "multimedia"; repo = "haruna"; rev = "v${version}"; - sha256 = "sha256-0s4v3YJhSssp2S9mppMXq0AtWXPIaqOYWPmJgKjXjDE="; + sha256 = "sha256-pFrmTaRvsqxJw34VULzfjx2k56kJgkB96nJtai2D1wY="; domain = "invent.kde.org"; }; From 61a8fae2cb887f365a9c70a31452dc07f5997ef3 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 8 Dec 2021 23:25:18 +0000 Subject: [PATCH 09/20] python3Packages.mahotas: 1.4.11 -> 1.4.12 --- pkgs/development/python-modules/mahotas/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/mahotas/default.nix b/pkgs/development/python-modules/mahotas/default.nix index 707ad9f3c8f..f788486d662 100644 --- a/pkgs/development/python-modules/mahotas/default.nix +++ b/pkgs/development/python-modules/mahotas/default.nix @@ -2,13 +2,13 @@ buildPythonPackage rec { pname = "mahotas"; - version = "1.4.11"; + version = "1.4.12"; src = fetchFromGitHub { owner = "luispedro"; repo = "mahotas"; rev = "v${version}"; - sha256 = "029gvy1fb855pvxvy8zwj44k4s7qpqi0161bg5wldfiprrysn1kw"; + sha256 = "1n19yha1cqyx7hnlici1wkl7n68dh0vbpsyydfhign2c0w9jvg42"; }; propagatedBuildInputs = [ numpy imread pillow scipy freeimage ]; From c81cf6a24297fcde7783d01cf4b8d87b8920c693 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 3 Dec 2021 17:00:22 +0000 Subject: [PATCH 10/20] python3Packages.mahotas: fix freeimage support use a much stronger binding to our specific freeimage that works reliably on linux. previously it didn't and the tests covering freeimage support were just being skipped as they assumed it to be disabled. once the binding works it reveals slight breakage in the tests themselves, fixed with an upstream patch. this breakage was already revealing itsef on darwin as the freeimage binding was "working" there. --- .../python-modules/mahotas/default.nix | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/mahotas/default.nix b/pkgs/development/python-modules/mahotas/default.nix index f788486d662..efc1bec7673 100644 --- a/pkgs/development/python-modules/mahotas/default.nix +++ b/pkgs/development/python-modules/mahotas/default.nix @@ -1,4 +1,15 @@ -{ buildPythonPackage, fetchFromGitHub, pillow, scipy, numpy, pytestCheckHook, imread, freeimage, lib, stdenv }: +{ buildPythonPackage +, fetchFromGitHub +, fetchpatch +, pillow +, scipy +, numpy +, pytestCheckHook +, imread +, freeimage +, lib +, stdenv +}: buildPythonPackage rec { pname = "mahotas"; @@ -11,11 +22,22 @@ buildPythonPackage rec { sha256 = "1n19yha1cqyx7hnlici1wkl7n68dh0vbpsyydfhign2c0w9jvg42"; }; + patches = [ + (fetchpatch { + name = "fix-freeimage-tests.patch"; + url = "https://github.com/luispedro/mahotas/commit/08cc4aa0cbd5dbd4c37580d52b822810c03b2c69.patch"; + sha256 = "0389sz7fyl8h42phw8sn4pxl4wc3brcrj9d05yga21gzil9bfi23"; + excludes = [ "ChangeLog" ]; + }) + ]; + propagatedBuildInputs = [ numpy imread pillow scipy freeimage ]; checkInputs = [ pytestCheckHook ]; postPatch = '' - substituteInPlace mahotas/io/freeimage.py --replace "/opt/local/lib" "${freeimage}/lib" + substituteInPlace mahotas/io/freeimage.py \ + --replace "ctypes.util.find_library('freeimage')" 'True' \ + --replace 'ctypes.CDLL(libname)' 'np.ctypeslib.load_library("libfreeimage", "${freeimage}/lib")' ''; # tests must be run in the build directory @@ -31,6 +53,11 @@ buildPythonPackage rec { "test_haralick3d" ]; + pythonImportsCheck = [ + "mahotas" + "mahotas.freeimage" + ]; + disabled = stdenv.isi686; # Failing tests meta = with lib; { From 1ae96f0cb4b0c79c56cbd1881775c7431fe8009b Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 9 Dec 2021 10:40:03 +0000 Subject: [PATCH 11/20] munin: 2.0.66 -> 2.0.69 --- pkgs/servers/monitoring/munin/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/monitoring/munin/default.nix b/pkgs/servers/monitoring/munin/default.nix index ac9b015480e..28ff8ac8070 100644 --- a/pkgs/servers/monitoring/munin/default.nix +++ b/pkgs/servers/monitoring/munin/default.nix @@ -3,14 +3,14 @@ }: stdenv.mkDerivation rec { - version = "2.0.66"; + version = "2.0.69"; pname = "munin"; src = fetchFromGitHub { owner = "munin-monitoring"; repo = "munin"; rev = version; - sha256 = "sha256-1aikMRY1YiSQNUnYqsw1Eew9D9JHbkX+BXNCof6YK50="; + sha256 = "sha256-p273O5JLFX1dA2caV3lVVL9YNTcGMSrC7DWieUfUmqI="; }; buildInputs = [ From 4d60d9d491f2e018197801d4541073e9f9f90810 Mon Sep 17 00:00:00 2001 From: "Markus S. Wamser" Date: Fri, 10 Dec 2021 12:59:18 +0100 Subject: [PATCH 12/20] crispyDoom: move build dep to python3 --- pkgs/games/crispy-doom/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/games/crispy-doom/default.nix b/pkgs/games/crispy-doom/default.nix index fea0d620a70..c507332399a 100644 --- a/pkgs/games/crispy-doom/default.nix +++ b/pkgs/games/crispy-doom/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, autoreconfHook, pkg-config, SDL2, SDL2_mixer, SDL2_net, fetchFromGitHub, python2 }: +{ lib, stdenv, autoreconfHook, pkg-config, SDL2, SDL2_mixer, SDL2_net, fetchFromGitHub, python3 }: stdenv.mkDerivation rec { pname = "crispy-doom"; @@ -13,10 +13,10 @@ stdenv.mkDerivation rec { postPatch = '' sed -e 's#/games#/bin#g' -i src{,/setup}/Makefile.am - for script in $(grep -lr '^#!/usr/bin/env python$'); do patchShebangs $script; done + for script in $(grep -lr '^#!/usr/bin/env python3$'); do patchShebangs $script; done ''; - nativeBuildInputs = [ autoreconfHook pkg-config python2 ]; + nativeBuildInputs = [ autoreconfHook pkg-config python3 ]; buildInputs = [ SDL2 SDL2_mixer SDL2_net ]; enableParallelBuilding = true; From ab693de868e1a0e6e75fd31027b165dce8420c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Mon, 6 Dec 2021 12:49:23 +0000 Subject: [PATCH 13/20] nixos/test-driver: make the test-driver a python package --- nixos/lib/test-driver/default.nix | 32 ++ nixos/lib/test-driver/setup.py | 13 + nixos/lib/test-driver/test_driver/__init__.py | 100 +++++ nixos/lib/test-driver/test_driver/driver.py | 161 +++++++ nixos/lib/test-driver/test_driver/logger.py | 101 +++++ .../machine.py} | 424 +----------------- nixos/lib/test-driver/test_driver/vlan.py | 58 +++ nixos/lib/testing-python.nix | 73 +-- 8 files changed, 498 insertions(+), 464 deletions(-) create mode 100644 nixos/lib/test-driver/default.nix create mode 100644 nixos/lib/test-driver/setup.py create mode 100755 nixos/lib/test-driver/test_driver/__init__.py create mode 100644 nixos/lib/test-driver/test_driver/driver.py create mode 100644 nixos/lib/test-driver/test_driver/logger.py rename nixos/lib/test-driver/{test-driver.py => test_driver/machine.py} (72%) mode change 100755 => 100644 create mode 100644 nixos/lib/test-driver/test_driver/vlan.py diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix new file mode 100644 index 00000000000..3f63bc705b9 --- /dev/null +++ b/nixos/lib/test-driver/default.nix @@ -0,0 +1,32 @@ +{ lib +, python3Packages +, enableOCR ? false +, qemu_pkg ? qemu_test +, coreutils +, imagemagick_light +, libtiff +, netpbm +, qemu_test +, socat +, tesseract4 +, vde2 +}: + +python3Packages.buildPythonApplication rec { + pname = "nixos-test-driver"; + version = "1.0"; + src = ./.; + + propagatedBuildInputs = [ coreutils netpbm python3Packages.colorama python3Packages.ptpython qemu_pkg socat vde2 ] + ++ (lib.optionals enableOCR [ imagemagick_light tesseract4 ]); + + doCheck = true; + checkInputs = with python3Packages; [ mypy pylint black ]; + checkPhase = '' + mypy --disallow-untyped-defs \ + --no-implicit-optional \ + --ignore-missing-imports ${src}/test_driver + pylint --errors-only ${src}/test_driver + black --check --diff ${src}/test_driver + ''; +} diff --git a/nixos/lib/test-driver/setup.py b/nixos/lib/test-driver/setup.py new file mode 100644 index 00000000000..15699547216 --- /dev/null +++ b/nixos/lib/test-driver/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name="nixos-test-driver", + version='1.0', + packages=find_packages(), + entry_points={ + "console_scripts": [ + "nixos-test-driver=test_driver:main", + "generate-driver-symbols=test_driver:generate_driver_symbols" + ] + }, +) diff --git a/nixos/lib/test-driver/test_driver/__init__.py b/nixos/lib/test-driver/test_driver/__init__.py new file mode 100755 index 00000000000..5477ab5cd03 --- /dev/null +++ b/nixos/lib/test-driver/test_driver/__init__.py @@ -0,0 +1,100 @@ +from pathlib import Path +import argparse +import ptpython.repl +import os +import time + +from test_driver.logger import rootlog +from test_driver.driver import Driver + + +class EnvDefault(argparse.Action): + """An argpars Action that takes values from the specified + environment variable as the flags default value. + """ + + def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs): # type: ignore + if not default and envvar: + if envvar in os.environ: + if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]): + default = os.environ[envvar].split() + else: + default = os.environ[envvar] + kwargs["help"] = ( + kwargs["help"] + f" (default from environment: {default})" + ) + if required and default: + required = False + super(EnvDefault, self).__init__( + default=default, required=required, nargs=nargs, **kwargs + ) + + def __call__(self, parser, namespace, values, option_string=None): # type: ignore + setattr(namespace, self.dest, values) + + +def main() -> None: + arg_parser = argparse.ArgumentParser(prog="nixos-test-driver") + arg_parser.add_argument( + "-K", + "--keep-vm-state", + help="re-use a VM state coming from a previous run", + action="store_true", + ) + arg_parser.add_argument( + "-I", + "--interactive", + help="drop into a python repl and run the tests interactively", + action="store_true", + ) + arg_parser.add_argument( + "--start-scripts", + metavar="START-SCRIPT", + action=EnvDefault, + envvar="startScripts", + nargs="*", + help="start scripts for participating virtual machines", + ) + arg_parser.add_argument( + "--vlans", + metavar="VLAN", + action=EnvDefault, + envvar="vlans", + nargs="*", + help="vlans to span by the driver", + ) + arg_parser.add_argument( + "testscript", + action=EnvDefault, + envvar="testScript", + help="the test script to run", + type=Path, + ) + + args = arg_parser.parse_args() + + if not args.keep_vm_state: + rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state") + + with Driver( + args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state + ) as driver: + if args.interactive: + ptpython.repl.embed(driver.test_symbols(), {}) + else: + tic = time.time() + driver.run_tests() + toc = time.time() + rootlog.info(f"test script finished in {(toc-tic):.2f}s") + + +def generate_driver_symbols() -> None: + """ + This generates a file with symbols of the test-driver code that can be used + in user's test scripts. That list is then used by pyflakes to lint those + scripts. + """ + d = Driver([], [], "") + test_symbols = d.test_symbols() + with open("driver-symbols", "w") as fp: + fp.write(",".join(test_symbols.keys())) diff --git a/nixos/lib/test-driver/test_driver/driver.py b/nixos/lib/test-driver/test_driver/driver.py new file mode 100644 index 00000000000..f3af98537ad --- /dev/null +++ b/nixos/lib/test-driver/test_driver/driver.py @@ -0,0 +1,161 @@ +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, Iterator, List +import os +import tempfile + +from test_driver.logger import rootlog +from test_driver.machine import Machine, NixStartScript, retry +from test_driver.vlan import VLan + + +class Driver: + """A handle to the driver that sets up the environment + and runs the tests""" + + tests: str + vlans: List[VLan] + machines: List[Machine] + + def __init__( + self, + start_scripts: List[str], + vlans: List[int], + tests: str, + keep_vm_state: bool = False, + ): + self.tests = tests + + tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) + tmp_dir.mkdir(mode=0o700, exist_ok=True) + + with rootlog.nested("start all VLans"): + self.vlans = [VLan(nr, tmp_dir) for nr in vlans] + + def cmd(scripts: List[str]) -> Iterator[NixStartScript]: + for s in scripts: + yield NixStartScript(s) + + self.machines = [ + Machine( + start_command=cmd, + keep_vm_state=keep_vm_state, + name=cmd.machine_name, + tmp_dir=tmp_dir, + ) + for cmd in cmd(start_scripts) + ] + + def __enter__(self) -> "Driver": + return self + + def __exit__(self, *_: Any) -> None: + with rootlog.nested("cleanup"): + for machine in self.machines: + machine.release() + + def subtest(self, name: str) -> Iterator[None]: + """Group logs under a given test name""" + with rootlog.nested(name): + try: + yield + return True + except Exception as e: + rootlog.error(f'Test "{name}" failed with error: "{e}"') + raise e + + def test_symbols(self) -> Dict[str, Any]: + @contextmanager + def subtest(name: str) -> Iterator[None]: + return self.subtest(name) + + general_symbols = dict( + start_all=self.start_all, + test_script=self.test_script, + machines=self.machines, + vlans=self.vlans, + driver=self, + log=rootlog, + os=os, + create_machine=self.create_machine, + subtest=subtest, + run_tests=self.run_tests, + join_all=self.join_all, + retry=retry, + serial_stdout_off=self.serial_stdout_off, + serial_stdout_on=self.serial_stdout_on, + Machine=Machine, # for typing + ) + machine_symbols = {m.name: m for m in self.machines} + # If there's exactly one machine, make it available under the name + # "machine", even if it's not called that. + if len(self.machines) == 1: + (machine_symbols["machine"],) = self.machines + vlan_symbols = { + f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans) + } + print( + "additionally exposed symbols:\n " + + ", ".join(map(lambda m: m.name, self.machines)) + + ",\n " + + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans)) + + ",\n " + + ", ".join(list(general_symbols.keys())) + ) + return {**general_symbols, **machine_symbols, **vlan_symbols} + + def test_script(self) -> None: + """Run the test script""" + with rootlog.nested("run the VM test script"): + symbols = self.test_symbols() # call eagerly + exec(self.tests, symbols, None) + + def run_tests(self) -> None: + """Run the test script (for non-interactive test runs)""" + self.test_script() + # TODO: Collect coverage data + for machine in self.machines: + if machine.is_up(): + machine.execute("sync") + + def start_all(self) -> None: + """Start all machines""" + with rootlog.nested("start all VMs"): + for machine in self.machines: + machine.start() + + def join_all(self) -> None: + """Wait for all machines to shut down""" + with rootlog.nested("wait for all VMs to finish"): + for machine in self.machines: + machine.wait_for_shutdown() + + def create_machine(self, args: Dict[str, Any]) -> Machine: + rootlog.warning( + "Using legacy create_machine(), please instantiate the" + "Machine class directly, instead" + ) + tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) + tmp_dir.mkdir(mode=0o700, exist_ok=True) + + if args.get("startCommand"): + start_command: str = args.get("startCommand", "") + cmd = NixStartScript(start_command) + name = args.get("name", cmd.machine_name) + else: + cmd = Machine.create_startcommand(args) # type: ignore + name = args.get("name", "machine") + + return Machine( + tmp_dir=tmp_dir, + start_command=cmd, + name=name, + keep_vm_state=args.get("keep_vm_state", False), + allow_reboot=args.get("allow_reboot", False), + ) + + def serial_stdout_on(self) -> None: + rootlog._print_serial_logs = True + + def serial_stdout_off(self) -> None: + rootlog._print_serial_logs = False diff --git a/nixos/lib/test-driver/test_driver/logger.py b/nixos/lib/test-driver/test_driver/logger.py new file mode 100644 index 00000000000..5b3091a5129 --- /dev/null +++ b/nixos/lib/test-driver/test_driver/logger.py @@ -0,0 +1,101 @@ +from colorama import Style +from contextlib import contextmanager +from typing import Any, Dict, Iterator +from queue import Queue, Empty +from xml.sax.saxutils import XMLGenerator +import codecs +import os +import sys +import time +import unicodedata + + +class Logger: + def __init__(self) -> None: + self.logfile = os.environ.get("LOGFILE", "/dev/null") + self.logfile_handle = codecs.open(self.logfile, "wb") + self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8") + self.queue: "Queue[Dict[str, str]]" = Queue() + + self.xml.startDocument() + self.xml.startElement("logfile", attrs={}) + + self._print_serial_logs = True + + @staticmethod + def _eprint(*args: object, **kwargs: Any) -> None: + print(*args, file=sys.stderr, **kwargs) + + def close(self) -> None: + self.xml.endElement("logfile") + self.xml.endDocument() + self.logfile_handle.close() + + def sanitise(self, message: str) -> str: + return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C") + + def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str: + if "machine" in attributes: + return "{}: {}".format(attributes["machine"], message) + return message + + def log_line(self, message: str, attributes: Dict[str, str]) -> None: + self.xml.startElement("line", attributes) + self.xml.characters(message) + self.xml.endElement("line") + + def info(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + + def warning(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + + def error(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + sys.exit(1) + + def log(self, message: str, attributes: Dict[str, str] = {}) -> None: + self._eprint(self.maybe_prefix(message, attributes)) + self.drain_log_queue() + self.log_line(message, attributes) + + def log_serial(self, message: str, machine: str) -> None: + self.enqueue({"msg": message, "machine": machine, "type": "serial"}) + if self._print_serial_logs: + self._eprint( + Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL + ) + + def enqueue(self, item: Dict[str, str]) -> None: + self.queue.put(item) + + def drain_log_queue(self) -> None: + try: + while True: + item = self.queue.get_nowait() + msg = self.sanitise(item["msg"]) + del item["msg"] + self.log_line(msg, item) + except Empty: + pass + + @contextmanager + def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]: + self._eprint(self.maybe_prefix(message, attributes)) + + self.xml.startElement("nest", attrs={}) + self.xml.startElement("head", attributes) + self.xml.characters(message) + self.xml.endElement("head") + + tic = time.time() + self.drain_log_queue() + yield + self.drain_log_queue() + toc = time.time() + self.log("(finished: {}, in {:.2f} seconds)".format(message, toc - tic)) + + self.xml.endElement("nest") + + +rootlog = Logger() diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test_driver/machine.py old mode 100755 new mode 100644 similarity index 72% rename from nixos/lib/test-driver/test-driver.py rename to nixos/lib/test-driver/test_driver/machine.py index 90c9e9be45c..b3dbe5126fc --- a/nixos/lib/test-driver/test-driver.py +++ b/nixos/lib/test-driver/test_driver/machine.py @@ -1,19 +1,11 @@ -#! /somewhere/python3 -from contextlib import contextmanager, _GeneratorContextManager -from queue import Queue, Empty -from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List, Iterable -from xml.sax.saxutils import XMLGenerator -from colorama import Style +from contextlib import _GeneratorContextManager from pathlib import Path -import queue -import io -import threading -import argparse +from queue import Queue +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple import base64 -import codecs +import io import os -import ptpython.repl -import pty +import queue import re import shlex import shutil @@ -21,8 +13,10 @@ import socket import subprocess import sys import tempfile +import threading import time -import unicodedata + +from test_driver.logger import rootlog CHAR_TO_KEY = { "A": "shift-a", @@ -88,115 +82,10 @@ CHAR_TO_KEY = { } -class Logger: - def __init__(self) -> None: - self.logfile = os.environ.get("LOGFILE", "/dev/null") - self.logfile_handle = codecs.open(self.logfile, "wb") - self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8") - self.queue: "Queue[Dict[str, str]]" = Queue() - - self.xml.startDocument() - self.xml.startElement("logfile", attrs={}) - - self._print_serial_logs = True - - @staticmethod - def _eprint(*args: object, **kwargs: Any) -> None: - print(*args, file=sys.stderr, **kwargs) - - def close(self) -> None: - self.xml.endElement("logfile") - self.xml.endDocument() - self.logfile_handle.close() - - def sanitise(self, message: str) -> str: - return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C") - - def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str: - if "machine" in attributes: - return "{}: {}".format(attributes["machine"], message) - return message - - def log_line(self, message: str, attributes: Dict[str, str]) -> None: - self.xml.startElement("line", attributes) - self.xml.characters(message) - self.xml.endElement("line") - - def info(self, *args, **kwargs) -> None: # type: ignore - self.log(*args, **kwargs) - - def warning(self, *args, **kwargs) -> None: # type: ignore - self.log(*args, **kwargs) - - def error(self, *args, **kwargs) -> None: # type: ignore - self.log(*args, **kwargs) - sys.exit(1) - - def log(self, message: str, attributes: Dict[str, str] = {}) -> None: - self._eprint(self.maybe_prefix(message, attributes)) - self.drain_log_queue() - self.log_line(message, attributes) - - def log_serial(self, message: str, machine: str) -> None: - self.enqueue({"msg": message, "machine": machine, "type": "serial"}) - if self._print_serial_logs: - self._eprint( - Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL - ) - - def enqueue(self, item: Dict[str, str]) -> None: - self.queue.put(item) - - def drain_log_queue(self) -> None: - try: - while True: - item = self.queue.get_nowait() - msg = self.sanitise(item["msg"]) - del item["msg"] - self.log_line(msg, item) - except Empty: - pass - - @contextmanager - def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]: - self._eprint(self.maybe_prefix(message, attributes)) - - self.xml.startElement("nest", attrs={}) - self.xml.startElement("head", attributes) - self.xml.characters(message) - self.xml.endElement("head") - - tic = time.time() - self.drain_log_queue() - yield - self.drain_log_queue() - toc = time.time() - self.log("(finished: {}, in {:.2f} seconds)".format(message, toc - tic)) - - self.xml.endElement("nest") - - -rootlog = Logger() - - def make_command(args: list) -> str: return " ".join(map(shlex.quote, (map(str, args)))) -def retry(fn: Callable, timeout: int = 900) -> None: - """Call the given function repeatedly, with 1 second intervals, - until it returns True or a timeout is reached. - """ - - for _ in range(timeout): - if fn(False): - return - time.sleep(1) - - if not fn(True): - raise Exception(f"action timed out after {timeout} seconds") - - def _perform_ocr_on_screenshot( screenshot_path: str, model_ids: Iterable[int] ) -> List[str]: @@ -228,6 +117,20 @@ def _perform_ocr_on_screenshot( return model_results +def retry(fn: Callable, timeout: int = 900) -> None: + """Call the given function repeatedly, with 1 second intervals, + until it returns True or a timeout is reached. + """ + + for _ in range(timeout): + if fn(False): + return + time.sleep(1) + + if not fn(True): + raise Exception(f"action timed out after {timeout} seconds") + + class StartCommand: """The Base Start Command knows how to append the necesary runtime qemu options as determined by a particular test driver @@ -1066,286 +969,3 @@ class Machine: self.shell.close() self.monitor.close() self.serial_thread.join() - - -class VLan: - """This class handles a VLAN that the run-vm scripts identify via its - number handles. The network's lifetime equals the object's lifetime. - """ - - nr: int - socket_dir: Path - - process: subprocess.Popen - pid: int - fd: io.TextIOBase - - def __repr__(self) -> str: - return f"" - - def __init__(self, nr: int, tmp_dir: Path): - self.nr = nr - self.socket_dir = tmp_dir / f"vde{self.nr}.ctl" - - # TODO: don't side-effect environment here - os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir) - - rootlog.info("start vlan") - pty_master, pty_slave = pty.openpty() - - self.process = subprocess.Popen( - ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700"], - stdin=pty_slave, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - ) - self.pid = self.process.pid - self.fd = os.fdopen(pty_master, "w") - self.fd.write("version\n") - - # TODO: perl version checks if this can be read from - # an if not, dies. we could hang here forever. Fix it. - assert self.process.stdout is not None - self.process.stdout.readline() - if not (self.socket_dir / "ctl").exists(): - rootlog.error("cannot start vde_switch") - - rootlog.info(f"running vlan (pid {self.pid})") - - def __del__(self) -> None: - rootlog.info(f"kill vlan (pid {self.pid})") - self.fd.close() - self.process.terminate() - - -class Driver: - """A handle to the driver that sets up the environment - and runs the tests""" - - tests: str - vlans: List[VLan] - machines: List[Machine] - - def __init__( - self, - start_scripts: List[str], - vlans: List[int], - tests: str, - keep_vm_state: bool = False, - ): - self.tests = tests - - tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) - tmp_dir.mkdir(mode=0o700, exist_ok=True) - - with rootlog.nested("start all VLans"): - self.vlans = [VLan(nr, tmp_dir) for nr in vlans] - - def cmd(scripts: List[str]) -> Iterator[NixStartScript]: - for s in scripts: - yield NixStartScript(s) - - self.machines = [ - Machine( - start_command=cmd, - keep_vm_state=keep_vm_state, - name=cmd.machine_name, - tmp_dir=tmp_dir, - ) - for cmd in cmd(start_scripts) - ] - - def __enter__(self) -> "Driver": - return self - - def __exit__(self, *_: Any) -> None: - with rootlog.nested("cleanup"): - for machine in self.machines: - machine.release() - - def subtest(self, name: str) -> Iterator[None]: - """Group logs under a given test name""" - with rootlog.nested(name): - try: - yield - return True - except Exception as e: - rootlog.error(f'Test "{name}" failed with error: "{e}"') - raise e - - def test_symbols(self) -> Dict[str, Any]: - @contextmanager - def subtest(name: str) -> Iterator[None]: - return self.subtest(name) - - general_symbols = dict( - start_all=self.start_all, - test_script=self.test_script, - machines=self.machines, - vlans=self.vlans, - driver=self, - log=rootlog, - os=os, - create_machine=self.create_machine, - subtest=subtest, - run_tests=self.run_tests, - join_all=self.join_all, - retry=retry, - serial_stdout_off=self.serial_stdout_off, - serial_stdout_on=self.serial_stdout_on, - Machine=Machine, # for typing - ) - machine_symbols = {m.name: m for m in self.machines} - # If there's exactly one machine, make it available under the name - # "machine", even if it's not called that. - if len(self.machines) == 1: - (machine_symbols["machine"],) = self.machines - vlan_symbols = { - f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans) - } - print( - "additionally exposed symbols:\n " - + ", ".join(map(lambda m: m.name, self.machines)) - + ",\n " - + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans)) - + ",\n " - + ", ".join(list(general_symbols.keys())) - ) - return {**general_symbols, **machine_symbols, **vlan_symbols} - - def test_script(self) -> None: - """Run the test script""" - with rootlog.nested("run the VM test script"): - symbols = self.test_symbols() # call eagerly - exec(self.tests, symbols, None) - - def run_tests(self) -> None: - """Run the test script (for non-interactive test runs)""" - self.test_script() - # TODO: Collect coverage data - for machine in self.machines: - if machine.is_up(): - machine.execute("sync") - - def start_all(self) -> None: - """Start all machines""" - with rootlog.nested("start all VMs"): - for machine in self.machines: - machine.start() - - def join_all(self) -> None: - """Wait for all machines to shut down""" - with rootlog.nested("wait for all VMs to finish"): - for machine in self.machines: - machine.wait_for_shutdown() - - def create_machine(self, args: Dict[str, Any]) -> Machine: - rootlog.warning( - "Using legacy create_machine(), please instantiate the" - "Machine class directly, instead" - ) - tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) - tmp_dir.mkdir(mode=0o700, exist_ok=True) - - if args.get("startCommand"): - start_command: str = args.get("startCommand", "") - cmd = NixStartScript(start_command) - name = args.get("name", cmd.machine_name) - else: - cmd = Machine.create_startcommand(args) # type: ignore - name = args.get("name", "machine") - - return Machine( - tmp_dir=tmp_dir, - start_command=cmd, - name=name, - keep_vm_state=args.get("keep_vm_state", False), - allow_reboot=args.get("allow_reboot", False), - ) - - def serial_stdout_on(self) -> None: - rootlog._print_serial_logs = True - - def serial_stdout_off(self) -> None: - rootlog._print_serial_logs = False - - -class EnvDefault(argparse.Action): - """An argpars Action that takes values from the specified - environment variable as the flags default value. - """ - - def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs): # type: ignore - if not default and envvar: - if envvar in os.environ: - if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]): - default = os.environ[envvar].split() - else: - default = os.environ[envvar] - kwargs["help"] = ( - kwargs["help"] + f" (default from environment: {default})" - ) - if required and default: - required = False - super(EnvDefault, self).__init__( - default=default, required=required, nargs=nargs, **kwargs - ) - - def __call__(self, parser, namespace, values, option_string=None): # type: ignore - setattr(namespace, self.dest, values) - - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(prog="nixos-test-driver") - arg_parser.add_argument( - "-K", - "--keep-vm-state", - help="re-use a VM state coming from a previous run", - action="store_true", - ) - arg_parser.add_argument( - "-I", - "--interactive", - help="drop into a python repl and run the tests interactively", - action="store_true", - ) - arg_parser.add_argument( - "--start-scripts", - metavar="START-SCRIPT", - action=EnvDefault, - envvar="startScripts", - nargs="*", - help="start scripts for participating virtual machines", - ) - arg_parser.add_argument( - "--vlans", - metavar="VLAN", - action=EnvDefault, - envvar="vlans", - nargs="*", - help="vlans to span by the driver", - ) - arg_parser.add_argument( - "testscript", - action=EnvDefault, - envvar="testScript", - help="the test script to run", - type=Path, - ) - - args = arg_parser.parse_args() - - if not args.keep_vm_state: - rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state") - - with Driver( - args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state - ) as driver: - if args.interactive: - ptpython.repl.embed(driver.test_symbols(), {}) - else: - tic = time.time() - driver.run_tests() - toc = time.time() - rootlog.info(f"test script finished in {(toc-tic):.2f}s") diff --git a/nixos/lib/test-driver/test_driver/vlan.py b/nixos/lib/test-driver/test_driver/vlan.py new file mode 100644 index 00000000000..e5c8f07b4ed --- /dev/null +++ b/nixos/lib/test-driver/test_driver/vlan.py @@ -0,0 +1,58 @@ +from pathlib import Path +import io +import os +import pty +import subprocess + +from test_driver.logger import rootlog + + +class VLan: + """This class handles a VLAN that the run-vm scripts identify via its + number handles. The network's lifetime equals the object's lifetime. + """ + + nr: int + socket_dir: Path + + process: subprocess.Popen + pid: int + fd: io.TextIOBase + + def __repr__(self) -> str: + return f"" + + def __init__(self, nr: int, tmp_dir: Path): + self.nr = nr + self.socket_dir = tmp_dir / f"vde{self.nr}.ctl" + + # TODO: don't side-effect environment here + os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir) + + rootlog.info("start vlan") + pty_master, pty_slave = pty.openpty() + + self.process = subprocess.Popen( + ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700"], + stdin=pty_slave, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + self.pid = self.process.pid + self.fd = os.fdopen(pty_master, "w") + self.fd.write("version\n") + + # TODO: perl version checks if this can be read from + # an if not, dies. we could hang here forever. Fix it. + assert self.process.stdout is not None + self.process.stdout.readline() + if not (self.socket_dir / "ctl").exists(): + rootlog.error("cannot start vde_switch") + + rootlog.info(f"running vlan (pid {self.pid})") + + def __del__(self) -> None: + rootlog.info(f"kill vlan (pid {self.pid})") + self.fd.close() + self.process.terminate() diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 4306d102b2d..365e2271457 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -16,65 +16,6 @@ rec { inherit pkgs; - # Reifies and correctly wraps the python test driver for - # the respective qemu version and with or without ocr support - pythonTestDriver = { - qemu_pkg ? pkgs.qemu_test - , enableOCR ? false - }: - let - name = "nixos-test-driver"; - testDriverScript = ./test-driver/test-driver.py; - ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; }; - imagemagick_tiff = imagemagick_light.override { inherit libtiff; }; - in stdenv.mkDerivation { - inherit name; - - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ (python3.withPackages (p: [ p.ptpython p.colorama ])) ]; - checkInputs = with python3Packages; [ pylint black mypy ]; - - dontUnpack = true; - - preferLocalBuild = true; - - buildPhase = '' - python < $out/test-script ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + ${testDriver}/bin/generate-driver-symbols ${lib.optionalString (!skipLint) '' PYFLAKES_BUILTINS="$( echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)}, - < ${lib.escapeShellArg "${testDriver}/nix-support/driver-symbols"} + < ${lib.escapeShellArg "driver-symbols"} )" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script ''} From 9d86e89892ebb1b088163e0bd649482ec8ee8b78 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Fri, 10 Dec 2021 15:13:52 +0000 Subject: [PATCH 14/20] jgmenu: 4.3.0 -> 4.4.0 --- pkgs/applications/misc/jgmenu/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/misc/jgmenu/default.nix b/pkgs/applications/misc/jgmenu/default.nix index 88274b8664d..45014ad81c9 100644 --- a/pkgs/applications/misc/jgmenu/default.nix +++ b/pkgs/applications/misc/jgmenu/default.nix @@ -15,13 +15,13 @@ stdenv.mkDerivation rec { pname = "jgmenu"; - version = "4.3.0"; + version = "4.4.0"; src = fetchFromGitHub { owner = "johanmalm"; repo = pname; rev = "v${version}"; - sha256 = "sha256-+JO/A7+6/yeYz0tP7vxSi04cS1bEet+3sAs7CYXKxI8="; + sha256 = "sha256-REzrN4tz+cFmKyJeOPOkzjvthsJdY3GButF7RdnzviE="; }; nativeBuildInputs = [ From 5030bcc5637e68b62ffb6cac098861c8f8193d7c Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Fri, 10 Dec 2021 11:08:54 +0100 Subject: [PATCH 15/20] python3Packages.ms-active-directory: init at 1.12.1 --- .../ms-active-directory/default.nix | 49 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 51 insertions(+) create mode 100644 pkgs/development/python-modules/ms-active-directory/default.nix diff --git a/pkgs/development/python-modules/ms-active-directory/default.nix b/pkgs/development/python-modules/ms-active-directory/default.nix new file mode 100644 index 00000000000..3262ba31cb6 --- /dev/null +++ b/pkgs/development/python-modules/ms-active-directory/default.nix @@ -0,0 +1,49 @@ +{ lib +, buildPythonPackage +, dnspython +, fetchFromGitHub +, ldap3 +, pyasn1 +, pycryptodome +, pythonOlder +, pytz +, six +}: + +buildPythonPackage rec { + pname = "ms-active-directory"; + version = "1.12.1"; + format = "setuptools"; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "zorn96"; + repo = "ms_active_directory"; + rev = "v${version}"; + sha256 = "sha256-mErQib8xTgo29iPAtiLnhxLXyFboAzyEW9A/QMseM6k="; + }; + + propagatedBuildInputs = [ + dnspython + ldap3 + pyasn1 + pycryptodome + pytz + six + ]; + + # Module has no tests + doCheck = false; + + pythonImportsCheck = [ + "ms_active_directory" + ]; + + meta = with lib; { + description = "Python module for integrating with Microsoft Active Directory domains"; + homepage = "https://github.com/zorn96/ms_active_directory/"; + license = with licenses; [ mit ]; + maintainers = with maintainers; [ fab ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index bc371b95a3f..627bd855cda 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4994,6 +4994,8 @@ in { mrkd = callPackage ../development/python-modules/mrkd { }; + ms-active-directory = callPackage ../development/python-modules/ms-active-directory { }; + ms-cv = callPackage ../development/python-modules/ms-cv { }; msal = callPackage ../development/python-modules/msal { }; From e3bb91c48b508dc0b53b5a96c2981672c83d5d8b Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Fri, 10 Dec 2021 10:25:55 -0800 Subject: [PATCH 16/20] python3Packages.ansible-lint: limit xdist cores to NIX_BUILD_CORES --- pkgs/development/python-modules/ansible-lint/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/ansible-lint/default.nix b/pkgs/development/python-modules/ansible-lint/default.nix index a6381362b38..2fac5e8a021 100644 --- a/pkgs/development/python-modules/ansible-lint/default.nix +++ b/pkgs/development/python-modules/ansible-lint/default.nix @@ -49,7 +49,7 @@ buildPythonPackage rec { ]; pytestFlagsArray = [ - "--numprocesses" "auto" + "--numprocesses" "$NIX_BUILD_CORES" ]; preCheck = '' From d78f228604d51a746c2741963886c0fc18b6db61 Mon Sep 17 00:00:00 2001 From: Nikolay Korotkiy Date: Thu, 9 Dec 2021 10:58:05 +0300 Subject: [PATCH 17/20] =?UTF-8?q?gpsprune:=2020.4=20=E2=86=92=2021?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/applications/misc/gpsprune/default.nix | 47 ++++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/pkgs/applications/misc/gpsprune/default.nix b/pkgs/applications/misc/gpsprune/default.nix index 583a8a1931b..839bcc281ba 100644 --- a/pkgs/applications/misc/gpsprune/default.nix +++ b/pkgs/applications/misc/gpsprune/default.nix @@ -1,43 +1,48 @@ -{ fetchurl, lib, stdenv, makeDesktopItem, makeWrapper, unzip, jdk }: +{ fetchurl, lib, stdenv, makeDesktopItem, makeWrapper, unzip, jre, copyDesktopItems }: stdenv.mkDerivation rec { pname = "gpsprune"; - version = "20.4"; + version = "21"; src = fetchurl { url = "https://activityworkshop.net/software/gpsprune/gpsprune_${version}.jar"; - sha256 = "sha256-ZTYkKyu0/axf2uLUmQHRW/2bQ6p2zK7xBF66ozbPS2c="; + sha256 = "sha256-FS6nf8K+qfEWDCQwmoxH1laJIONMtwmb/H89SVJtV1E="; }; - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ jdk ]; + dontUnpack = true; - desktopItem = makeDesktopItem { - name = "gpsprune"; - exec = "gpsprune"; - icon = "gpsprune"; - desktopName = "GpsPrune"; - genericName = "GPS Data Editor"; - comment = meta.description; - categories = "Education;Geoscience;"; - }; + nativeBuildInputs = [ makeWrapper copyDesktopItems ]; + buildInputs = [ jre ]; - buildCommand = '' - mkdir -p $out/bin $out/share/java - cp -v $src $out/share/java/gpsprune.jar - makeWrapper ${jdk}/bin/java $out/bin/gpsprune \ + desktopItems = [ + (makeDesktopItem { + name = "gpsprune"; + exec = "gpsprune"; + icon = "gpsprune"; + desktopName = "GpsPrune"; + genericName = "GPS Data Editor"; + comment = meta.description; + categories = "Education;Geoscience;"; + }) + ]; + + installPhase = '' + runHook preInstall + + install -Dm644 ${src} $out/share/java/gpsprune.jar + makeWrapper ${jre}/bin/java $out/bin/gpsprune \ --add-flags "-jar $out/share/java/gpsprune.jar" - mkdir -p $out/share/applications - cp $desktopItem/share/applications"/"* $out/share/applications mkdir -p $out/share/pixmaps ${unzip}/bin/unzip -p $src tim/prune/gui/images/window_icon_64.png > $out/share/pixmaps/gpsprune.png + + runHook postInstall ''; meta = with lib; { description = "Application for viewing, editing and converting GPS coordinate data"; homepage = "https://activityworkshop.net/software/gpsprune/"; license = licenses.gpl2Plus; - maintainers = [ maintainers.rycee ]; + maintainers = with maintainers; [ rycee ]; platforms = platforms.all; }; } From c80eda54d53d4e3360b88e4d9deea8f6e402f0f3 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Fri, 10 Dec 2021 22:18:46 +0100 Subject: [PATCH 18/20] chromiumBeta: 97.0.4692.36 -> 97.0.4692.45 --- .../networking/browsers/chromium/upstream-info.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.json b/pkgs/applications/networking/browsers/chromium/upstream-info.json index badec8ae119..193f1d3e268 100644 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.json +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.json @@ -18,9 +18,9 @@ } }, "beta": { - "version": "97.0.4692.36", - "sha256": "0p0f19svnymql8skx6alb6zy4fmc5115dc2avs8h2mca1q8n5r0s", - "sha256bin64": "08p0rwn4jglrzma1vf4jnyqaffnk0c8xwc7jkgfpkasm43d72zim", + "version": "97.0.4692.45", + "sha256": "1x55hys3340inrwwp4lzw5vq3vrg56288m746d4p2ligvsy19byp", + "sha256bin64": "1jyhq27fb4pzbxlg0ssfz66hza9g8cbsqyx70ydkjqs9sf4yqqcw", "deps": { "gn": { "version": "2021-11-03", From 389c60770a38a64e06a62479b592446baaaec72e Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Fri, 10 Dec 2021 22:23:49 +0100 Subject: [PATCH 19/20] trezor-suite: 21.11.2 -> 21.12.2 --- pkgs/applications/blockchains/trezor-suite/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/blockchains/trezor-suite/default.nix b/pkgs/applications/blockchains/trezor-suite/default.nix index 440fec3c44b..2075c95aad9 100644 --- a/pkgs/applications/blockchains/trezor-suite/default.nix +++ b/pkgs/applications/blockchains/trezor-suite/default.nix @@ -8,7 +8,7 @@ let pname = "trezor-suite"; - version = "21.11.2"; + version = "21.12.2"; name = "${pname}-${version}"; suffix = { @@ -19,8 +19,8 @@ let src = fetchurl { url = "https://github.com/trezor/${pname}/releases/download/v${version}/Trezor-Suite-${version}-${suffix}.AppImage"; sha512 = { # curl -Lfs https://github.com/trezor/trezor-suite/releases/latest/download/latest-linux{-arm64,}.yml | grep ^sha512 | sed 's/: /-/' - aarch64-linux = "sha512-QX5Ak2F1aD846BuGNcP1K/2c77Ut3LK3UiXsUPqiSBGZ9YRgdzROqdGjCVnTBBhxeCfGYQDhWmpuOpNbLr4eYg=="; - x86_64-linux = "sha512-ckMlZoLEq3aLzyhoWf2rRE3XxNQhqo6rUHF2NKoV08sXz+Zth2Lk+P3te1vwFQl+Efryl84RTwVGWKmloZ8k9A=="; + aarch64-linux = "sha512-LzcTFSNN/loYaTDt+QpW8QpSgOTw2097IYdc7mC57Mn4NR/X2hycYZ9ZfZjBh9QFfVu/4R3UN2sA177v6Inomg=="; + x86_64-linux = "sha512-W/voBZrXaJVDN4eSUDD6lyBR9BqboD2k2/azI1pWm1NFUmDZFM+OGzyiPB3n+6SziAhca32Ot5Wy27sfmIjh3g=="; }.${stdenv.hostPlatform.system} or (throw "Unsupported system: ${stdenv.hostPlatform.system}"); }; From e0c33ded5f4e86811efa45a9e32f00a6f7836eeb Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Fri, 10 Dec 2021 18:31:25 +0000 Subject: [PATCH 20/20] gnome.gpaste: 3.42.1 -> 3.42.2 --- pkgs/desktops/gnome/misc/gpaste/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/desktops/gnome/misc/gpaste/default.nix b/pkgs/desktops/gnome/misc/gpaste/default.nix index 98670c99311..83928c04827 100644 --- a/pkgs/desktops/gnome/misc/gpaste/default.nix +++ b/pkgs/desktops/gnome/misc/gpaste/default.nix @@ -17,14 +17,14 @@ }: stdenv.mkDerivation rec { - version = "3.42.1"; + version = "3.42.2"; pname = "gpaste"; src = fetchFromGitHub { owner = "Keruspe"; repo = "GPaste"; rev = "v${version}"; - sha256 = "sha256-yoJ/k9cXXF5ELKF0JXGtxsUjfQ/S1sccLRQOQG7YMXo="; + sha256 = "sha256-VWtq1jPwUHHIDpVaSYQ0FiihlfulRofFmacMyv/buMw="; }; patches = [