From a11a2a6d18df7a33d2804803e733f0255c0b0a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 17 Oct 2022 20:39:24 +0200 Subject: [PATCH 1/7] python310Packages.flask-login: normalise package name --- pkgs/applications/misc/archivy/default.nix | 2 +- pkgs/applications/misc/octoprint/default.nix | 2 +- pkgs/applications/networking/flexget/default.nix | 2 +- pkgs/applications/networking/powerdns-admin/default.nix | 2 +- pkgs/development/python-modules/apache-airflow/default.nix | 4 ++-- pkgs/development/python-modules/flask-appbuilder/default.nix | 4 ++-- .../development/python-modules/flask-security-too/default.nix | 4 ++-- pkgs/development/python-modules/sentry-sdk/default.nix | 2 +- .../python-modules/sqlalchemy-continuum/default.nix | 4 ++-- pkgs/tools/admin/pgadmin/default.nix | 2 +- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 2 +- 12 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pkgs/applications/misc/archivy/default.nix b/pkgs/applications/misc/archivy/default.nix index c8b0e00b127..b146599c2b2 100644 --- a/pkgs/applications/misc/archivy/default.nix +++ b/pkgs/applications/misc/archivy/default.nix @@ -42,7 +42,7 @@ buildPythonApplication rec { click-plugins elasticsearch flask-compress - flask_login + flask-login flask-wtf html2text python-dotenv diff --git a/pkgs/applications/misc/octoprint/default.nix b/pkgs/applications/misc/octoprint/default.nix index 474bdd0b642..c2a4ebbcbf6 100644 --- a/pkgs/applications/misc/octoprint/default.nix +++ b/pkgs/applications/misc/octoprint/default.nix @@ -95,7 +95,7 @@ let flask flask-babel flask_assets - flask_login + flask-login flask-limiter frozendict future diff --git a/pkgs/applications/networking/flexget/default.nix b/pkgs/applications/networking/flexget/default.nix index 7ce86986b70..07519dd74ed 100644 --- a/pkgs/applications/networking/flexget/default.nix +++ b/pkgs/applications/networking/flexget/default.nix @@ -60,7 +60,7 @@ python3Packages.buildPythonApplication rec { cherrypy flask-compress flask-cors - flask_login + flask-login flask-restful flask-restx flask diff --git a/pkgs/applications/networking/powerdns-admin/default.nix b/pkgs/applications/networking/powerdns-admin/default.nix index d889fdcf100..1fc698fe190 100644 --- a/pkgs/applications/networking/powerdns-admin/default.nix +++ b/pkgs/applications/networking/powerdns-admin/default.nix @@ -18,7 +18,7 @@ let }; pythonDeps = with python.pkgs; [ - flask flask_assets flask_login flask-sqlalchemy flask_migrate flask-seasurf flask_mail flask-session flask-sslify + flask flask_assets flask-login flask-sqlalchemy flask_migrate flask-seasurf flask_mail flask-session flask-sslify mysqlclient psycopg2 sqlalchemy cffi configobj cryptography bcrypt requests python-ldap pyotp qrcode dnspython gunicorn python3-saml pytz cssmin rjsmin authlib bravado-core diff --git a/pkgs/development/python-modules/apache-airflow/default.nix b/pkgs/development/python-modules/apache-airflow/default.nix index 091020c7632..438dc46038c 100644 --- a/pkgs/development/python-modules/apache-airflow/default.nix +++ b/pkgs/development/python-modules/apache-airflow/default.nix @@ -20,7 +20,7 @@ , deprecated , dill , flask -, flask_login +, flask-login , flask-appbuilder , flask-caching , flask-session @@ -159,7 +159,7 @@ buildPythonPackage rec { flask-caching flask-session flask-wtf - flask_login + flask-login GitPython graphviz gunicorn diff --git a/pkgs/development/python-modules/flask-appbuilder/default.nix b/pkgs/development/python-modules/flask-appbuilder/default.nix index 2ef78ecf8d4..b122d6020bc 100644 --- a/pkgs/development/python-modules/flask-appbuilder/default.nix +++ b/pkgs/development/python-modules/flask-appbuilder/default.nix @@ -8,7 +8,7 @@ , email-validator , flask , flask-babel -, flask_login +, flask-login , flask-openid , flask-sqlalchemy , flask-wtf @@ -59,7 +59,7 @@ buildPythonPackage rec { email-validator flask flask-babel - flask_login + flask-login flask-openid flask-sqlalchemy flask-wtf diff --git a/pkgs/development/python-modules/flask-security-too/default.nix b/pkgs/development/python-modules/flask-security-too/default.nix index 480e711c1c0..bdefbc3db29 100644 --- a/pkgs/development/python-modules/flask-security-too/default.nix +++ b/pkgs/development/python-modules/flask-security-too/default.nix @@ -25,7 +25,7 @@ , blinker , email-validator , flask -, flask_login +, flask-login , flask_principal , flask-wtf , itsdangerous @@ -57,7 +57,7 @@ buildPythonPackage rec { blinker email-validator flask - flask_login + flask-login flask_principal flask-wtf itsdangerous diff --git a/pkgs/development/python-modules/sentry-sdk/default.nix b/pkgs/development/python-modules/sentry-sdk/default.nix index cd052afdacb..c52cdeef956 100644 --- a/pkgs/development/python-modules/sentry-sdk/default.nix +++ b/pkgs/development/python-modules/sentry-sdk/default.nix @@ -19,7 +19,7 @@ , django , falcon , flask -, flask_login +, flask-login , httpx , pure-eval , pyramid diff --git a/pkgs/development/python-modules/sqlalchemy-continuum/default.nix b/pkgs/development/python-modules/sqlalchemy-continuum/default.nix index a97f3e4ddbd..0d2ef66cfa8 100644 --- a/pkgs/development/python-modules/sqlalchemy-continuum/default.nix +++ b/pkgs/development/python-modules/sqlalchemy-continuum/default.nix @@ -2,7 +2,7 @@ , fetchPypi , buildPythonPackage , flask -, flask_login +, flask-login , flask-sqlalchemy , flexmock , pytestCheckHook @@ -32,7 +32,7 @@ buildPythonPackage rec { pytestCheckHook sqlalchemy-i18n flask - flask_login + flask-login flask-sqlalchemy flexmock ]; diff --git a/pkgs/tools/admin/pgadmin/default.nix b/pkgs/tools/admin/pgadmin/default.nix index a183319b832..c36c555d193 100644 --- a/pkgs/tools/admin/pgadmin/default.nix +++ b/pkgs/tools/admin/pgadmin/default.nix @@ -29,7 +29,7 @@ let buildDeps = with pythonPackages; [ flask flask-gravatar - flask_login + flask-login flask_mail flask_migrate flask-sqlalchemy diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index e6c9434a119..6093211e9a9 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -73,6 +73,7 @@ mapAliases ({ email_validator = email-validator; # added 2022-06-22 fake_factory = throw "fake_factory has been removed because it is unused and deprecated by upstream since 2016."; # added 2022-05-30 faulthandler = throw "faulthandler is built into ${python.executable}"; # added 2021-07-12 + flask_login = flask-login; # added 2022-10-17 flask_sqlalchemy = flask-sqlalchemy; # added 2022-07-20 flask_testing = flask-testing; # added 2022-04-25 flask_wtf = flask-wtf; # added 2022-05-24 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5823cd11d4a..e35e0e98c79 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3441,7 +3441,7 @@ in { flask-limiter = callPackage ../development/python-modules/flask-limiter { }; - flask_login = callPackage ../development/python-modules/flask-login { }; + flask-login = callPackage ../development/python-modules/flask-login { }; flask_mail = callPackage ../development/python-modules/flask-mail { }; From 05250cc252880f47f27257733a6cb53a84b4b662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 17 Oct 2022 20:33:59 +0200 Subject: [PATCH 2/7] python310Packages.inscriptis: init at 2.3.1 --- .../python-modules/inscriptis/default.nix | 38 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 40 insertions(+) create mode 100644 pkgs/development/python-modules/inscriptis/default.nix diff --git a/pkgs/development/python-modules/inscriptis/default.nix b/pkgs/development/python-modules/inscriptis/default.nix new file mode 100644 index 00000000000..2fa224b3f21 --- /dev/null +++ b/pkgs/development/python-modules/inscriptis/default.nix @@ -0,0 +1,38 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, lxml +, pytestCheckHook +, requests +}: + +buildPythonPackage rec { + pname = "inscriptis"; + version = "2.3.1"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "weblyzard"; + repo = "inscriptis"; + rev = version; + sha256 = "sha256-an/FTbujN2VnTYa0wngM8ugV1LNHJWM32RVqIbaW0KY="; + }; + + propagatedBuildInputs = [ + lxml + requests + ]; + + checkInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ "inscriptis" ]; + + meta = with lib; { + description = "inscriptis - HTML to text converter"; + homepage = "https://github.com/weblyzard/inscriptis"; + license = licenses.asl20; + maintainers = with maintainers; [ SuperSandro2000 ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index e35e0e98c79..7c8d74853fe 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4614,6 +4614,8 @@ in { inquirer = callPackage ../development/python-modules/inquirer { }; + inscriptis = callPackage ../development/python-modules/inscriptis { }; + insegel = callPackage ../development/python-modules/insegel { }; installer = callPackage ../development/python-modules/installer { }; From a25188054f4f3309c6d893f0ba1b8215f8dc4197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 17 Oct 2022 23:12:42 +0200 Subject: [PATCH 3/7] python310Packages.jsonpath-ng: 1.5.2 -> 1.5.3 --- pkgs/development/python-modules/jsonpath-ng/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/jsonpath-ng/default.nix b/pkgs/development/python-modules/jsonpath-ng/default.nix index da3a03c2a3e..fce18a6da2c 100644 --- a/pkgs/development/python-modules/jsonpath-ng/default.nix +++ b/pkgs/development/python-modules/jsonpath-ng/default.nix @@ -9,13 +9,14 @@ buildPythonPackage rec { pname = "jsonpath-ng"; - version = "1.5.2"; + version = "1.5.3"; src = fetchFromGitHub { owner = "h2non"; repo = pname; - rev = "v${version}"; - sha256 = "1cxjwhx0nj85a3awnl7j6afnk07awzv45qfwxl5jqbbc9cxh5bd6"; + # missing tag https://github.com/h2non/jsonpath-ng/issues/114 + rev = "cce4a3d4063ac8af928795acc53beb27a2bfd101"; + sha256 = "sha256-+9iQHQs5TQhZFeIqMlsa3FFPfZEktAWy1lSdJU7kZrc="; }; propagatedBuildInputs = [ From bccd6352d3abc4ca6bfb87c00cc695e92a146181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 17 Oct 2022 23:22:07 +0200 Subject: [PATCH 4/7] python310Packages.selenium: 4.4.2 -> 4.5.0, add SuperSandro2000 as maintainer --- pkgs/development/python-modules/selenium/default.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/selenium/default.nix b/pkgs/development/python-modules/selenium/default.nix index 4c7eedf281d..2656f460ff6 100644 --- a/pkgs/development/python-modules/selenium/default.nix +++ b/pkgs/development/python-modules/selenium/default.nix @@ -12,14 +12,15 @@ buildPythonPackage rec { pname = "selenium"; - version = "4.4.2"; + version = "4.5.0"; disabled = pythonOlder "3.7"; src = fetchFromGitHub { owner = "SeleniumHQ"; repo = "selenium"; - rev = "refs/tags/selenium-${version}-python"; # check if there is a newer tag with -python suffix - hash = "sha256-sJJ3i4mnGp5fDgo64p6B2vRCqp/Wm99VoyRLyy4nBH8="; + # check if there is a newer tag with or without -python suffix + rev = "refs/tags/selenium-${version}"; + hash = "sha256-K90CQYTeX9GKpP0ahxLx2HO5HG0P6MN7jeWmHtfiOns="; }; postPatch = '' @@ -50,6 +51,6 @@ buildPythonPackage rec { description = "Bindings for Selenium WebDriver"; homepage = "https://selenium.dev/"; license = licenses.asl20; - maintainers = with maintainers; [ jraygauthier ]; + maintainers = with maintainers; [ jraygauthier SuperSandro2000 ]; }; } From 5d80a5129d31eb7580b012d8d98a3c6eb9089caf Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Wed, 4 May 2022 16:48:46 -0400 Subject: [PATCH 5/7] playwright: init at 1.27.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannik Rödel Co-authored-by: Sandro --- .../python-modules/playwright/default.nix | 221 ++++++++++++++++++ .../playwright/driver-location.patch | 47 ++++ .../python-modules/playwright/update.sh | 31 +++ pkgs/top-level/all-packages.nix | 2 + pkgs/top-level/python-packages.nix | 4 + 5 files changed, 305 insertions(+) create mode 100644 pkgs/development/python-modules/playwright/default.nix create mode 100644 pkgs/development/python-modules/playwright/driver-location.patch create mode 100755 pkgs/development/python-modules/playwright/update.sh diff --git a/pkgs/development/python-modules/playwright/default.nix b/pkgs/development/python-modules/playwright/default.nix new file mode 100644 index 00000000000..cea76281957 --- /dev/null +++ b/pkgs/development/python-modules/playwright/default.nix @@ -0,0 +1,221 @@ +{ lib +, stdenv +, buildPythonPackage +, chromium +, ffmpeg +, firefox +, git +, greenlet +, jq +, nodejs +, fetchFromGitHub +, fetchurl +, makeFontsConf +, makeWrapper +, pyee +, python +, pythonOlder +, runCommand +, setuptools-scm +, unzip +}: + +let + inherit (stdenv.hostPlatform) system; + throwSystem = throw "Unsupported system: ${system}"; + + driverVersion = "1.27.1"; + + driver = let + suffix = { + x86_64-linux = "linux"; + aarch64-linux = "linux-arm64"; + x86_64-darwin = "mac"; + aarch64-darwin = "mac-arm64"; + }.${system} or throwSystem; + filename = "playwright-${driverVersion}-${suffix}.zip"; + in stdenv.mkDerivation { + pname = "playwright-driver"; + version = driverVersion; + + src = fetchurl { + url = "https://playwright.azureedge.net/builds/driver/${filename}"; + sha256 = { + x86_64-linux = "0x71b4kb8hlyacixipgfbgjgrbmhckxpbmrs2xk8iis7n5kg7539"; + aarch64-linux = "125lih7g2gj91k7j196wy5a5746wyfr8idj3ng369yh5wl7lfcfv"; + x86_64-darwin = "0z2kww4iby1izkwn6z2ai94y87bkjvwak8awdmjm8sgg00pa9l1a"; + aarch64-darwin = "0qajh4ac5lr1sznb2c471r5c5g2r0dk2pyqz8vhvnbk36r524h1h"; + }.${system} or throwSystem; + }; + + sourceRoot = "."; + + nativeBuildInputs = [ unzip ]; + + postPatch = '' + # Use Nix's NodeJS instead of the bundled one. + substituteInPlace playwright.sh --replace '"$SCRIPT_PATH/node"' '"${nodejs}/bin/node"' + rm node + + # Hard-code the script path to $out directory to avoid a dependency on coreutils + substituteInPlace playwright.sh \ + --replace 'SCRIPT_PATH="$(cd "$(dirname "$0")" ; pwd -P)"' "SCRIPT_PATH=$out" + + patchShebangs playwright.sh package/bin/*.sh + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + mv playwright.sh $out/bin/playwright + mv package $out/ + + runHook postInstall + ''; + + passthru = { + inherit filename; + }; + }; + + browsers-mac = stdenv.mkDerivation { + pname = "playwright-browsers"; + version = driverVersion; + + src = runCommand "playwright-browsers-base" { + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = { + x86_64-darwin = "0z2kww4iby1izkwn6z2ai94y87bkjvwak8awdmjm8sgg00pa9l1a"; + }.${system} or throwSystem; + } '' + export PLAYWRIGHT_BROWSERS_PATH=$out + ${driver}/bin/playwright install + rm -r $out/.links + ''; + + installPhase = '' + mkdir $out + cp -r * $out/ + ''; + }; + + browsers-linux = { withFirefox ? true, withChromium ? true }: let + fontconfig = makeFontsConf { + fontDirectories = []; + }; + in runCommand ("playwright-browsers" + + lib.optionalString (withFirefox && !withChromium) "-firefox" + + lib.optionalString (!withFirefox && withChromium) "-chromium") + { + nativeBuildInputs = [ + makeWrapper + jq + ]; + } ('' + BROWSERS_JSON=${driver}/share/playwright-driver/package/browsers.json + '' + lib.optionalString withChromium '' + CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON) + mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux + + # See here for the Chrome options: + # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738 + makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \ + --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \ + --set FONTCONFIG_FILE ${fontconfig} + '' + lib.optionalString withFirefox '' + FIREFOX_REVISION=$(jq -r '.browsers[] | select(.name == "firefox").revision' $BROWSERS_JSON) + mkdir -p $out/firefox-$FIREFOX_REVISION + ln -s ${firefox}/bin/firefox $out/firefox-$FIREFOX_REVISION/firefox + '' + '' + FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON) + mkdir -p $out/ffmpeg-$FFMPEG_REVISION + ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux + ''); +in +buildPythonPackage rec { + pname = "playwright"; + version = "1.27.1"; + format = "setuptools"; + disabled = pythonOlder "3.7"; + + src = fetchFromGitHub { + owner = "microsoft"; + repo = "playwright-python"; + rev = "v${version}"; + sha256 = "sha256-cI/4GdkmTikoP9O0Skh/0jCxxRypRua0231iKcxtBcY="; + }; + + patches = [ + # This patches two things: + # - The driver location, which is now a static package in the Nix store. + # - The setup script, which would try to download the driver package from + # a CDN and patch wheels so that they include it. We don't want this + # we have our own driver build. + ./driver-location.patch + ]; + + postPatch = '' + # if setuptools_scm is not listing files via git almost all python files are excluded + export HOME=$(mktemp -d) + git init . + git add -A . + git config --global user.email "nixpkgs" + git config --global user.name "nixpkgs" + git commit -m "workaround setuptools-scm" + + substituteInPlace setup.py \ + --replace "greenlet==1.1.3" "greenlet>=1.1.3" \ + --replace "pyee==8.1.0" "pyee>=8.1.0" \ + --replace "setuptools-scm==7.0.5" "setuptools-scm>=7.0.5" \ + --replace "wheel==0.37.1" "wheel>=0.37.1" + + # Skip trying to download and extract the driver. + # This is done manually in postInstall instead. + substituteInPlace setup.py \ + --replace "self._download_and_extract_local_driver(base_wheel_bundles)" "" + + # Set the correct driver path with the help of a patch in patches + substituteInPlace playwright/_impl/_driver.py \ + --replace "@driver@" "${driver}/bin/playwright" + ''; + + + nativeBuildInputs = [ git setuptools-scm ]; + + propagatedBuildInputs = [ + greenlet + pyee + ]; + + postInstall = '' + ln -s ${driver} $out/${python.sitePackages}/playwright/driver + ''; + + # Skip tests because they require network access. + doCheck = false; + + pythonImportsCheck = [ + "playwright" + ]; + + passthru = { + inherit driver; + browsers = { + x86_64-linux = browsers-linux { }; + aarch64-linux = browsers-linux { }; + x86_64-darwin = browsers-mac; + aarch64-darwin = browsers-mac; + }.${system} or throwSystem; + browsers-chromium = browsers-linux { withFirefox = false; }; + browsers-firefox = browsers-linux { withChromium = false; }; + }; + + meta = with lib; { + description = "Python version of the Playwright testing and automation library"; + homepage = "https://github.com/microsoft/playwright-python"; + license = licenses.asl20; + maintainers = with maintainers; [ techknowlogick yrd SuperSandro2000 ]; + }; +} diff --git a/pkgs/development/python-modules/playwright/driver-location.patch b/pkgs/development/python-modules/playwright/driver-location.patch new file mode 100644 index 00000000000..a7f79840c65 --- /dev/null +++ b/pkgs/development/python-modules/playwright/driver-location.patch @@ -0,0 +1,47 @@ +diff --git a/playwright/_impl/_driver.py b/playwright/_impl/_driver.py +index f3b911f..d00e509 100644 +--- a/playwright/_impl/_driver.py ++++ b/playwright/_impl/_driver.py +@@ -23,11 +23,7 @@ from playwright._repo_version import version + + + def compute_driver_executable() -> Path: +- package_path = Path(inspect.getfile(playwright)).parent +- platform = sys.platform +- if platform == "win32": +- return package_path / "driver" / "playwright.cmd" +- return package_path / "driver" / "playwright.sh" ++ return Path("@driver@") + + + if sys.version_info.major == 3 and sys.version_info.minor == 7: +diff --git a/setup.py b/setup.py +index 3487a6a..05112c2 100644 +--- a/setup.py ++++ b/setup.py +@@ -141,25 +141,8 @@ class PlaywrightBDistWheelCommand(BDistWheelCommand): + base_wheel_location: str = glob.glob(os.path.join(self.dist_dir, "*.whl"))[0] + without_platform = base_wheel_location[:-7] + for wheel_bundle in wheels: +- download_driver(wheel_bundle["zip_name"]) +- zip_file = ( +- f"driver/playwright-{driver_version}-{wheel_bundle['zip_name']}.zip" +- ) +- with zipfile.ZipFile(zip_file, "r") as zip: +- extractall(zip, f"driver/{wheel_bundle['zip_name']}") + wheel_location = without_platform + wheel_bundle["wheel"] + shutil.copy(base_wheel_location, wheel_location) +- with zipfile.ZipFile(wheel_location, "a") as zip: +- driver_root = os.path.abspath(f"driver/{wheel_bundle['zip_name']}") +- for dir_path, _, files in os.walk(driver_root): +- for file in files: +- from_path = os.path.join(dir_path, file) +- to_path = os.path.relpath(from_path, driver_root) +- zip.write(from_path, f"playwright/driver/{to_path}") +- zip.writestr( +- "playwright/driver/README.md", +- f"{wheel_bundle['wheel']} driver package", +- ) + os.remove(base_wheel_location) + if InWheel: + for whlfile in glob.glob(os.path.join(self.dist_dir, "*.whl")): diff --git a/pkgs/development/python-modules/playwright/update.sh b/pkgs/development/python-modules/playwright/update.sh new file mode 100755 index 00000000000..51dd8723c85 --- /dev/null +++ b/pkgs/development/python-modules/playwright/update.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl gnused nix-prefetch common-updater-scripts +set -euo pipefail + +root="$(dirname "$(readlink -f "$0")")" + +version=$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s https://api.github.com/repos/microsoft/playwright-python/releases/latest | jq -r '.tag_name | sub("^v"; "")') + +# Most of the time, this should be the latest stable release of the Node-based +# Playwright version, but that isn't a guarantee, so this needs to be specified +# as well: +setup_py_url="https://github.com/microsoft/playwright-python/raw/v${version}/setup.py" +driver_version=$(curl -Ls "$setup_py_url" | grep '^driver_version =' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') + +fetch_driver_arch() { + nix-prefetch-url "https://playwright.azureedge.net/builds/driver/playwright-${version}-${1}.zip" +} + +replace_sha() { + sed -i "s|$1 = \".\{44,52\}\"|$1 = \"$2\"|" "$root/default.nix" +} + +# Replace SHAs for the driver downloads +replace_sha "x86_64-linux" "$(fetch_driver_arch "linux")" +replace_sha "x86_64-darwin" "$(fetch_driver_arch "mac")" +replace_sha "aarch64-linux" "$(fetch_driver_arch "linux-arm64")" +replace_sha "aarch64-darwin" "$(fetch_driver_arch "mac-arm64")" + +# Update the version stamps +sed -i "s/driverVersion = \"[^\$]*\"/driverVersion = \"$driver_version\"/" "$root/default.nix" +update-source-version playwright "$version" --rev="v$version" diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4c4a47223a0..5710817b86d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10254,6 +10254,8 @@ with pkgs; playbar2 = libsForQt5.callPackage ../applications/audio/playbar2 { }; + playwright = with python3Packages; toPythonApplication playwright; + please = callPackage ../tools/security/please { }; plecost = callPackage ../tools/security/plecost { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 7c8d74853fe..58af505febd 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -7036,6 +7036,10 @@ in { pkuseg = callPackage ../development/python-modules/pkuseg { }; + playwright = callPackage ../development/python-modules/playwright { + inherit (pkgs) jq; + }; + pmsensor = callPackage ../development/python-modules/pmsensor { }; ppdeep = callPackage ../development/python-modules/ppdeep { }; From eba04bfbe8637465797b77da477e09c6c78064a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 17 Oct 2022 20:14:51 +0200 Subject: [PATCH 6/7] changedetection-io: init at 0.39.20.4 change --- .../web-apps/changedetection-io/default.nix | 92 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 94 insertions(+) create mode 100644 pkgs/servers/web-apps/changedetection-io/default.nix diff --git a/pkgs/servers/web-apps/changedetection-io/default.nix b/pkgs/servers/web-apps/changedetection-io/default.nix new file mode 100644 index 00000000000..c86357637e4 --- /dev/null +++ b/pkgs/servers/web-apps/changedetection-io/default.nix @@ -0,0 +1,92 @@ +{ lib +, fetchFromGitHub +, python3 +}: +let + py = python3.override { + packageOverrides = final: prev: { + flask = prev.flask.overridePythonAttrs (old: rec { + version = "2.1.3"; + src = old.src.override { + inherit version; + sha256 = "sha256-FZcuUBffBXXD1sCQuhaLbbkCWeYgrI1+qBOjlrrVtss="; + }; + }); + flask-restful = prev.flask-restful.overridePythonAttrs (old: rec { + disabledTests = old.disabledTests or [ ] ++ [ + # fails because of flask or werkzeug downgrade + "test_redirect" + ]; + }); + werkzeug = prev.werkzeug.overridePythonAttrs (old: rec { + version = "2.0.3"; + src = old.src.override { + inherit version; + sha256 = "sha256-uGP4/wV8UiFktgZ8niiwQRYbS+W6TQ2s7qpQoWOCLTw="; + }; + }); + }; + }; +in +py.pkgs.buildPythonApplication rec { + pname = "changedetection-io"; + version = "0.39.20.3"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "dgtlmoon"; + repo = "changedetection.io"; + rev = version; + sha256 = "sha256-0Sv/1YoZuSnslQgMOu+uHTxb9QewXPC0tLAvzJA4Aa8="; + }; + + postPatch = '' + substituteInPlace requirements.txt \ + --replace "bs4" "beautifulsoup4" \ + --replace "cryptography ~= 3.4" "cryptography" \ + --replace "selenium ~= 4.1.0" "selenium" + ''; + + propagatedBuildInputs = with py.pkgs; [ + flask + flask-wtf + eventlet + validators + timeago + inscriptis + feedgen + flask-login + flask-restful + pytz + brotli + requests + urllib3 + chardet + wtforms + jsonpath-ng + jq + apprise + paho-mqtt + cryptography + beautifulsoup4 + lxml + selenium + werkzeug + playwright + ] ++ requests.optional-dependencies.socks; + + # tests can currently not be run in one pytest invocation and without docker + doCheck = false; + + checkInputs = with py.pkgs; [ + pytest-flask + pytestCheckHook + ]; + + meta = with lib; { + homepage = "https://github.com/dgtlmoon/changedetection.io"; + description = "Simplest self-hosted free open source website change detection tracking, monitoring and notification service"; + license = licenses.asl20; + maintainers = with maintainers; [ SuperSandro2000 ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5710817b86d..068ebf4145f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3548,6 +3548,8 @@ with pkgs; cambalache = callPackage ../development/tools/cambalache { }; + changedetection-io = callPackage ../servers/web-apps/changedetection-io { }; + clipster = callPackage ../tools/misc/clipster { }; clockify = callPackage ../applications/office/clockify { From c9aab9ba9737c0b4601d816792a31cf04a75d651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 18 Oct 2022 03:04:25 +0200 Subject: [PATCH 7/7] nixos/changedetection-io: init --- nixos/modules/module-list.nix | 1 + .../services/web-apps/changedetection-io.nix | 218 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 nixos/modules/services/web-apps/changedetection-io.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c400fadef3a..47f254136b0 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1069,6 +1069,7 @@ ./services/web-apps/calibre-web.nix ./services/web-apps/code-server.nix ./services/web-apps/baget.nix + ./services/web-apps/changedetection-io.nix ./services/web-apps/convos.nix ./services/web-apps/dex.nix ./services/web-apps/discourse.nix diff --git a/nixos/modules/services/web-apps/changedetection-io.nix b/nixos/modules/services/web-apps/changedetection-io.nix new file mode 100644 index 00000000000..83d8b32c0c8 --- /dev/null +++ b/nixos/modules/services/web-apps/changedetection-io.nix @@ -0,0 +1,218 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.changedetection-io; +in +{ + options.services.changedetection-io = { + enable = mkEnableOption (lib.mdDoc "changedetection-io"); + + user = mkOption { + default = "changedetection-io"; + type = types.str; + description = lib.mdDoc '' + User account under which changedetection-io runs. + ''; + }; + + group = mkOption { + default = "changedetection-io"; + type = types.str; + description = lib.mdDoc '' + Group account under which changedetection-io runs. + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = "localhost"; + description = lib.mdDoc "Address the server will listen on."; + }; + + port = mkOption { + type = types.port; + default = 5000; + description = lib.mdDoc "Port the server will listen on."; + }; + + datastorePath = mkOption { + type = types.str; + default = "/var/lib/changedetection-io"; + description = lib.mdDoc '' + The directory used to store all data for changedetection-io. + ''; + }; + + baseURL = mkOption { + type = types.nullOr types.str; + default = null; + example = "https://changedetection-io.example"; + description = lib.mdDoc '' + The base url used in notifications and `{base_url}` token. + ''; + }; + + behindProxy = mkOption { + type = types.bool; + default = false; + description = lib.mdDoc '' + Enable this option when changedetection-io runs behind a reverse proxy, so that it trusts X-* headers. + It is recommend to run changedetection-io behind a TLS reverse proxy. + ''; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + example = "/run/secrets/changedetection-io.env"; + description = lib.mdDoc '' + Securely pass environment variabels to changedetection-io. + + This can be used to set for example a frontend password reproducible via `SALTED_PASS` + which convinetly also deactivates nags about the hosted version. + `SALTED_PASS` should be 64 characters long while the first 32 are the salt and the second the frontend password. + It can easily be retrieved from the settings file when first set via the frontend with the following command: + ``jq -r .settings.application.password /var/lib/changedetection-io/url-watches.json`` + ''; + }; + + webDriverSupport = mkOption { + type = types.bool; + default = false; + description = lib.mdDoc '' + Enable support for fetching web pages using WebDriver and Chromium. + This starts a headless chromium controlled by puppeteer in an oci container. + + ::: {.note} + Playwright can currently leak memory. + See https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher#playwright-memory-leak + ::: + ''; + }; + + playwrightSupport = mkOption { + type = types.bool; + default = false; + description = lib.mdDoc '' + Enable support for fetching web pages using playwright and Chromium. + This starts a headless Chromium controlled by puppeteer in an oci container. + + ::: {.note} + Playwright can currently leak memory. + See https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher#playwright-memory-leak + ::: + ''; + }; + + chromePort = mkOption { + type = types.port; + default = 4444; + description = lib.mdDoc '' + A free port on which webDriverSupport or playwrightSupport listen on localhost. + ''; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = !((cfg.webDriverSupport == true) && (cfg.playwrightSupport == true)); + message = "'services.changedetection-io.webDriverSupport' and 'services.changedetion-io.playwrightSupport' cannot be used together."; + } + ]; + + systemd = let + defaultStateDir = cfg.datastorePath == "/var/lib/changedetection-io"; + in { + services.changedetection-io = { + wantedBy = [ "mutli-user.target" ]; + after = [ "network.target" ]; + preStart = '' + mkdir -p ${cfg.datastorePath} + ''; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + StateDirectory = mkIf defaultStateDir "changedetion-io"; + StateDirectoryMode = mkIf defaultStateDir "0750"; + WorkingDirectory = cfg.datastorePath; + Environment = lib.optional (cfg.baseURL != null) "BASE_URL=${cfg.baseURL}" + ++ lib.optional cfg.behindProxy "USE_X_SETTINGS=1" + ++ lib.optional cfg.webDriverSupport "WEBDRIVER_URL=http://127.0.0.1:${toString cfg.chromePort}/wd/hub" + ++ lib.optional cfg.playwrightSupport "PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:${toString cfg.chromePort}/?stealth=1&--disable-web-security=true"; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + ExecStart = '' + ${pkgs.changedetection-io}/bin/changedetection.py \ + -h ${cfg.listenAddress} -p ${toString cfg.port} -d ${cfg.datastorePath} + ''; + ProtectHome = true; + ProtectSystem = true; + Restart = "on-failure"; + }; + }; + tmpfiles.rules = mkIf defaultStateDir [ + "d ${cfg.datastorePath} 0750 ${cfg.user} ${cfg.group} - -" + ]; + }; + + users = { + users = optionalAttrs (cfg.user == "changedetection-io") { + "changedetection-io" = { + isSystemUser = true; + group = "changedetection-io"; + }; + }; + + groups = optionalAttrs (cfg.group == "changedetection-io") { + "changedetection-io" = { }; + }; + }; + + virtualisation = { + oci-containers.containers = lib.mkMerge [ + (mkIf cfg.webDriverSupport { + changedetection-io-webdriver = { + image = "selenium/standalone-chrome"; + environment = { + VNC_NO_PASSWORD = "1"; + SCREEN_WIDTH = "1920"; + SCREEN_HEIGHT = "1080"; + SCREEN_DEPTH = "24"; + }; + ports = [ + "127.0.0.1:${toString cfg.chromePort}:4444" + ]; + volumes = [ + "/dev/shm:/dev/shm" + ]; + extraOptions = [ "--network=bridge" ]; + }; + }) + + (mkIf cfg.playwrightSupport { + changedetection-io-playwright = { + image = "browserless/chrome"; + environment = { + SCREEN_WIDTH = "1920"; + SCREEN_HEIGHT = "1024"; + SCREEN_DEPTH = "16"; + ENABLE_DEBUGGER = "false"; + PREBOOT_CHROME = "true"; + CONNECTION_TIMEOUT = "300000"; + MAX_CONCURRENT_SESSIONS = "10"; + CHROME_REFRESH_TIME = "600000"; + DEFAULT_BLOCK_ADS = "true"; + DEFAULT_STEALTH = "true"; + }; + ports = [ + "127.0.0.1:${toString cfg.chromePort}:3000" + ]; + extraOptions = [ "--network=bridge" ]; + }; + }) + ]; + }; + }; +}