From f56443508415e32bb3652122fc3d97e36237281d Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Fri, 2 Jun 2023 16:00:58 +0200 Subject: [PATCH] Fix asset generation and tidy up a little --- flake.nix | 81 +++--------------- nginx-erpnext-conf.nix | 9 +- node/erpnext-apps-sites.nix | 68 +++++++++++++++ node/esbuild-js-fix-css-build.patch | 18 ---- node/frappe-assets.nix | 112 ------------------------ node/frappe-modules.nix | 21 ----- overlay.nix | 5 ++ python-overlay.nix | 127 ++++++++++++++++++++++++++++ python.nix | 127 ---------------------------- scripts/run-erpnext.nix | 58 +++++++++++++ 10 files changed, 275 insertions(+), 351 deletions(-) create mode 100644 node/erpnext-apps-sites.nix delete mode 100644 node/esbuild-js-fix-css-build.patch delete mode 100644 node/frappe-assets.nix delete mode 100644 node/frappe-modules.nix create mode 100644 overlay.nix create mode 100644 python-overlay.nix delete mode 100644 python.nix create mode 100644 scripts/run-erpnext.nix diff --git a/flake.nix b/flake.nix index c29901a..37bc0df 100644 --- a/flake.nix +++ b/flake.nix @@ -6,78 +6,18 @@ inputs.pip2nix = { url = "github:nix-community/pip2nix"; flake = false; - # inputs.nixpkgs.follows = "nixpkgs"; # inputs.flake-utils.follows = "flake-utils"; }; outputs = {nixpkgs, flake-utils, pip2nix, ...}: flake-utils.lib.eachDefaultSystem (system: let - pkgs = import nixpkgs { inherit system; - config.allowBroken = true; - }; - #attrsets = pkgs.lib.attrsets; - #overrideFiles = (attrsets.filterAttrs (f: type: type == "regular") (builtins.readDir ./python)); - #overridePackageNames = attrsets.mapAttrsToList (f: _: builtins.replaceStrings [".nix"] [""] f) overrideFiles; - py = pkgs.callPackage ./python.nix { inherit pkgs; }; - # packageOverrides = pself: psuper: - # attrsets.genAttrs overridePackageNames (o: pself.callPackage ./python/${o}.nix {}); - #}); - #newversion = python3.pkgs.newversion.overrideAttrs(old: { - # propagatedBuildInputs = old.propagatedBuildInputs ++ [python3.pkgs.setuptools]; - #}); - assets = pkgs.callPackage ./node/frappe-assets.nix {}; - # Source: https://github.com/frappe/frappe_docker/blob/main/resources/nginx-template.conf - nginx-conf = pkgs.callPackage ./nginx-erpnext-conf.nix {inherit pkgs;}; - penv = py.buildEnv.override { - extraLibs = [ py.pkgs.frappe py.pkgs.erpnext ]; + pkgs = import nixpkgs { + inherit system; + overlays = [ + (import ./python-overlay.nix) + (import ./overlay.nix) + ]; }; - erpnextSrc = pkgs.callPackage ./srcs/erpnext.nix {}; - frappeModules = pkgs.callPackage ./node/frappe-modules.nix {}; - runErpNext = pkgs.writeShellScriptBin "runErpNext" '' - export PYTHON_PATH=${penv}/${py.sitePackages} - # The upstream installer bench CLI wants mysql in its PATH - export PATH=${pkgs.mariadb-client}/bin:''$PATH - - hostname=localhost - tmp=/tmp/erpnext - - mkdir -p $tmp/apps $tmp/sites $tmp/config/pids $tmp/logs/nginx $tmp/env/bin - - for f in ${assets}/share/sites/*; do - ln -s "$f" "$tmp/sites/$(basename $f)" - done - - cat >$tmp/sites/common_site_config.json < $tmp/sites/apps.txt - - ln -s ${erpnextSrc} $tmp/apps/erpnext - ln -s ${frappeModules}/libexec/frappe-framework/deps/frappe-framework $tmp/apps/frappe - - ln -s ${penv} $tmp/env - ln -sf ${nginx-conf} $tmp/nginx-erpnext.conf - - cd $tmp - # Upstream initializes the DB with this command - ${py.pkgs.bench}/bin/bench new-site localhost --mariadb-root-password password --admin-password admin - ${py.pkgs.bench}/bin/bench --site localhost install-app erpnext - - echo "Workdir: $tmp" - ${pkgs.nodejs}/bin/node $tmp/apps/frappe/socketio.js & - ${penv}/bin/gunicorn --chdir="$tmp/sites" --bind=0.0.0.0:9090 --threads=4 --workers=2 --worker-class=gthread --worker-tmp-dir=/dev/shm --timeout=120 --preload frappe.app:application - ''; in rec { packages = { devEnv = pkgs.buildEnv { @@ -86,11 +26,12 @@ pkgs.dasel ]; }; - inherit pkgs runErpNext assets; + inherit pkgs; + run-erpnext = pkgs.run-erpnext; pip2nix = import "${pip2nix}/default.nix" { inherit pkgs; pythonPackages = "python310Packages"; }; - erpnext = py.pkgs.erpnext; - bench = py.pkgs.bench; - pythonPkgs = py.pkgs; + erpnext = pkgs.python3.pkgs.erpnext; + bench = pkgs.python3.pkgs.bench; + pythonPkgs = pkgs.python3.pkgs; }; }); } diff --git a/nginx-erpnext-conf.nix b/nginx-erpnext-conf.nix index 3990883..829310e 100644 --- a/nginx-erpnext-conf.nix +++ b/nginx-erpnext-conf.nix @@ -1,4 +1,7 @@ -{ pkgs }: +# From https://github.com/frappe/frappe_docker/blob/main/resources/nginx-template.conf +{ writeText +, nginx +}: let backend = "127.0.0.1:9090"; socketio = "127.0.0.1:3000"; @@ -9,7 +12,7 @@ let client_max_body_size = "50m"; proxy_read_timeout = "120"; in -pkgs.writeText "erpnext.conf" '' +writeText "erpnext.conf" '' user nginx; worker_processes auto; @@ -23,7 +26,7 @@ events { http { - include ${pkgs.nginx}/conf/mime.types; + include ${nginx}/conf/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' diff --git a/node/erpnext-apps-sites.nix b/node/erpnext-apps-sites.nix new file mode 100644 index 0000000..5e243a8 --- /dev/null +++ b/node/erpnext-apps-sites.nix @@ -0,0 +1,68 @@ +{ fetchFromGitHub +, fetchYarnDeps +, runCommand +, nodejs +, yarn +, path +, nodePackages +}: +let + erpnextSrc = import ../srcs/erpnext.nix {inherit fetchFromGitHub; }; + erpnextOfflineCache = fetchYarnDeps { + yarnLock = "${erpnextSrc}/yarn.lock"; + sha256 = "sha256-Vho4BSbxcsVYExLvUaeoc3xIpbXoCUP/4jw4RwGnWGY="; + }; + + frappeSrc = import ../srcs/frappe.nix {inherit fetchFromGitHub; }; + frappeOfflineCache = fetchYarnDeps { + yarnLock = "${frappeSrc}/yarn.lock"; + sha256 = "sha256-PBdMUz9gJIoQaqQYbdk+xnd8CyZPmdeyz/9WznCb4Ss="; + }; + + # Copied from nixpkgs:pkgs/development/tools/yarn2nix-moretea/yarn2nix/default.nix + fixup_yarn_lock = runCommand "fixup_yarn_lock" {buildInputs = [ nodejs ];} '' + mkdir -p $out/lib + mkdir -p $out/bin + + cp ${path}/pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js $out/lib/urlToName.js + cp ${path}/pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_yarn_lock.js $out/bin/fixup_yarn_lock + + patchShebangs $out + ''; + + mkApp = name: src: yarnOfflineCache: runCommand "${name}-app" { + buildInputs = [fixup_yarn_lock yarn nodePackages.node-gyp-build]; + } '' + mkdir -p $out/share/apps + cp -r ${src} $out/share/apps/${name} + chmod -R +w $out/share/apps/${name} + + export HOME=$(mktemp -d) + yarn config --offline set yarn-offline-mirror ${yarnOfflineCache} + + cd $out/share/apps/${name} + fixup_yarn_lock yarn.lock + yarn --offline --ignore-scripts install + ''; + + frappeApp = mkApp "frappe" frappeSrc frappeOfflineCache; + erpnextApp = mkApp "erpnext" erpnextSrc erpnextOfflineCache; + + assets = runCommand "frappe-erpnext-assets" {buildInputs = [yarn]; } '' + mkdir -p $out/share/sites $out/share/apps + + # Cannot symlink because the code which traverses path to find sites + # directory gets confused. + cp -r ${frappeApp}/share/apps/frappe $out/share/apps/frappe + cp -r ${erpnextApp}/share/apps/erpnext $out/share/apps/erpnext + + cat > $out/share/sites/apps.txt < sites/apps.txt - - cp -r ${erpnextSrc} apps/erpnext - cp -r $src apps/frappe - - chmod u+rw apps/erpnext - chmod u+rw apps/frappe - chmod -R u+rw sites/assets/erpnext - chmod -R u+rw sites/assets/frappe - - cp -r ${erpnext-modules}/node_modules apps/erpnext/node_modules - cp -r $node_modules apps/frappe/node_modules - - yarn --offline production - - # Frappe expects node_modules linked to this directory - ln -s ${erpnext-modules}/node_modules sites/assets/erpnext/node_modules - ln -s $node_modules sites/assets/frappe/node_modules - - # Clean up - rm sites/apps.txt - - mv sites deps/ - - runHook postBuild - ''; - - # Do not attempt generating a tarball - doDist = false; - }; - frappe-assets = stdenv.mkDerivation { - pname = "erpnext-assets"; - version = "14.24.3"; - phases = [ "installPhase" ]; - installPhase = '' - mkdir -p $out/share - cp -r "${frappe-modules}/libexec/frappe-framework/deps/sites/" $out/share/sites - ''; - }; -in frappe-assets diff --git a/node/frappe-modules.nix b/node/frappe-modules.nix deleted file mode 100644 index 5003195..0000000 --- a/node/frappe-modules.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ mkYarnPackage -, fetchFromGitHub -, fetchYarnDeps -}: -mkYarnPackage { - pname = "frappe-modules"; - version = "14.36.1"; - - src = import ../srcs/frappe.nix {inherit fetchFromGitHub; }; - - packageJSON = ../frappe-package.json; - yarnFlags = [ "--production" ]; - - offlineCache = fetchYarnDeps { - yarnLock = "$src/yarn.lock"; - hash = "sha256-PBdMUz9gJIoQaqQYbdk+xnd8CyZPmdeyz/9WznCb4Ss="; - }; - - # Do not attempt generating a tarball - doDist = false; -} diff --git a/overlay.nix b/overlay.nix new file mode 100644 index 0000000..8925d74 --- /dev/null +++ b/overlay.nix @@ -0,0 +1,5 @@ +self: super: { + run-erpnext = self.callPackage ./scripts/run-erpnext.nix {}; + erpnext-apps-sites = self.callPackage ./node/erpnext-apps-sites.nix {}; + erpnext-nginx-conf = self.callPackage ./nginx-erpnext-conf.nix {}; +} diff --git a/python-overlay.nix b/python-overlay.nix new file mode 100644 index 0000000..da76514 --- /dev/null +++ b/python-overlay.nix @@ -0,0 +1,127 @@ +self: super: { + python3 = super.python3.override { + packageOverrides = pyself: pysuper: { + bench = pyself.callPackage ./python/bench.nix {}; + erpnext = pyself.callPackage ./python/erpnext.nix {}; + frappe = pyself.callPackage ./python/frappe.nix {}; + + email-reply-parser = pyself.callPackage ./python/email-reply-parser.nix {}; + git-url-parse = pyself.callPackage ./python/git-url-parse.nix {}; + gocardless-pro = pyself.callPackage ./python/gocardless-pro.nix {}; + honcho = pyself.callPackage ./python/honcho.nix {}; + jsonobject = pyself.callPackage ./python/jsonobject.nix {}; + maxminddb-geolite2 = pyself.callPackage ./python/maxminddb-geolite2.nix {}; + posthog = pyself.callPackage ./python/posthog.nix {}; + psycopg2-binary = pyself.callPackage ./python/psycopg2-binary.nix {}; + pypdf2 = pyself.callPackage ./python/pypdf2.nix {}; + pypika = pyself.callPackage ./python/pypika.nix {}; + python-youtube = pyself.callPackage ./python/python-youtube.nix {}; + rauth = pyself.callPackage ./python/rauth.nix {}; + redisearch = pyself.callPackage ./python/redisearch.nix {}; + rejson = pyself.callPackage ./python/rejson.nix {}; + taxjar = pyself.callPackage ./python/taxjar.nix {}; + traceback-with-variables = pyself.callPackage ./python/traceback-with-variables.nix {}; + + barcodenumber = pysuper.barcodenumber.overridePythonAttrs (oldAttrs: (rec { + version = "0.5.0"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-VZfHLwSF9aDoy5L1x4O2mu8/f2ijYKgyjCrQ1KKY5Ho="; + }; + })); + bleach = pysuper.bleach.overridePythonAttrs (oldAttrs: (rec { + version = "3.3.1"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-MGSDpal5VHQWCtV/zj3dG1BVHpge7Y4VpYLTTO8oqvo="; + }; + })); + phonenumbers = pysuper.phonenumbers.overridePythonAttrs (oldAttrs: (rec { + version = "8.12.40"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-APKVWkVrRY+barDSQykEnD5zWMRN/Bl5/kkIztQPHrg="; + }; + })); + plaid-python = pysuper.plaid-python.overridePythonAttrs (oldAttrs: (rec { + version = "7.2.1"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-ryrTJug3fIyG2XGE9gwL5BzXH1B1IB39szMcyF1N5RM="; + }; + propagatedBuildInputs = [ pysuper.requests ]; + checkInputs = [ pysuper.pytest ]; + # Integration tests require API keys and internet access + checkPhase = "py.test -rxs ./tests/unit"; + })); + pycountry = pysuper.pycountry.overridePythonAttrs (oldAttrs: (rec { + version = "20.7.3"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-gQhKU9NFQ0TAKS3uvCD80KFIjBNtSQAxLL1GXPVSy0I="; + }; + })); + pymysql = pysuper.pymysql.overridePythonAttrs (oldAttrs: (rec { + version = "1.0.3"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-PdqUPvNpQGinXWnQcXVdvsrO4a35ofxbIGgw0rZ9Jeg="; + }; + format = "pyproject"; + buildInputs = [pysuper.setuptools]; + })); + pypng = pysuper.pypng.overridePythonAttrs (oldAttrs: (rec { + version = "0.20220715.0"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-c5xDO6lvB4MV3lTA25da7lN8vD4dCuTtmqsMoeQn4sE="; + }; + })); + pytz = pysuper.pytz.overridePythonAttrs (oldAttrs: (rec { + version = "2022.1"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-HnYOL+aoFjvAs9mhnE+ENCr6Cir/6/qoSwG5eKAuyqc="; + }; + })); + redis = pysuper.redis.overridePythonAttrs (oldAttrs: (rec { + version = "3.5.3"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-Dn4M/KhmDeqLfVzYxPbF4p4R8xFYwLCukaOX8A5aBaI="; + }; + pythonImportsCheck = []; + # tests require a running redis + doCheck = false; + })); + tweepy = pysuper.tweepy.overridePythonAttrs (oldAttrs: (rec { + version = "3.10.0"; + src = pysuper.fetchPypi { + inherit version; + inherit (oldAttrs) pname; + sha256 = "sha256-duaVS4BspHDdqHf1fbh5L/8GoL66DtQ+/DgFdx458Go="; + }; + doCheck = false; + pythonImportsCheck = []; + })); + unidecode = pysuper.unidecode.overridePythonAttrs (oldAttrs: (rec { + version = "1.2.0"; + src = pysuper.fetchPypi { + inherit version; + pname = "Unidecode"; + sha256 = "sha256-jXOpfTh6lWkiNE9rdCQ8LGdxWUZZd4dEstvarY9rcn0="; + }; + })); + }; + }; +} + diff --git a/python.nix b/python.nix deleted file mode 100644 index d6b6fc7..0000000 --- a/python.nix +++ /dev/null @@ -1,127 +0,0 @@ -{ pkgs }: - -pkgs.python3.override { - packageOverrides = self: super: { - bench = self.callPackage ./python/bench.nix {}; - erpnext = self.callPackage ./python/erpnext.nix {}; - frappe = self.callPackage ./python/frappe.nix {}; - - email-reply-parser = self.callPackage ./python/email-reply-parser.nix {}; - git-url-parse = self.callPackage ./python/git-url-parse.nix {}; - gocardless-pro = self.callPackage ./python/gocardless-pro.nix {}; - honcho = self.callPackage ./python/honcho.nix {}; - jsonobject = self.callPackage ./python/jsonobject.nix {}; - maxminddb-geolite2 = self.callPackage ./python/maxminddb-geolite2.nix {}; - posthog = self.callPackage ./python/posthog.nix {}; - psycopg2-binary = self.callPackage ./python/psycopg2-binary.nix {}; - pypdf2 = self.callPackage ./python/pypdf2.nix {}; - pypika = self.callPackage ./python/pypika.nix {}; - python-youtube = self.callPackage ./python/python-youtube.nix {}; - rauth = self.callPackage ./python/rauth.nix {}; - redisearch = self.callPackage ./python/redisearch.nix {}; - rejson = self.callPackage ./python/rejson.nix {}; - taxjar = self.callPackage ./python/taxjar.nix {}; - traceback-with-variables = self.callPackage ./python/traceback-with-variables.nix {}; - - barcodenumber = super.barcodenumber.overridePythonAttrs (oldAttrs: (rec { - version = "0.5.0"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-VZfHLwSF9aDoy5L1x4O2mu8/f2ijYKgyjCrQ1KKY5Ho="; - }; - })); - bleach = super.bleach.overridePythonAttrs (oldAttrs: (rec { - version = "3.3.1"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-MGSDpal5VHQWCtV/zj3dG1BVHpge7Y4VpYLTTO8oqvo="; - }; - })); - phonenumbers = super.phonenumbers.overridePythonAttrs (oldAttrs: (rec { - version = "8.12.40"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-APKVWkVrRY+barDSQykEnD5zWMRN/Bl5/kkIztQPHrg="; - }; - })); - plaid-python = super.plaid-python.overridePythonAttrs (oldAttrs: (rec { - version = "7.2.1"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-ryrTJug3fIyG2XGE9gwL5BzXH1B1IB39szMcyF1N5RM="; - }; - propagatedBuildInputs = [ super.requests ]; - checkInputs = [ super.pytest ]; - # Integration tests require API keys and internet access - checkPhase = "py.test -rxs ./tests/unit"; - })); - pycountry = super.pycountry.overridePythonAttrs (oldAttrs: (rec { - version = "20.7.3"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-gQhKU9NFQ0TAKS3uvCD80KFIjBNtSQAxLL1GXPVSy0I="; - }; - })); - pymysql = super.pymysql.overridePythonAttrs (oldAttrs: (rec { - version = "1.0.3"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-PdqUPvNpQGinXWnQcXVdvsrO4a35ofxbIGgw0rZ9Jeg="; - }; - format = "pyproject"; - buildInputs = [super.setuptools]; - })); - pypng = super.pypng.overridePythonAttrs (oldAttrs: (rec { - version = "0.20220715.0"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-c5xDO6lvB4MV3lTA25da7lN8vD4dCuTtmqsMoeQn4sE="; - }; - })); - pytz = super.pytz.overridePythonAttrs (oldAttrs: (rec { - version = "2022.1"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-HnYOL+aoFjvAs9mhnE+ENCr6Cir/6/qoSwG5eKAuyqc="; - }; - })); - redis = super.redis.overridePythonAttrs (oldAttrs: (rec { - version = "3.5.3"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-Dn4M/KhmDeqLfVzYxPbF4p4R8xFYwLCukaOX8A5aBaI="; - }; - pythonImportsCheck = []; - # tests require a running redis - doCheck = false; - })); - tweepy = super.tweepy.overridePythonAttrs (oldAttrs: (rec { - version = "3.10.0"; - src = super.fetchPypi { - inherit version; - inherit (oldAttrs) pname; - sha256 = "sha256-duaVS4BspHDdqHf1fbh5L/8GoL66DtQ+/DgFdx458Go="; - }; - doCheck = false; - pythonImportsCheck = []; - })); - unidecode = super.unidecode.overridePythonAttrs (oldAttrs: (rec { - version = "1.2.0"; - src = super.fetchPypi { - inherit version; - pname = "Unidecode"; - sha256 = "sha256-jXOpfTh6lWkiNE9rdCQ8LGdxWUZZd4dEstvarY9rcn0="; - }; - })); - }; -} - diff --git a/scripts/run-erpnext.nix b/scripts/run-erpnext.nix new file mode 100644 index 0000000..bd08e0f --- /dev/null +++ b/scripts/run-erpnext.nix @@ -0,0 +1,58 @@ +{ mariadb-client +, python3 +, nodejs +, writeShellApplication +, erpnext-apps-sites +, erpnext-nginx-conf +}: +let + penv = python3.buildEnv.override { + extraLibs = [ + python3.pkgs.frappe + python3.pkgs.erpnext + python3.pkgs.bench + ]; + }; +in writeShellApplication { + name = "run-erpnext"; + runtimeInputs = [mariadb-client nodejs penv]; + text = '' + export PYTHON_PATH=${penv}/${python3.sitePackages} + + tmp=/tmp/erpnext + + mkdir -p $tmp/sites $tmp/config/pids $tmp/logs/nginx $tmp/env/bin + + ln -s ${erpnext-apps-sites}/share/apps $tmp/apps + + for f in ${erpnext-apps-sites}/share/sites/*; do + ln -s "$f" "$tmp/sites/$(basename "$f")" + done + + cat >$tmp/sites/common_site_config.json <