Merge pull request #252005 from risicle/ris-apache-airflow-2.7.0

apache-airflow: 2.6.2 -> 2.7.0

pkgs/servers/apache-airflow/package.json

@@ -7,7 +7,7 @@
     "dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development",
     "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress",
     "build": "NODE_ENV=production webpack --progress --mode production",
-    "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc",
+    "lint": "eslint --ignore-path=.eslintignore --max-warnings=0 --ext .js,.jsx,.ts,.tsx . && tsc",
     "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc",
     "format": "yarn prettier --write .",
     "generate-api-types": "npx openapi-typescript \"../api_connexion/openapi/v1.yaml\" --output static/js/types/api-generated.ts && node alias-rest-types.js static/js/types/api-generated.ts"
@@ -29,6 +29,18 @@
     "database",
     "flask"
   ],
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
   "devDependencies": {
     "@babel/core": "^7.18.5",
     "@babel/eslint-parser": "^7.18.2",
@@ -41,6 +53,7 @@
     "@types/color": "^3.0.3",
     "@types/react": "^18.0.12",
     "@types/react-dom": "^18.0.5",
+    "@types/react-syntax-highlighter": "^15.5.6",
     "@types/react-table": "^7.7.12",
     "@typescript-eslint/eslint-plugin": "^5.13.0",
     "@typescript-eslint/parser": "^5.0.0",
@@ -65,6 +78,7 @@
     "file-loader": "^6.0.0",
     "imports-loader": "^1.1.0",
     "jest": "^27.3.1",
+    "jest-canvas-mock": "^2.5.1",
     "mini-css-extract-plugin": "^1.6.2",
     "moment": "^2.29.4",
     "moment-locales-webpack-plugin": "^1.2.0",
@@ -72,7 +86,7 @@
     "openapi-typescript": "^5.4.1",
     "prettier": "^2.8.4",
     "style-loader": "^1.2.1",
-    "stylelint": "^13.6.1",
+    "stylelint": "^15.10.1",
     "stylelint-config-prettier": "^9.0.5",
     "stylelint-config-standard": "^20.0.0",
     "terser-webpack-plugin": "<5.0.0",
@@ -85,14 +99,12 @@
     "webpack-manifest-plugin": "^4.0.0"
   },
   "dependencies": {
-    "@chakra-ui/react": "^2.2.0",
+    "@chakra-ui/react": "2.4.2",
     "@emotion/cache": "^11.9.3",
     "@emotion/react": "^11.9.3",
     "@emotion/styled": "^11",
     "@visx/group": "^2.10.0",
-    "@visx/marker": "^2.12.2",
     "@visx/shape": "^2.12.2",
-    "@visx/zoom": "^2.10.0",
     "axios": "^0.26.0",
     "bootstrap-3-typeahead": "^4.0.2",
     "camelcase-keys": "^7.0.0",
@@ -106,6 +118,7 @@
     "dagre-d3": "^0.6.4",
     "datatables.net": "^1.11.4",
     "datatables.net-bs": "^1.11.4",
+    "echarts": "^5.4.2",
     "elkjs": "^0.7.1",
     "eonasdan-bootstrap-datetimepicker": "^4.17.47",
     "framer-motion": "^6.0.0",
@@ -116,14 +129,15 @@
     "nvd3": "^1.8.6",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
-    "react-icons": "^4.3.1",
+    "react-icons": "^4.9.0",
     "react-json-view": "^1.21.3",
     "react-markdown": "^8.0.4",
     "react-query": "^3.39.1",
     "react-router-dom": "^6.3.0",
+    "react-syntax-highlighter": "^15.5.0",
     "react-table": "^7.8.0",
     "react-textarea-autosize": "^8.3.4",
-    "reactflow": "^11.4.0",
+    "reactflow": "^11.7.4",
     "redoc": "^2.0.0-rc.72",
     "remark-gfm": "^3.0.1",
     "swagger-ui-dist": "4.1.3",

pkgs/servers/apache-airflow/python-package.nix

@@ -26,6 +26,7 @@
 , flask-session
 , flask-wtf
 , gitpython
+, google-re2
 , graphviz
 , gunicorn
 , httpx
@@ -45,6 +46,8 @@
 , mdit-py-plugins
 , numpy
 , openapi-spec-validator
+, opentelemetry-api
+, opentelemetry-exporter-otlp
 , pandas
 , pathspec
 , pendulum
@@ -58,6 +61,7 @@
 , python-slugify
 , python3-openid
 , pythonOlder
+, pythonRelaxDepsHook
 , pyyaml
 , rich
 , rich-argparse
@@ -82,7 +86,7 @@
 , enabledProviders ? []
 }:
 let
-  version = "2.6.2";
+  version = "2.7.0";
 
   airflow-src = fetchFromGitHub rec {
     owner = "apache";
@@ -91,7 +95,7 @@ let
     # Download using the git protocol rather than using tarballs, because the
     # GitHub archive tarballs don't appear to include tests
     forceFetchGit = true;
-    hash = "sha256-ejZw71lMhfnqy4Ziha8/ufmX+SkOfopkeCskf02ZQgA=";
+    hash = "sha256-zB4PWcPkm+lat4tNfVld051RHlC1dW2EbgyoxDao52o=";
   };
 
   # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
@@ -167,6 +171,7 @@ buildPythonPackage rec {
     flask-wtf
     flask-login
     gitpython
+    google-re2
     graphviz
     gunicorn
     httpx
@@ -185,6 +190,8 @@ buildPythonPackage rec {
     mdit-py-plugins
     numpy
     openapi-spec-validator
+    opentelemetry-api
+    opentelemetry-exporter-otlp
     pandas
     pathspec
     pendulum
@@ -216,6 +223,7 @@ buildPythonPackage rec {
 
   buildInputs = [
     airflow-frontend
+    pythonRelaxDepsHook
   ];
 
   nativeCheckInputs = [
@@ -231,16 +239,22 @@ buildPythonPackage rec {
   INSTALL_PROVIDERS_FROM_SOURCES = "true";
 
   postPatch = ''
-    substituteInPlace setup.cfg \
-      --replace "colorlog>=4.0.2, <5.0" "colorlog" \
-      --replace "flask-appbuilder==4.3.0" "flask-appbuilder>=4.3.0" \
-      --replace "pathspec~=0.9.0" "pathspec"
+    # https://github.com/apache/airflow/issues/33854
+    substituteInPlace pyproject.toml \
+      --replace '[project]' $'[project]\nname = "apache-airflow"\nversion = "${version}"'
   '' + lib.optionalString stdenv.isDarwin ''
     # Fix failing test on Hydra
     substituteInPlace airflow/utils/db.py \
       --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
   '';
 
+  pythonRelaxDeps = [
+    "colorlog"
+    "flask-appbuilder"
+    "opentelemetry-api"
+    "pathspec"
+  ];
+
   # allow for gunicorn processes to have access to Python packages
   makeWrapperArgs = [
     "--prefix PYTHONPATH : $PYTHONPATH"
@@ -288,7 +302,7 @@ buildPythonPackage rec {
     update-source-version ${pname} "$new_version"
 
     # Update frontend
-    cd ./pkgs/development/python-modules/apache-airflow
+    cd ./pkgs/servers/apache-airflow
     curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
     curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
     yarn2nix > yarn.nix

pkgs/servers/apache-airflow/update-providers.py

@@ -21,6 +21,7 @@ PKG_PREFERENCES = {
     "dnspython": "dnspython",
     "elasticsearch-dsl": "elasticsearch-dsl",
     "google-api-python-client": "google-api-python-client",
+    "protobuf": "protobuf",
     "psycopg2-binary": "psycopg2",
     "requests_toolbelt": "requests-toolbelt",
 }
@@ -46,7 +47,7 @@ def get_file_from_github(version: str, path: str):
 
 
 def repository_root() -> Path:
-    return Path(os.path.dirname(sys.argv[0])) / "../../../.."
+    return Path(os.path.dirname(sys.argv[0])) / "../../.."
 
 
 def dump_packages() -> Dict[str, Dict[str, str]]: