From 68c59791fb6644ac733d99d0147b09bce4cb8319 Mon Sep 17 00:00:00 2001 From: emilylange Date: Sat, 29 Jul 2023 18:53:34 +0200 Subject: [PATCH] chromium,ungoogled-chromium: fix ofborg maintainer pings ofborg uses `builtins.unsafeGetAttrPos` internally, to figure out which maintainers need to be pinged. e.g: `builtins.unsafeGetAttrPos "version" drv` When using a `.json` file containing the version via `lib.importJSON`, this will always return `null` and thus leading to no pings at all. This commit works around this, resulting in properly working pings for any changes to the upstream-info file. A similar thing has been done for element-{web,desktop} in the past. --- .../networking/browsers/chromium/README.md | 10 +-- .../networking/browsers/chromium/common.nix | 7 +- .../networking/browsers/chromium/default.nix | 18 +++-- .../networking/browsers/chromium/update.py | 31 +++++---- .../browsers/chromium/upstream-info.json | 64 ------------------ .../browsers/chromium/upstream-info.nix | 65 +++++++++++++++++++ .../tools/selenium/chromedriver/default.nix | 2 +- 7 files changed, 105 insertions(+), 92 deletions(-) delete mode 100644 pkgs/applications/networking/browsers/chromium/upstream-info.json create mode 100644 pkgs/applications/networking/browsers/chromium/upstream-info.nix diff --git a/pkgs/applications/networking/browsers/chromium/README.md b/pkgs/applications/networking/browsers/chromium/README.md index 4c93daee4a3..c5a537147c4 100644 --- a/pkgs/applications/networking/browsers/chromium/README.md +++ b/pkgs/applications/networking/browsers/chromium/README.md @@ -17,9 +17,9 @@ Hydra). We use these channels for testing and to fix build errors in advance so that `chromium` updates are trivial and can be merged fast. - `google-chrome`, `google-chrome-beta`, `google-chrome-dev`: Updated via - Chromium's `upstream-info.json` + Chromium's `upstream-info.nix` - `ungoogled-chromium`: @squalus - - `chromedriver`: Updated via Chromium's `upstream-info.json` and not built + - `chromedriver`: Updated via Chromium's `upstream-info.nix` and not built from source. # Upstream links @@ -35,9 +35,9 @@ # Updating Chromium Simply run `./pkgs/applications/networking/browsers/chromium/update.py` to -update `upstream-info.json`. After updates it is important to test at least +update `upstream-info.nix`. After updates it is important to test at least `nixosTests.chromium` (or basic manual testing) and `google-chrome` (which -reuses `upstream-info.json`). +reuses `upstream-info.nix`). Note: Due to the script downloading many large tarballs it might be necessary to adjust the available tmpfs size (it defaults to 10% of the @@ -75,7 +75,7 @@ All updates are considered security critical and should be ported to the stable channel ASAP. When there is a new stable release the old one should receive security updates for roughly one month. After that it is important to mark Chromium as insecure (see 69e4ae56c4b for an example; it is important that the -tested job still succeeds and that all browsers that use `upstream-info.json` +tested job still succeeds and that all browsers that use `upstream-info.nix` are marked as insecure). ## Major version updates diff --git a/pkgs/applications/networking/browsers/chromium/common.nix b/pkgs/applications/networking/browsers/chromium/common.nix index a3f46ba1f12..183a83f52e8 100644 --- a/pkgs/applications/networking/browsers/chromium/common.nix +++ b/pkgs/applications/networking/browsers/chromium/common.nix @@ -374,7 +374,12 @@ let gn = gnChromium; }; }; - }; + } + # overwrite `version` with the exact same `version` from the same source, + # except it internally points to `upstream-info.nix` for + # `builtins.unsafeGetAttrPos`, which is used by ofborg to decide + # which maintainers need to be pinged. + // builtins.removeAttrs upstream-info (builtins.filter (e: e != "version") (builtins.attrNames upstream-info)); # Remove some extraAttrs we supplied to the base attributes already. in stdenv.mkDerivation (base // removeAttrs extraAttrs [ diff --git a/pkgs/applications/networking/browsers/chromium/default.nix b/pkgs/applications/networking/browsers/chromium/default.nix index 70b1de5253b..03b432d1942 100644 --- a/pkgs/applications/networking/browsers/chromium/default.nix +++ b/pkgs/applications/networking/browsers/chromium/default.nix @@ -22,11 +22,11 @@ let llvmPackages = llvmPackages_16; stdenv = llvmPackages.stdenv; - upstream-info = (lib.importJSON ./upstream-info.json).${channel}; + upstream-info = (import ./upstream-info.nix).${channel}; # Helper functions for changes that depend on specific versions: warnObsoleteVersionConditional = min-version: result: - let ungoogled-version = (lib.importJSON ./upstream-info.json).ungoogled-chromium.version; + let ungoogled-version = (import ./upstream-info.nix).ungoogled-chromium.version; in lib.warnIf (lib.versionAtLeast ungoogled-version min-version) "chromium: ungoogled version ${ungoogled-version} is newer than a conditional bounded at ${min-version}. You can safely delete it." @@ -71,10 +71,10 @@ let # Use the latest stable Chrome version if necessary: version = if chromium.upstream-info.sha256bin64 != null then chromium.upstream-info.version - else (lib.importJSON ./upstream-info.json).stable.version; + else (import ./upstream-info.nix).stable.version; sha256 = if chromium.upstream-info.sha256bin64 != null then chromium.upstream-info.sha256bin64 - else (lib.importJSON ./upstream-info.json).stable.sha256bin64; + else (import ./upstream-info.nix).stable.sha256bin64; in fetchurl { urls = map (repo: "${repo}/${pkgName}/${pkgName}_${version}-1_amd64.deb") [ "https://dl.google.com/linux/chrome/deb/pool/main/g" @@ -139,8 +139,6 @@ let sandboxExecutableName = chromium.browser.passthru.sandboxExecutableName; - version = chromium.browser.version; - # We want users to be able to enableWideVine without rebuilding all of # chromium, so we have a separate derivation here that copies chromium # and adds the unfree WidevineCdm. @@ -157,7 +155,7 @@ let in stdenv.mkDerivation { pname = lib.optionalString ungoogled "ungoogled-" + "chromium${suffix}"; - inherit version; + inherit (chromium.browser) version; nativeBuildInputs = [ makeWrapper ed @@ -236,3 +234,9 @@ in stdenv.mkDerivation { inherit chromeSrc sandboxExecutableName; }; } +# the following is a complicated and long-winded variant of +# `inherit (chromium.browser) version`, with the added benefit +# that it keeps the pointer to upstream-info.nix for +# builtins.unsafeGetAttrPos, which is what ofborg uses to +# decide which maintainers need to be pinged. +// builtins.removeAttrs chromium.browser (builtins.filter (e: e != "version") (builtins.attrNames chromium.browser)) diff --git a/pkgs/applications/networking/browsers/chromium/update.py b/pkgs/applications/networking/browsers/chromium/update.py index 380d33c1242..b8af11ee61d 100755 --- a/pkgs/applications/networking/browsers/chromium/update.py +++ b/pkgs/applications/networking/browsers/chromium/update.py @@ -1,8 +1,8 @@ #! /usr/bin/env nix-shell -#! nix-shell -i python -p python3 nix nix-prefetch-git +#! nix-shell -i python -p python3 nix nixfmt nix-prefetch-git """This script automatically updates chromium, google-chrome, chromedriver, and ungoogled-chromium -via upstream-info.json.""" +via upstream-info.nix.""" # Usage: ./update.py [--commit] import base64 @@ -23,16 +23,23 @@ RELEASES_URL = 'https://versionhistory.googleapis.com/v1/chrome/platforms/linux/ DEB_URL = 'https://dl.google.com/linux/chrome/deb/pool/main/g' BUCKET_URL = 'https://commondatastorage.googleapis.com/chromium-browser-official' -JSON_PATH = dirname(abspath(__file__)) + '/upstream-info.json' +PIN_PATH = dirname(abspath(__file__)) + '/upstream-info.nix' UNGOOGLED_FLAGS_PATH = dirname(abspath(__file__)) + '/ungoogled-flags.toml' COMMIT_MESSAGE_SCRIPT = dirname(abspath(__file__)) + '/get-commit-message.py' -def load_json(path): - """Loads the given JSON file.""" - with open(path, 'r') as f: - return json.load(f) +def load_as_json(path): + """Loads the given nix file as JSON.""" + out = subprocess.check_output(['nix-instantiate', '--eval', '--strict', '--json', path]) + return json.loads(out) +def save_dict_as_nix(path, input): + """Saves the given dict/JSON as nix file.""" + json_string = json.dumps(input) + nix = subprocess.check_output(['nix-instantiate', '--eval', '--expr', '{ json }: builtins.fromJSON json', '--argstr', 'json', json_string]) + formatted = subprocess.check_output(['nixfmt'], input=nix) + with open(path, 'w') as out: + out.write(formatted.decode()) def nix_prefetch_url(url, algo='sha256'): """Prefetches the content of the given URL.""" @@ -160,7 +167,7 @@ def print_updates(channels_old, channels_new): channels = {} -last_channels = load_json(JSON_PATH) +last_channels = load_as_json(PIN_PATH) print(f'GET {RELEASES_URL}', file=sys.stderr) @@ -225,9 +232,7 @@ if len(sys.argv) == 2 and sys.argv[1] == '--commit': version_new = sorted_channels[channel_name]['version'] if LooseVersion(version_old) < LooseVersion(version_new): last_channels[channel_name] = sorted_channels[channel_name] - with open(JSON_PATH, 'w') as out: - json.dump(last_channels, out, indent=2) - out.write('\n') + save_dict_as_nix(PIN_PATH, last_channels) attr_name = channel_name_to_attr_name(channel_name) commit_message = f'{attr_name}: {version_old} -> {version_new}' if channel_name == 'stable': @@ -238,7 +243,5 @@ if len(sys.argv) == 2 and sys.argv[1] == '--commit': subprocess.run(['git', 'add', JSON_PATH], check=True) subprocess.run(['git', 'commit', '--file=-'], input=commit_message.encode(), check=True) else: - with open(JSON_PATH, 'w') as out: - json.dump(sorted_channels, out, indent=2) - out.write('\n') + save_dict_as_nix(PIN_PATH, sorted_channels) print_updates(last_channels, sorted_channels) diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.json b/pkgs/applications/networking/browsers/chromium/upstream-info.json deleted file mode 100644 index d5b5e212505..00000000000 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "stable": { - "version": "115.0.5790.110", - "sha256": "0wgp44qnvmdqf2kk870ndm51rcvar36li2qq632ay4n8gfpbrm79", - "sha256bin64": "1w2jl92x78s4vxv4p1imkz7qaq51yvs0wiz2bclbjz0hjlw9akr3", - "deps": { - "gn": { - "version": "2023-05-19", - "url": "https://gn.googlesource.com/gn", - "rev": "e9e83d9095d3234adf68f3e2866f25daf766d5c7", - "sha256": "0y07c18xskq4mclqiz3a63fz8jicz2kqridnvdhqdf75lhp61f8a" - } - }, - "chromedriver": { - "version": "115.0.5790.98", - "sha256_linux": "1797qmb213anvp9lmrkj6wmfdwkdfswmshmk1816zankw5dl883j", - "sha256_darwin": "1c41cb7zh13ny4xvpwy7703cnjrkmqxd3n8zpja7n6a38mi8mgsk", - "sha256_darwin_aarch64": "1kliszw10jnnlhzi8jrdzjq0r7vfn6ksk1spsh2rfn2hmghccv2d" - } - }, - "beta": { - "version": "116.0.5845.50", - "sha256": "0r5m2bcrh2zpl2m8wnzyl4afh8s0dh2m2fnfjf50li94694vy4jz", - "sha256bin64": "047wsszg4c23vxq93a335iymiqpy7lw5izzz4f0zk1a4sijafd59", - "deps": { - "gn": { - "version": "2023-06-09", - "url": "https://gn.googlesource.com/gn", - "rev": "4bd1a77e67958fb7f6739bd4542641646f264e5d", - "sha256": "14h9jqspb86sl5lhh6q0kk2rwa9zcak63f8drp7kb3r4dx08vzsw" - } - } - }, - "dev": { - "version": "117.0.5897.3", - "sha256": "0pyf3k58m26lkc6v6mqpwvhyaj6bbyywl4c17cxb5zmzc1zmc5ia", - "sha256bin64": "10w5dm68aaffgdq0xqi4ans2w7byisqqld09pz5vpk350gy16fjh", - "deps": { - "gn": { - "version": "2023-07-12", - "url": "https://gn.googlesource.com/gn", - "rev": "fae280eabe5d31accc53100137459ece19a7a295", - "sha256": "02javy4jsllwl4mxl2zmg964jvzw800w6gbmr5z6jdkip24fw0kj" - } - } - }, - "ungoogled-chromium": { - "version": "115.0.5790.110", - "sha256": "0wgp44qnvmdqf2kk870ndm51rcvar36li2qq632ay4n8gfpbrm79", - "sha256bin64": "1w2jl92x78s4vxv4p1imkz7qaq51yvs0wiz2bclbjz0hjlw9akr3", - "deps": { - "gn": { - "version": "2023-05-19", - "url": "https://gn.googlesource.com/gn", - "rev": "e9e83d9095d3234adf68f3e2866f25daf766d5c7", - "sha256": "0y07c18xskq4mclqiz3a63fz8jicz2kqridnvdhqdf75lhp61f8a" - }, - "ungoogled-patches": { - "rev": "115.0.5790.110-1", - "sha256": "1jahy4jl5bnnzl6433hln0dj3b39v5zqd90n8zf7ss45wqrff91b" - } - } - } -} diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.nix b/pkgs/applications/networking/browsers/chromium/upstream-info.nix new file mode 100644 index 00000000000..5bf8819390d --- /dev/null +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.nix @@ -0,0 +1,65 @@ +{ + beta = { + deps = { + gn = { + rev = "4bd1a77e67958fb7f6739bd4542641646f264e5d"; + sha256 = "14h9jqspb86sl5lhh6q0kk2rwa9zcak63f8drp7kb3r4dx08vzsw"; + url = "https://gn.googlesource.com/gn"; + version = "2023-06-09"; + }; + }; + sha256 = "0r5m2bcrh2zpl2m8wnzyl4afh8s0dh2m2fnfjf50li94694vy4jz"; + sha256bin64 = "047wsszg4c23vxq93a335iymiqpy7lw5izzz4f0zk1a4sijafd59"; + version = "116.0.5845.50"; + }; + dev = { + deps = { + gn = { + rev = "fae280eabe5d31accc53100137459ece19a7a295"; + sha256 = "02javy4jsllwl4mxl2zmg964jvzw800w6gbmr5z6jdkip24fw0kj"; + url = "https://gn.googlesource.com/gn"; + version = "2023-07-12"; + }; + }; + sha256 = "0pyf3k58m26lkc6v6mqpwvhyaj6bbyywl4c17cxb5zmzc1zmc5ia"; + sha256bin64 = "10w5dm68aaffgdq0xqi4ans2w7byisqqld09pz5vpk350gy16fjh"; + version = "117.0.5897.3"; + }; + stable = { + chromedriver = { + sha256_darwin = "1c41cb7zh13ny4xvpwy7703cnjrkmqxd3n8zpja7n6a38mi8mgsk"; + sha256_darwin_aarch64 = + "1kliszw10jnnlhzi8jrdzjq0r7vfn6ksk1spsh2rfn2hmghccv2d"; + sha256_linux = "1797qmb213anvp9lmrkj6wmfdwkdfswmshmk1816zankw5dl883j"; + version = "115.0.5790.98"; + }; + deps = { + gn = { + rev = "e9e83d9095d3234adf68f3e2866f25daf766d5c7"; + sha256 = "0y07c18xskq4mclqiz3a63fz8jicz2kqridnvdhqdf75lhp61f8a"; + url = "https://gn.googlesource.com/gn"; + version = "2023-05-19"; + }; + }; + sha256 = "0wgp44qnvmdqf2kk870ndm51rcvar36li2qq632ay4n8gfpbrm79"; + sha256bin64 = "1w2jl92x78s4vxv4p1imkz7qaq51yvs0wiz2bclbjz0hjlw9akr3"; + version = "115.0.5790.110"; + }; + ungoogled-chromium = { + deps = { + gn = { + rev = "e9e83d9095d3234adf68f3e2866f25daf766d5c7"; + sha256 = "0y07c18xskq4mclqiz3a63fz8jicz2kqridnvdhqdf75lhp61f8a"; + url = "https://gn.googlesource.com/gn"; + version = "2023-05-19"; + }; + ungoogled-patches = { + rev = "115.0.5790.110-1"; + sha256 = "1jahy4jl5bnnzl6433hln0dj3b39v5zqd90n8zf7ss45wqrff91b"; + }; + }; + sha256 = "0wgp44qnvmdqf2kk870ndm51rcvar36li2qq632ay4n8gfpbrm79"; + sha256bin64 = "1w2jl92x78s4vxv4p1imkz7qaq51yvs0wiz2bclbjz0hjlw9akr3"; + version = "115.0.5790.110"; + }; +} diff --git a/pkgs/development/tools/selenium/chromedriver/default.nix b/pkgs/development/tools/selenium/chromedriver/default.nix index d8201fabb2f..601b124beeb 100644 --- a/pkgs/development/tools/selenium/chromedriver/default.nix +++ b/pkgs/development/tools/selenium/chromedriver/default.nix @@ -6,7 +6,7 @@ }: let - upstream-info = (lib.importJSON ../../../../applications/networking/browsers/chromium/upstream-info.json).stable.chromedriver; + upstream-info = (import ../../../../applications/networking/browsers/chromium/upstream-info.nix).stable.chromedriver; allSpecs = { x86_64-linux = { system = "linux64";