From 2123e325c969344b79482b5335e1a8c9770f505f Mon Sep 17 00:00:00 2001 From: Flakebi Date: Sat, 20 Feb 2021 10:09:50 +0100 Subject: [PATCH] vimPlugins: make update.py generic over editor Move the script to maintainers/scripts/pluginupdate.py. Importing it from the vim and kakoune update scripts is done in the commit afterwards to cleanup the diff. --- .../scripts/pluginupdate.py | 164 +++++++----------- 1 file changed, 61 insertions(+), 103 deletions(-) rename pkgs/misc/vim-plugins/update.py => maintainers/scripts/pluginupdate.py (80%) mode change 100755 => 100644 diff --git a/pkgs/misc/vim-plugins/update.py b/maintainers/scripts/pluginupdate.py old mode 100755 new mode 100644 similarity index 80% rename from pkgs/misc/vim-plugins/update.py rename to maintainers/scripts/pluginupdate.py index f5d7434fe27..79c79c0f093 --- a/pkgs/misc/vim-plugins/update.py +++ b/maintainers/scripts/pluginupdate.py @@ -1,5 +1,4 @@ -#!/usr/bin/env nix-shell -#!nix-shell -p nix-prefetch-git -p python3 -p python3Packages.GitPython nix -i python3 +# Used by pkgs/misc/vim-plugins/update.py and pkgs/applications/editors/kakoune/plugins/update.py # format: # $ nix run nixpkgs.python3Packages.black -c black update.py @@ -35,10 +34,6 @@ ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " -ROOT = Path(__file__).parent -DEFAULT_IN = ROOT.joinpath("vim-plugin-names") -DEFAULT_OUT = ROOT.joinpath("generated.nix") -DEPRECATED = ROOT.joinpath("deprecated.json") def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): """Retry calling the decorated function using an exponential backoff. @@ -70,13 +65,15 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa return deco_retry + def make_request(url: str) -> urllib.request.Request: token = os.getenv("GITHUB_API_TOKEN") headers = {} if token is not None: - headers["Authorization"] = f"token {token}" + headers["Authorization"] = f"token {token}" return urllib.request.Request(url, headers=headers) + class Repo: def __init__( self, owner: str, name: str, branch: str, alias: Optional[str] @@ -181,27 +178,34 @@ class Plugin: return copy -GET_PLUGINS = f"""(with import {{}}; -let - inherit (vimUtils.override {{inherit vim;}}) buildVimPluginFrom2Nix; - generated = callPackage {ROOT}/generated.nix {{ - inherit buildVimPluginFrom2Nix; - }}; - hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value; - getChecksum = name: value: - if hasChecksum value then {{ - submodules = value.src.fetchSubmodules or false; - sha256 = value.src.outputHash; - rev = value.src.rev; - }} else null; - checksums = lib.mapAttrs getChecksum generated; -in lib.filterAttrs (n: v: v != null) checksums)""" +class Editor: + """The configuration of the update script.""" + + def __init__( + self, + name: str, + root: Path, + get_plugins: str, + generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None], + default_in: Optional[Path] = None, + default_out: Optional[Path] = None, + deprecated: Optional[Path] = None, + cache_file: Optional[str] = None, + ): + self.name = name + self.root = root + self.get_plugins = get_plugins + self.generate_nix = generate_nix + self.default_in = default_in or root.joinpath(f"{name}-plugin-names") + self.default_out = default_out or root.joinpath("generated.nix") + self.deprecated = deprecated or root.joinpath("deprecated.json") + self.cache_file = cache_file or f"{name}-plugin-cache.json" class CleanEnvironment(object): def __enter__(self) -> None: self.old_environ = os.environ.copy() - local_pkgs = str(ROOT.joinpath("../../..")) + local_pkgs = str(Path(__file__).parent.parent.parent) os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}" self.empty_config = NamedTemporaryFile() self.empty_config.write(b"{}") @@ -213,9 +217,9 @@ class CleanEnvironment(object): self.empty_config.close() -def get_current_plugins() -> List[Plugin]: +def get_current_plugins(editor: Editor) -> List[Plugin]: with CleanEnvironment(): - out = subprocess.check_output(["nix", "eval", "--json", GET_PLUGINS]) + out = subprocess.check_output(["nix", "eval", "--json", editor.get_plugins]) data = json.loads(out) plugins = [] for name, attr in data.items(): @@ -319,7 +323,7 @@ def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, str, Optional[str return plugins -def get_cache_path() -> Optional[Path]: +def get_cache_path(cache_file_name: str) -> Optional[Path]: xdg_cache = os.environ.get("XDG_CACHE_HOME", None) if xdg_cache is None: home = os.environ.get("HOME", None) @@ -327,12 +331,12 @@ def get_cache_path() -> Optional[Path]: return None xdg_cache = str(Path(home, ".cache")) - return Path(xdg_cache, "vim-plugin-cache.json") + return Path(xdg_cache, cache_file_name) class Cache: - def __init__(self, initial_plugins: List[Plugin]) -> None: - self.cache_file = get_cache_path() + def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None: + self.cache_file = get_cache_path(cache_file_name) downloads = {} for plugin in initial_plugins: @@ -385,55 +389,11 @@ def prefetch( return (owner, repo, e, {}) -header = ( - "# This file has been generated by ./pkgs/misc/vim-plugins/update.py. Do not edit!" -) - - -def generate_nix(plugins: List[Tuple[str, str, Plugin]], outfile: str): - sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) - - with open(outfile, "w+") as f: - f.write(header) - f.write( - """ -{ lib, buildVimPluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }: -let - packages = ( self: -{""" - ) - for owner, repo, plugin in sorted_plugins: - if plugin.has_submodules: - submodule_attr = "\n fetchSubmodules = true;" - else: - submodule_attr = "" - - f.write( - f""" - {plugin.normalized_name} = buildVimPluginFrom2Nix {{ - pname = "{plugin.normalized_name}"; - version = "{plugin.version}"; - src = fetchFromGitHub {{ - owner = "{owner}"; - repo = "{repo}"; - rev = "{plugin.commit}"; - sha256 = "{plugin.sha256}";{submodule_attr} - }}; - meta.homepage = "https://github.com/{owner}/{repo}/"; - }}; -""" - ) - f.write( - """ -}); -in lib.fix' (lib.extends overrides packages) -""" - ) - print(f"updated {outfile}") - - def rewrite_input( - input_file: Path, redirects: Dict[str, str] = None, append: Tuple = () + input_file: Path, + deprecated: Path, + redirects: Dict[str, str] = None, + append: Tuple = (), ): with open(input_file, "r") as f: lines = f.readlines() @@ -444,7 +404,7 @@ def rewrite_input( lines = [redirects.get(line, line) for line in lines] cur_date_iso = datetime.now().strftime("%Y-%m-%d") - with open(DEPRECATED, "r") as f: + with open(deprecated, "r") as f: deprecations = json.load(f) for old, new in redirects.items(): old_plugin = fetch_plugin_from_pluginline(old) @@ -454,7 +414,7 @@ def rewrite_input( "new": new_plugin.normalized_name, "date": cur_date_iso, } - with open(DEPRECATED, "w") as f: + with open(deprecated, "w") as f: json.dump(deprecations, f, indent=4, sort_keys=True) lines = sorted(lines, key=str.casefold) @@ -463,11 +423,11 @@ def rewrite_input( f.writelines(lines) -def parse_args(): +def parse_args(editor: Editor): parser = argparse.ArgumentParser( description=( - "Updates nix derivations for vim plugins" - f"By default from {DEFAULT_IN} to {DEFAULT_OUT}" + f"Updates nix derivations for {editor.name} plugins" + f"By default from {editor.default_in} to {editor.default_out}" ) ) parser.add_argument( @@ -475,20 +435,20 @@ def parse_args(): dest="add_plugins", default=[], action="append", - help="Plugin to add to vimPlugins from Github in the form owner/repo", + help=f"Plugin to add to {editor.name}Plugins from Github in the form owner/repo", ) parser.add_argument( "--input-names", "-i", dest="input_file", - default=DEFAULT_IN, + default=editor.default_in, help="A list of plugins in the form owner/repo", ) parser.add_argument( "--out", "-o", dest="outfile", - default=DEFAULT_OUT, + default=editor.default_out, help="Filename to save generated nix code", ) parser.add_argument( @@ -512,8 +472,8 @@ def commit(repo: git.Repo, message: str, files: List[Path]) -> None: print("no changes in working tree to commit") -def get_update(input_file: str, outfile: str, proc: int): - cache: Cache = Cache(get_current_plugins()) +def get_update(input_file: str, outfile: str, proc: int, editor: Editor): + cache: Cache = Cache(get_current_plugins(editor), editor.cache_file) _prefetch = functools.partial(prefetch, cache=cache) def update() -> dict: @@ -527,42 +487,40 @@ def get_update(input_file: str, outfile: str, proc: int): plugins, redirects = check_results(results) - generate_nix(plugins, outfile) + editor.generate_nix(plugins, outfile) return redirects return update -def main(): - args = parse_args() - nixpkgs_repo = git.Repo(ROOT, search_parent_directories=True) - update = get_update(args.input_file, args.outfile, args.proc) +def update_plugins(editor: Editor): + """The main entry function of this module. All input arguments are grouped in the `Editor`.""" + + args = parse_args(editor) + nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True) + update = get_update(args.input_file, args.outfile, args.proc, editor) redirects = update() - rewrite_input(args.input_file, redirects) - commit(nixpkgs_repo, "vimPlugins: update", [args.outfile]) + rewrite_input(args.input_file, editor.deprecated, redirects) + commit(nixpkgs_repo, f"{editor.name}Plugins: update", [args.outfile]) if redirects: update() commit( nixpkgs_repo, - "vimPlugins: resolve github repository redirects", - [args.outfile, args.input_file, DEPRECATED], + f"{editor.name}Plugins: resolve github repository redirects", + [args.outfile, args.input_file, editor.deprecated], ) for plugin_line in args.add_plugins: - rewrite_input(args.input_file, append=(plugin_line + "\n",)) + rewrite_input(args.input_fil, editor.deprecated, append=(plugin_line + "\n",)) update() plugin = fetch_plugin_from_pluginline(plugin_line) commit( nixpkgs_repo, - "vimPlugins.{name}: init at {version}".format( - name=plugin.normalized_name, version=plugin.version + "{editor}Plugins.{name}: init at {version}".format( + editor=editor.name, name=plugin.normalized_name, version=plugin.version ), [args.outfile, args.input_file], ) - - -if __name__ == "__main__": - main()