b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

vimPlugins: make update.py generic over editor

Browse source 
Move the script to maintainers/scripts/pluginupdate.py.
Importing it from the vim and kakoune update scripts
is done in the commit afterwards to cleanup the diff.
This commit is contained in:
Flakebi 2021-02-20 10:09:50 +01:00
parent 9e0a3e5a64
commit 2123e325c9
No known key found for this signature in database
GPG key ID: 38E7ED984D7DCD02
1 changed files with 61 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
pkgs/misc/vim-plugins/update.py → maintainers/scripts/pluginupdate.py Executable file → Normal file
View file

@ -1,5 +1,4 @@
#!/usr/bin/env nix-shell # Used by pkgs/misc/vim-plugins/update.py and pkgs/applications/editors/kakoune/plugins/update.py
#!nix-shell -p nix-prefetch-git -p python3 -p python3Packages.GitPython nix -i python3
# format: # format:
# $ nix run nixpkgs.python3Packages.black -c black update.py # $ nix run nixpkgs.python3Packages.black -c black update.py
@ -35,10 +34,6 @@ ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here
ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # "
ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # "
ROOT = Path(__file__).parent
DEFAULT_IN = ROOT.joinpath("vim-plugin-names")
DEFAULT_OUT = ROOT.joinpath("generated.nix")
DEPRECATED = ROOT.joinpath("deprecated.json")
def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
"""Retry calling the decorated function using an exponential backoff. """Retry calling the decorated function using an exponential backoff.
@ -70,13 +65,15 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa
return deco_retry return deco_retry
def make_request(url: str) -> urllib.request.Request: def make_request(url: str) -> urllib.request.Request:
token = os.getenv("GITHUB_API_TOKEN") token = os.getenv("GITHUB_API_TOKEN")
headers = {} headers = {}
if token is not None: if token is not None:
headers["Authorization"] = f"token {token}" headers["Authorization"] = f"token {token}"
return urllib.request.Request(url, headers=headers) return urllib.request.Request(url, headers=headers)
class Repo: class Repo:
def __init__( def __init__(
self, owner: str, name: str, branch: str, alias: Optional[str] self, owner: str, name: str, branch: str, alias: Optional[str]
@ -181,27 +178,34 @@ class Plugin:
return copy return copy
GET_PLUGINS = f"""(with import <localpkgs> {{}}; class Editor:
let """The configuration of the update script."""
inherit (vimUtils.override {{inherit vim;}}) buildVimPluginFrom2Nix;
generated = callPackage {ROOT}/generated.nix {{ def __init__(
inherit buildVimPluginFrom2Nix; self,
}}; name: str,
hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value; root: Path,
getChecksum = name: value: get_plugins: str,
if hasChecksum value then {{ generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None],
submodules = value.src.fetchSubmodules or false; default_in: Optional[Path] = None,
sha256 = value.src.outputHash; default_out: Optional[Path] = None,
rev = value.src.rev; deprecated: Optional[Path] = None,
}} else null; cache_file: Optional[str] = None,
checksums = lib.mapAttrs getChecksum generated; ):
in lib.filterAttrs (n: v: v != null) checksums)""" self.name = name
self.root = root
self.get_plugins = get_plugins
self.generate_nix = generate_nix
self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
self.default_out = default_out or root.joinpath("generated.nix")
self.deprecated = deprecated or root.joinpath("deprecated.json")
self.cache_file = cache_file or f"{name}-plugin-cache.json"
class CleanEnvironment(object): class CleanEnvironment(object):
def __enter__(self) -> None: def __enter__(self) -> None:
self.old_environ = os.environ.copy() self.old_environ = os.environ.copy()
local_pkgs = str(ROOT.joinpath("../../..")) local_pkgs = str(Path(__file__).parent.parent.parent)
os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}" os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
self.empty_config = NamedTemporaryFile() self.empty_config = NamedTemporaryFile()
self.empty_config.write(b"{}") self.empty_config.write(b"{}")
@ -213,9 +217,9 @@ class CleanEnvironment(object):
self.empty_config.close() self.empty_config.close()
def get_current_plugins() -> List[Plugin]: def get_current_plugins(editor: Editor) -> List[Plugin]:
with CleanEnvironment(): with CleanEnvironment():
out = subprocess.check_output(["nix", "eval", "--json", GET_PLUGINS]) out = subprocess.check_output(["nix", "eval", "--json", editor.get_plugins])
data = json.loads(out) data = json.loads(out)
plugins = [] plugins = []
for name, attr in data.items(): for name, attr in data.items():
@ -319,7 +323,7 @@ def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, str, Optional[str
return plugins return plugins
def get_cache_path() -> Optional[Path]: def get_cache_path(cache_file_name: str) -> Optional[Path]:
xdg_cache = os.environ.get("XDG_CACHE_HOME", None) xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
if xdg_cache is None: if xdg_cache is None:
home = os.environ.get("HOME", None) home = os.environ.get("HOME", None)
@ -327,12 +331,12 @@ def get_cache_path() -> Optional[Path]:
return None return None
xdg_cache = str(Path(home, ".cache")) xdg_cache = str(Path(home, ".cache"))
return Path(xdg_cache, "vim-plugin-cache.json") return Path(xdg_cache, cache_file_name)
class Cache: class Cache:
def __init__(self, initial_plugins: List[Plugin]) -> None: def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None:
self.cache_file = get_cache_path() self.cache_file = get_cache_path(cache_file_name)
downloads = {} downloads = {}
for plugin in initial_plugins: for plugin in initial_plugins:
@ -385,55 +389,11 @@ def prefetch(
return (owner, repo, e, {}) return (owner, repo, e, {})
header = (
"# This file has been generated by ./pkgs/misc/vim-plugins/update.py. Do not edit!"
)
def generate_nix(plugins: List[Tuple[str, str, Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(header)
f.write(
"""
{ lib, buildVimPluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
let
packages = ( self:
{"""
)
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
f.write(
f"""
{plugin.normalized_name} = buildVimPluginFrom2Nix {{
pname = "{plugin.normalized_name}";
version = "{plugin.version}";
src = fetchFromGitHub {{
owner = "{owner}";
repo = "{repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
)
f.write(
"""
});
in lib.fix' (lib.extends overrides packages)
"""
)
print(f"updated {outfile}")
def rewrite_input( def rewrite_input(
input_file: Path, redirects: Dict[str, str] = None, append: Tuple = () input_file: Path,
deprecated: Path,
redirects: Dict[str, str] = None,
append: Tuple = (),
): ):
with open(input_file, "r") as f: with open(input_file, "r") as f:
lines = f.readlines() lines = f.readlines()
@ -444,7 +404,7 @@ def rewrite_input(
lines = [redirects.get(line, line) for line in lines] lines = [redirects.get(line, line) for line in lines]
cur_date_iso = datetime.now().strftime("%Y-%m-%d") cur_date_iso = datetime.now().strftime("%Y-%m-%d")
with open(DEPRECATED, "r") as f: with open(deprecated, "r") as f:
deprecations = json.load(f) deprecations = json.load(f)
for old, new in redirects.items(): for old, new in redirects.items():
old_plugin = fetch_plugin_from_pluginline(old) old_plugin = fetch_plugin_from_pluginline(old)
@ -454,7 +414,7 @@ def rewrite_input(
"new": new_plugin.normalized_name, "new": new_plugin.normalized_name,
"date": cur_date_iso, "date": cur_date_iso,
} }
with open(DEPRECATED, "w") as f: with open(deprecated, "w") as f:
json.dump(deprecations, f, indent=4, sort_keys=True) json.dump(deprecations, f, indent=4, sort_keys=True)
lines = sorted(lines, key=str.casefold) lines = sorted(lines, key=str.casefold)
@ -463,11 +423,11 @@ def rewrite_input(
f.writelines(lines) f.writelines(lines)
def parse_args(): def parse_args(editor: Editor):
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=( description=(
"Updates nix derivations for vim plugins" f"Updates nix derivations for {editor.name} plugins"
f"By default from {DEFAULT_IN} to {DEFAULT_OUT}" f"By default from {editor.default_in} to {editor.default_out}"
) )
) )
parser.add_argument( parser.add_argument(
@ -475,20 +435,20 @@ def parse_args():
dest="add_plugins", dest="add_plugins",
default=[], default=[],
action="append", action="append",
help="Plugin to add to vimPlugins from Github in the form owner/repo", help=f"Plugin to add to {editor.name}Plugins from Github in the form owner/repo",
) )
parser.add_argument( parser.add_argument(
"--input-names", "--input-names",
"-i", "-i",
dest="input_file", dest="input_file",
default=DEFAULT_IN, default=editor.default_in,
help="A list of plugins in the form owner/repo", help="A list of plugins in the form owner/repo",
) )
parser.add_argument( parser.add_argument(
"--out", "--out",
"-o", "-o",
dest="outfile", dest="outfile",
default=DEFAULT_OUT, default=editor.default_out,
help="Filename to save generated nix code", help="Filename to save generated nix code",
) )
parser.add_argument( parser.add_argument(
@ -512,8 +472,8 @@ def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
print("no changes in working tree to commit") print("no changes in working tree to commit")
def get_update(input_file: str, outfile: str, proc: int): def get_update(input_file: str, outfile: str, proc: int, editor: Editor):
cache: Cache = Cache(get_current_plugins()) cache: Cache = Cache(get_current_plugins(editor), editor.cache_file)
_prefetch = functools.partial(prefetch, cache=cache) _prefetch = functools.partial(prefetch, cache=cache)
def update() -> dict: def update() -> dict:
@ -527,42 +487,40 @@ def get_update(input_file: str, outfile: str, proc: int):
plugins, redirects = check_results(results) plugins, redirects = check_results(results)
generate_nix(plugins, outfile) editor.generate_nix(plugins, outfile)
return redirects return redirects
return update return update
def main(): def update_plugins(editor: Editor):
args = parse_args() """The main entry function of this module. All input arguments are grouped in the `Editor`."""
nixpkgs_repo = git.Repo(ROOT, search_parent_directories=True)
update = get_update(args.input_file, args.outfile, args.proc) args = parse_args(editor)
nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
update = get_update(args.input_file, args.outfile, args.proc, editor)
redirects = update() redirects = update()
rewrite_input(args.input_file, redirects) rewrite_input(args.input_file, editor.deprecated, redirects)
commit(nixpkgs_repo, "vimPlugins: update", [args.outfile]) commit(nixpkgs_repo, f"{editor.name}Plugins: update", [args.outfile])
if redirects: if redirects:
update() update()
commit( commit(
nixpkgs_repo, nixpkgs_repo,
"vimPlugins: resolve github repository redirects", f"{editor.name}Plugins: resolve github repository redirects",
[args.outfile, args.input_file, DEPRECATED], [args.outfile, args.input_file, editor.deprecated],
) )
for plugin_line in args.add_plugins: for plugin_line in args.add_plugins:
rewrite_input(args.input_file, append=(plugin_line + "\n",)) rewrite_input(args.input_fil, editor.deprecated, append=(plugin_line + "\n",))
update() update()
plugin = fetch_plugin_from_pluginline(plugin_line) plugin = fetch_plugin_from_pluginline(plugin_line)
commit( commit(
nixpkgs_repo, nixpkgs_repo,
"vimPlugins.{name}: init at {version}".format( "{editor}Plugins.{name}: init at {version}".format(
name=plugin.normalized_name, version=plugin.version editor=editor.name, name=plugin.normalized_name, version=plugin.version
), ),
[args.outfile, args.input_file], [args.outfile, args.input_file],
) )
if __name__ == "__main__":
main()