update.py: updater vim updater
We want to support plugins on other hosts than github. A more structured format will allow more features too.
This commit is contained in:
parent
4dae95a040
commit
ec6cf1aa51
|
@ -8,6 +8,7 @@
|
||||||
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
|
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import csv
|
||||||
import functools
|
import functools
|
||||||
import http
|
import http
|
||||||
import json
|
import json
|
||||||
|
@ -28,7 +29,7 @@ from pathlib import Path
|
||||||
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
|
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
|
||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, asdict
|
||||||
|
|
||||||
import git
|
import git
|
||||||
|
|
||||||
|
@ -85,21 +86,30 @@ def make_request(url: str, token=None) -> urllib.request.Request:
|
||||||
headers["Authorization"] = f"token {token}"
|
headers["Authorization"] = f"token {token}"
|
||||||
return urllib.request.Request(url, headers=headers)
|
return urllib.request.Request(url, headers=headers)
|
||||||
|
|
||||||
|
|
||||||
|
Redirects = Dict['Repo', 'Repo']
|
||||||
|
|
||||||
class Repo:
|
class Repo:
|
||||||
def __init__(
|
def __init__(
|
||||||
self, uri: str, branch: str, alias: Optional[str]
|
self, uri: str, branch: str
|
||||||
) -> None:
|
) -> None:
|
||||||
self.uri = uri
|
self.uri = uri
|
||||||
'''Url to the repo'''
|
'''Url to the repo'''
|
||||||
self.branch = branch
|
self._branch = branch
|
||||||
self.alias = alias
|
# {old_uri: new_uri}
|
||||||
self.redirect: Dict[str, str] = {}
|
self.redirect: Redirects = {}
|
||||||
self.token = "dummy_token"
|
self.token = "dummy_token"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def name(self):
|
def name(self):
|
||||||
return self.uri.split('/')[-1]
|
return self.uri.split('/')[-1]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def branch(self):
|
||||||
|
return self._branch or "HEAD"
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f"{self.uri}"
|
||||||
def __repr__(self) -> str:
|
def __repr__(self) -> str:
|
||||||
return f"Repo({self.name}, {self.uri})"
|
return f"Repo({self.name}, {self.uri})"
|
||||||
|
|
||||||
|
@ -109,6 +119,7 @@ class Repo:
|
||||||
|
|
||||||
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
||||||
def latest_commit(self) -> Tuple[str, datetime]:
|
def latest_commit(self) -> Tuple[str, datetime]:
|
||||||
|
log.debug("Latest commit")
|
||||||
loaded = self._prefetch(None)
|
loaded = self._prefetch(None)
|
||||||
updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
|
updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
|
||||||
|
|
||||||
|
@ -124,6 +135,7 @@ class Repo:
|
||||||
return loaded
|
return loaded
|
||||||
|
|
||||||
def prefetch(self, ref: Optional[str]) -> str:
|
def prefetch(self, ref: Optional[str]) -> str:
|
||||||
|
print("Prefetching")
|
||||||
loaded = self._prefetch(ref)
|
loaded = self._prefetch(ref)
|
||||||
return loaded["sha256"]
|
return loaded["sha256"]
|
||||||
|
|
||||||
|
@ -137,21 +149,22 @@ class Repo:
|
||||||
|
|
||||||
class RepoGitHub(Repo):
|
class RepoGitHub(Repo):
|
||||||
def __init__(
|
def __init__(
|
||||||
self, owner: str, repo: str, branch: str, alias: Optional[str]
|
self, owner: str, repo: str, branch: str
|
||||||
) -> None:
|
) -> None:
|
||||||
self.owner = owner
|
self.owner = owner
|
||||||
self.repo = repo
|
self.repo = repo
|
||||||
self.token = None
|
self.token = None
|
||||||
'''Url to the repo'''
|
'''Url to the repo'''
|
||||||
super().__init__(self.url(""), branch, alias)
|
super().__init__(self.url(""), branch)
|
||||||
log.debug("Instantiating github repo %s/%s", self.owner, self.repo)
|
log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def name(self):
|
def name(self):
|
||||||
return self.repo
|
return self.repo
|
||||||
|
|
||||||
def url(self, path: str) -> str:
|
def url(self, path: str) -> str:
|
||||||
return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
|
res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
|
||||||
|
return res
|
||||||
|
|
||||||
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
||||||
def has_submodules(self) -> bool:
|
def has_submodules(self) -> bool:
|
||||||
|
@ -168,6 +181,7 @@ class RepoGitHub(Repo):
|
||||||
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
|
||||||
def latest_commit(self) -> Tuple[str, datetime]:
|
def latest_commit(self) -> Tuple[str, datetime]:
|
||||||
commit_url = self.url(f"commits/{self.branch}.atom")
|
commit_url = self.url(f"commits/{self.branch}.atom")
|
||||||
|
log.debug("Sending request to %s", commit_url)
|
||||||
commit_req = make_request(commit_url, self.token)
|
commit_req = make_request(commit_url, self.token)
|
||||||
with urllib.request.urlopen(commit_req, timeout=10) as req:
|
with urllib.request.urlopen(commit_req, timeout=10) as req:
|
||||||
self._check_for_redirect(commit_url, req)
|
self._check_for_redirect(commit_url, req)
|
||||||
|
@ -191,12 +205,9 @@ class RepoGitHub(Repo):
|
||||||
new_owner, new_name = (
|
new_owner, new_name = (
|
||||||
urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
|
urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
|
||||||
)
|
)
|
||||||
end_line = "\n" if self.alias is None else f" as {self.alias}\n"
|
|
||||||
plugin_line = "{owner}/{name}" + end_line
|
|
||||||
|
|
||||||
old_plugin = plugin_line.format(owner=self.owner, name=self.name)
|
new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
|
||||||
new_plugin = plugin_line.format(owner=new_owner, name=new_name)
|
self.redirect[self] = new_repo
|
||||||
self.redirect[old_plugin] = new_plugin
|
|
||||||
|
|
||||||
|
|
||||||
def prefetch(self, commit: str) -> str:
|
def prefetch(self, commit: str) -> str:
|
||||||
|
@ -207,9 +218,9 @@ class RepoGitHub(Repo):
|
||||||
return sha256
|
return sha256
|
||||||
|
|
||||||
def prefetch_github(self, ref: str) -> str:
|
def prefetch_github(self, ref: str) -> str:
|
||||||
data = subprocess.check_output(
|
cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
|
||||||
["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
|
log.debug("Running %s", cmd)
|
||||||
)
|
data = subprocess.check_output(cmd)
|
||||||
return data.strip().decode("utf-8")
|
return data.strip().decode("utf-8")
|
||||||
|
|
||||||
def as_nix(self, plugin: "Plugin") -> str:
|
def as_nix(self, plugin: "Plugin") -> str:
|
||||||
|
@ -239,21 +250,38 @@ class PluginDesc:
|
||||||
else:
|
else:
|
||||||
return self.alias
|
return self.alias
|
||||||
|
|
||||||
|
def __lt__(self, other):
|
||||||
|
return self.repo.name < other.repo.name
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
|
||||||
|
branch = row["branch"]
|
||||||
|
repo = make_repo(row['repo'], branch.strip())
|
||||||
|
repo.token = config.github_token
|
||||||
|
return PluginDesc(repo, branch.strip(), row["alias"])
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
|
||||||
|
branch = "HEAD"
|
||||||
|
alias = None
|
||||||
|
uri = line
|
||||||
|
if " as " in uri:
|
||||||
|
uri, alias = uri.split(" as ")
|
||||||
|
alias = alias.strip()
|
||||||
|
if "@" in uri:
|
||||||
|
uri, branch = uri.split("@")
|
||||||
|
repo = make_repo(uri.strip(), branch.strip())
|
||||||
|
repo.token = config.github_token
|
||||||
|
return PluginDesc(repo, branch.strip(), alias)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
class Plugin:
|
class Plugin:
|
||||||
def __init__(
|
name: str
|
||||||
self,
|
commit: str
|
||||||
name: str,
|
has_submodules: bool
|
||||||
commit: str,
|
sha256: str
|
||||||
has_submodules: bool,
|
date: Optional[datetime] = None
|
||||||
sha256: str,
|
|
||||||
date: Optional[datetime] = None,
|
|
||||||
) -> None:
|
|
||||||
self.name = name
|
|
||||||
self.commit = commit
|
|
||||||
self.has_submodules = has_submodules
|
|
||||||
self.sha256 = sha256
|
|
||||||
self.date = date
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def normalized_name(self) -> str:
|
def normalized_name(self) -> str:
|
||||||
|
@ -270,6 +298,17 @@ class Plugin:
|
||||||
return copy
|
return copy
|
||||||
|
|
||||||
|
|
||||||
|
def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
|
||||||
|
log.debug("Load plugins from csv %s", input_file)
|
||||||
|
plugins = []
|
||||||
|
with open(input_file, newline='') as csvfile:
|
||||||
|
log.debug("Writing into %s", input_file)
|
||||||
|
reader = csv.DictReader(csvfile,)
|
||||||
|
for line in reader:
|
||||||
|
plugin = PluginDesc.load_from_csv(config, line)
|
||||||
|
plugins.append(plugin)
|
||||||
|
|
||||||
|
return plugins
|
||||||
|
|
||||||
class Editor:
|
class Editor:
|
||||||
"""The configuration of the update script."""
|
"""The configuration of the update script."""
|
||||||
|
@ -298,14 +337,8 @@ class Editor:
|
||||||
return get_current_plugins(self)
|
return get_current_plugins(self)
|
||||||
|
|
||||||
def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
|
def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
|
||||||
plugins = []
|
'''CSV spec'''
|
||||||
with open(plugin_file) as f:
|
return load_plugins_from_csv(config, plugin_file)
|
||||||
for line in f:
|
|
||||||
if line.startswith("#"):
|
|
||||||
continue
|
|
||||||
plugin = parse_plugin_line(config, line)
|
|
||||||
plugins.append(plugin)
|
|
||||||
return plugins
|
|
||||||
|
|
||||||
def generate_nix(self, plugins, outfile: str):
|
def generate_nix(self, plugins, outfile: str):
|
||||||
'''Returns nothing for now, writes directly to outfile'''
|
'''Returns nothing for now, writes directly to outfile'''
|
||||||
|
@ -316,11 +349,11 @@ class Editor:
|
||||||
_prefetch = functools.partial(prefetch, cache=cache)
|
_prefetch = functools.partial(prefetch, cache=cache)
|
||||||
|
|
||||||
def update() -> dict:
|
def update() -> dict:
|
||||||
plugin_names = self.load_plugin_spec(config, input_file)
|
plugins = self.load_plugin_spec(config, input_file)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
pool = Pool(processes=config.proc)
|
pool = Pool(processes=config.proc)
|
||||||
results = pool.map(_prefetch, plugin_names)
|
results = pool.map(_prefetch, plugins)
|
||||||
finally:
|
finally:
|
||||||
cache.store()
|
cache.store()
|
||||||
|
|
||||||
|
@ -423,6 +456,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
|
||||||
data = json.loads(out)
|
data = json.loads(out)
|
||||||
plugins = []
|
plugins = []
|
||||||
for name, attr in data.items():
|
for name, attr in data.items():
|
||||||
|
print("get_current_plugins: name %s" % name)
|
||||||
p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
|
p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
|
||||||
plugins.append(p)
|
plugins.append(p)
|
||||||
return plugins
|
return plugins
|
||||||
|
@ -431,7 +465,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
|
||||||
def prefetch_plugin(
|
def prefetch_plugin(
|
||||||
p: PluginDesc,
|
p: PluginDesc,
|
||||||
cache: "Optional[Cache]" = None,
|
cache: "Optional[Cache]" = None,
|
||||||
) -> Tuple[Plugin, Dict[str, str]]:
|
) -> Tuple[Plugin, Redirects]:
|
||||||
repo, branch, alias = p.repo, p.branch, p.alias
|
repo, branch, alias = p.repo, p.branch, p.alias
|
||||||
name = alias or p.repo.name
|
name = alias or p.repo.name
|
||||||
commit = None
|
commit = None
|
||||||
|
@ -454,11 +488,6 @@ def prefetch_plugin(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def fetch_plugin_from_pluginline(config: FetchConfig, plugin_line: str) -> Plugin:
|
|
||||||
plugin, _ = prefetch_plugin(parse_plugin_line(config, plugin_line))
|
|
||||||
return plugin
|
|
||||||
|
|
||||||
|
|
||||||
def print_download_error(plugin: str, ex: Exception):
|
def print_download_error(plugin: str, ex: Exception):
|
||||||
print(f"{plugin}: {ex}", file=sys.stderr)
|
print(f"{plugin}: {ex}", file=sys.stderr)
|
||||||
ex_traceback = ex.__traceback__
|
ex_traceback = ex.__traceback__
|
||||||
|
@ -468,14 +497,14 @@ def print_download_error(plugin: str, ex: Exception):
|
||||||
]
|
]
|
||||||
print("\n".join(tb_lines))
|
print("\n".join(tb_lines))
|
||||||
|
|
||||||
|
|
||||||
def check_results(
|
def check_results(
|
||||||
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Dict[str, str]]]
|
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Redirects]]
|
||||||
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Dict[str, str]]:
|
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
|
||||||
''' '''
|
''' '''
|
||||||
failures: List[Tuple[str, Exception]] = []
|
failures: List[Tuple[str, Exception]] = []
|
||||||
plugins = []
|
plugins = []
|
||||||
redirects: Dict[str, str] = {}
|
# {old: new} plugindesc
|
||||||
|
redirects: Dict[Repo, Repo] = {}
|
||||||
for (pdesc, result, redirect) in results:
|
for (pdesc, result, redirect) in results:
|
||||||
if isinstance(result, Exception):
|
if isinstance(result, Exception):
|
||||||
failures.append((pdesc.name, result))
|
failures.append((pdesc.name, result))
|
||||||
|
@ -495,31 +524,17 @@ def check_results(
|
||||||
|
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def make_repo(uri, branch, alias) -> Repo:
|
def make_repo(uri: str, branch) -> Repo:
|
||||||
'''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
|
'''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
|
||||||
# dumb check to see if it's of the form owner/repo (=> github) or https://...
|
# dumb check to see if it's of the form owner/repo (=> github) or https://...
|
||||||
res = uri.split('/')
|
res = urlparse(uri)
|
||||||
if len(res) <= 2:
|
if res.netloc in [ "github.com", ""]:
|
||||||
repo = RepoGitHub(res[0], res[1], branch, alias)
|
res = res.path.strip('/').split('/')
|
||||||
|
repo = RepoGitHub(res[0], res[1], branch)
|
||||||
else:
|
else:
|
||||||
repo = Repo(uri.strip(), branch, alias)
|
repo = Repo(uri.strip(), branch)
|
||||||
return repo
|
return repo
|
||||||
|
|
||||||
def parse_plugin_line(config: FetchConfig, line: str) -> PluginDesc:
|
|
||||||
branch = "HEAD"
|
|
||||||
alias = None
|
|
||||||
uri = line
|
|
||||||
if " as " in uri:
|
|
||||||
uri, alias = uri.split(" as ")
|
|
||||||
alias = alias.strip()
|
|
||||||
if "@" in uri:
|
|
||||||
uri, branch = uri.split("@")
|
|
||||||
|
|
||||||
repo = make_repo(uri.strip(), branch.strip(), alias)
|
|
||||||
repo.token = config.github_token
|
|
||||||
|
|
||||||
return PluginDesc(repo, branch.strip(), alias)
|
|
||||||
|
|
||||||
|
|
||||||
def get_cache_path(cache_file_name: str) -> Optional[Path]:
|
def get_cache_path(cache_file_name: str) -> Optional[Path]:
|
||||||
xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
|
xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
|
||||||
|
@ -585,27 +600,27 @@ def prefetch(
|
||||||
return (pluginDesc, e, {})
|
return (pluginDesc, e, {})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def rewrite_input(
|
def rewrite_input(
|
||||||
config: FetchConfig,
|
config: FetchConfig,
|
||||||
input_file: Path,
|
input_file: Path,
|
||||||
deprecated: Path,
|
deprecated: Path,
|
||||||
redirects: Dict[str, str] = None,
|
# old pluginDesc and the new
|
||||||
append: Tuple = (),
|
redirects: Dict[PluginDesc, PluginDesc] = {},
|
||||||
|
append: List[PluginDesc] = [],
|
||||||
):
|
):
|
||||||
with open(input_file, "r") as f:
|
plugins = load_plugins_from_csv(config, input_file,)
|
||||||
lines = f.readlines()
|
|
||||||
|
|
||||||
lines.extend(append)
|
plugins.extend(append)
|
||||||
|
|
||||||
if redirects:
|
if redirects:
|
||||||
lines = [redirects.get(line, line) for line in lines]
|
|
||||||
|
|
||||||
cur_date_iso = datetime.now().strftime("%Y-%m-%d")
|
cur_date_iso = datetime.now().strftime("%Y-%m-%d")
|
||||||
with open(deprecated, "r") as f:
|
with open(deprecated, "r") as f:
|
||||||
deprecations = json.load(f)
|
deprecations = json.load(f)
|
||||||
for old, new in redirects.items():
|
for old, new in redirects.items():
|
||||||
old_plugin = fetch_plugin_from_pluginline(config, old)
|
old_plugin, _ = prefetch_plugin(old)
|
||||||
new_plugin = fetch_plugin_from_pluginline(config, new)
|
new_plugin, _ = prefetch_plugin(new)
|
||||||
if old_plugin.normalized_name != new_plugin.normalized_name:
|
if old_plugin.normalized_name != new_plugin.normalized_name:
|
||||||
deprecations[old_plugin.normalized_name] = {
|
deprecations[old_plugin.normalized_name] = {
|
||||||
"new": new_plugin.normalized_name,
|
"new": new_plugin.normalized_name,
|
||||||
|
@ -615,10 +630,14 @@ def rewrite_input(
|
||||||
json.dump(deprecations, f, indent=4, sort_keys=True)
|
json.dump(deprecations, f, indent=4, sort_keys=True)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
lines = sorted(lines, key=str.casefold)
|
|
||||||
|
|
||||||
with open(input_file, "w") as f:
|
with open(input_file, "w") as f:
|
||||||
f.writelines(lines)
|
log.debug("Writing into %s", input_file)
|
||||||
|
# fields = dataclasses.fields(PluginDesc)
|
||||||
|
fieldnames = ['repo', 'branch', 'alias']
|
||||||
|
writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
|
||||||
|
writer.writeheader()
|
||||||
|
for plugin in sorted(plugins):
|
||||||
|
writer.writerow(asdict(plugin))
|
||||||
|
|
||||||
|
|
||||||
def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
|
def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
|
||||||
|
@ -660,9 +679,11 @@ def update_plugins(editor: Editor, args):
|
||||||
)
|
)
|
||||||
|
|
||||||
for plugin_line in args.add_plugins:
|
for plugin_line in args.add_plugins:
|
||||||
editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=(plugin_line + "\n",))
|
pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
|
||||||
|
append = [ pdesc ]
|
||||||
|
editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
|
||||||
update()
|
update()
|
||||||
plugin = fetch_plugin_from_pluginline(fetch_config, plugin_line)
|
plugin, _ = prefetch_plugin(pdesc, )
|
||||||
if autocommit:
|
if autocommit:
|
||||||
commit(
|
commit(
|
||||||
nixpkgs_repo,
|
nixpkgs_repo,
|
||||||
|
|
Loading…
Reference in a new issue