update.py: updater vim updater

We want to support plugins on other hosts than github.
A more structured format will allow more features too.

maintainers/scripts/pluginupdate.py

@@ -8,6 +8,7 @@
 # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
 
 import argparse
+import csv
 import functools
 import http
 import json
@@ -28,7 +29,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union, Any, Callable
 from urllib.parse import urljoin, urlparse
 from tempfile import NamedTemporaryFile
-from dataclasses import dataclass
+from dataclasses import dataclass, asdict
 
 import git
 
@@ -85,21 +86,30 @@ def make_request(url: str, token=None) -> urllib.request.Request:
         headers["Authorization"] = f"token {token}"
     return urllib.request.Request(url, headers=headers)
 
+
+Redirects = Dict['Repo', 'Repo']
+
 class Repo:
     def __init__(
-        self, uri: str, branch: str, alias: Optional[str]
+        self, uri: str, branch: str
     ) -> None:
         self.uri = uri
         '''Url to the repo'''
-        self.branch = branch
+        self._branch = branch
-        self.alias = alias
+        # {old_uri: new_uri}
-        self.redirect: Dict[str, str] = {}
+        self.redirect: Redirects = {}
         self.token = "dummy_token"
 
     @property
     def name(self):
         return self.uri.split('/')[-1]
 
+    @property
+    def branch(self):
+        return self._branch or "HEAD"
+
+    def __str__(self) -> str:
+        return f"{self.uri}"
     def __repr__(self) -> str:
         return f"Repo({self.name}, {self.uri})"
 
@@ -109,6 +119,7 @@ class Repo:
 
     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def latest_commit(self) -> Tuple[str, datetime]:
+        log.debug("Latest commit")
         loaded = self._prefetch(None)
         updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
 
@@ -124,6 +135,7 @@ class Repo:
         return loaded
 
     def prefetch(self, ref: Optional[str]) -> str:
+        print("Prefetching")
         loaded = self._prefetch(ref)
         return loaded["sha256"]
 
@@ -137,21 +149,22 @@ class Repo:
 
 class RepoGitHub(Repo):
     def __init__(
-        self, owner: str, repo: str, branch: str, alias: Optional[str]
+        self, owner: str, repo: str, branch: str
     ) -> None:
         self.owner = owner
         self.repo = repo
         self.token = None
         '''Url to the repo'''
-        super().__init__(self.url(""), branch, alias)
+        super().__init__(self.url(""), branch)
-        log.debug("Instantiating github repo %s/%s", self.owner, self.repo)
+        log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
 
     @property
     def name(self):
         return self.repo
 
     def url(self, path: str) -> str:
-        return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
+        res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
+        return res
 
     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def has_submodules(self) -> bool:
@@ -168,6 +181,7 @@ class RepoGitHub(Repo):
     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def latest_commit(self) -> Tuple[str, datetime]:
         commit_url = self.url(f"commits/{self.branch}.atom")
+        log.debug("Sending request to %s", commit_url)
         commit_req = make_request(commit_url, self.token)
         with urllib.request.urlopen(commit_req, timeout=10) as req:
             self._check_for_redirect(commit_url, req)
@@ -191,12 +205,9 @@ class RepoGitHub(Repo):
             new_owner, new_name = (
                 urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
             )
-            end_line = "\n" if self.alias is None else f" as {self.alias}\n"
-            plugin_line = "{owner}/{name}" + end_line
 
-            old_plugin = plugin_line.format(owner=self.owner, name=self.name)
+            new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
-            new_plugin = plugin_line.format(owner=new_owner, name=new_name)
+            self.redirect[self] = new_repo
-            self.redirect[old_plugin] = new_plugin
 
 
     def prefetch(self, commit: str) -> str:
@@ -207,9 +218,9 @@ class RepoGitHub(Repo):
         return sha256
 
     def prefetch_github(self, ref: str) -> str:
-        data = subprocess.check_output(
+        cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
-            ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
+        log.debug("Running %s", cmd)
-        )
+        data = subprocess.check_output(cmd)
         return data.strip().decode("utf-8")
 
     def as_nix(self, plugin: "Plugin") -> str:
@@ -239,21 +250,38 @@ class PluginDesc:
         else:
             return self.alias
 
+    def __lt__(self, other):
+        return self.repo.name < other.repo.name
 
+    @staticmethod
+    def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
+        branch = row["branch"]
+        repo = make_repo(row['repo'], branch.strip())
+        repo.token = config.github_token
+        return PluginDesc(repo, branch.strip(), row["alias"])
+
+
+    @staticmethod
+    def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
+        branch = "HEAD"
+        alias = None
+        uri = line
+        if " as " in uri:
+            uri, alias = uri.split(" as ")
+            alias = alias.strip()
+        if "@" in uri:
+            uri, branch = uri.split("@")
+        repo = make_repo(uri.strip(), branch.strip())
+        repo.token = config.github_token
+        return PluginDesc(repo, branch.strip(), alias)
+
+@dataclass
 class Plugin:
-    def __init__(
+    name: str
-        self,
+    commit: str
-        name: str,
+    has_submodules: bool
-        commit: str,
+    sha256: str
-        has_submodules: bool,
+    date: Optional[datetime] = None
-        sha256: str,
-        date: Optional[datetime] = None,
-    ) -> None:
-        self.name = name
-        self.commit = commit
-        self.has_submodules = has_submodules
-        self.sha256 = sha256
-        self.date = date
 
     @property
     def normalized_name(self) -> str:
@@ -270,6 +298,17 @@ class Plugin:
         return copy
 
 
+def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
+    log.debug("Load plugins from csv %s", input_file)
+    plugins = []
+    with open(input_file, newline='') as csvfile:
+        log.debug("Writing into %s", input_file)
+        reader = csv.DictReader(csvfile,)
+        for line in reader:
+            plugin = PluginDesc.load_from_csv(config, line)
+            plugins.append(plugin)
+
+    return plugins
 
 class Editor:
     """The configuration of the update script."""
@@ -298,14 +337,8 @@ class Editor:
         return get_current_plugins(self)
 
     def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
-        plugins = []
+        '''CSV spec'''
-        with open(plugin_file) as f:
+        return load_plugins_from_csv(config, plugin_file)
-            for line in f:
-                if line.startswith("#"):
-                    continue
-                plugin = parse_plugin_line(config, line)
-                plugins.append(plugin)
-        return plugins
 
     def generate_nix(self, plugins, outfile: str):
         '''Returns nothing for now, writes directly to outfile'''
@@ -316,11 +349,11 @@ class Editor:
         _prefetch = functools.partial(prefetch, cache=cache)
 
         def update() -> dict:
-            plugin_names = self.load_plugin_spec(config, input_file)
+            plugins = self.load_plugin_spec(config, input_file)
 
             try:
                 pool = Pool(processes=config.proc)
-                results = pool.map(_prefetch, plugin_names)
+                results = pool.map(_prefetch, plugins)
             finally:
                 cache.store()
 
@@ -423,6 +456,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
     data = json.loads(out)
     plugins = []
     for name, attr in data.items():
+        print("get_current_plugins: name %s" % name)
         p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
         plugins.append(p)
     return plugins
@@ -431,7 +465,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
 def prefetch_plugin(
     p: PluginDesc,
     cache: "Optional[Cache]" = None,
-) -> Tuple[Plugin, Dict[str, str]]:
+) -> Tuple[Plugin, Redirects]:
     repo, branch, alias = p.repo, p.branch, p.alias
     name = alias or p.repo.name
     commit = None
@@ -454,11 +488,6 @@ def prefetch_plugin(
     )
 
 
-def fetch_plugin_from_pluginline(config: FetchConfig, plugin_line: str) -> Plugin:
-    plugin, _ = prefetch_plugin(parse_plugin_line(config, plugin_line))
-    return plugin
-
-
 def print_download_error(plugin: str, ex: Exception):
     print(f"{plugin}: {ex}", file=sys.stderr)
     ex_traceback = ex.__traceback__
@@ -468,14 +497,14 @@ def print_download_error(plugin: str, ex: Exception):
     ]
     print("\n".join(tb_lines))
 
-
 def check_results(
-    results: List[Tuple[PluginDesc, Union[Exception, Plugin], Dict[str, str]]]
+    results: List[Tuple[PluginDesc, Union[Exception, Plugin], Redirects]]
-) -> Tuple[List[Tuple[PluginDesc, Plugin]], Dict[str, str]]:
+) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
     ''' '''
     failures: List[Tuple[str, Exception]] = []
     plugins = []
-    redirects: Dict[str, str] = {}
+    # {old: new} plugindesc
+    redirects: Dict[Repo, Repo] = {}
     for (pdesc, result, redirect) in results:
         if isinstance(result, Exception):
             failures.append((pdesc.name, result))
@@ -495,31 +524,17 @@ def check_results(
 
         sys.exit(1)
 
-def make_repo(uri, branch, alias) -> Repo:
+def make_repo(uri: str, branch) -> Repo:
     '''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
     # dumb check to see if it's of the form owner/repo (=> github) or https://...
-    res = uri.split('/')
+    res = urlparse(uri)
-    if len(res) <= 2:
+    if res.netloc in [ "github.com", ""]:
-        repo = RepoGitHub(res[0], res[1], branch, alias)
+        res = res.path.strip('/').split('/')
+        repo = RepoGitHub(res[0], res[1], branch)
     else:
-        repo = Repo(uri.strip(), branch, alias)
+        repo = Repo(uri.strip(), branch)
     return repo
 
-def parse_plugin_line(config: FetchConfig, line: str) -> PluginDesc:
-    branch = "HEAD"
-    alias = None
-    uri = line
-    if " as " in uri:
-        uri, alias = uri.split(" as ")
-        alias = alias.strip()
-    if "@" in uri:
-        uri, branch = uri.split("@")
-
-    repo = make_repo(uri.strip(), branch.strip(), alias)
-    repo.token = config.github_token
-
-    return PluginDesc(repo, branch.strip(), alias)
-
 
 def get_cache_path(cache_file_name: str) -> Optional[Path]:
     xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
@@ -585,27 +600,27 @@ def prefetch(
         return (pluginDesc, e, {})
 
 
+
 def rewrite_input(
     config: FetchConfig,
     input_file: Path,
     deprecated: Path,
-    redirects: Dict[str, str] = None,
+    # old pluginDesc and the new
-    append: Tuple = (),
+    redirects: Dict[PluginDesc, PluginDesc] = {},
+    append: List[PluginDesc] = [],
 ):
-    with open(input_file, "r") as f:
+    plugins = load_plugins_from_csv(config, input_file,)
-        lines = f.readlines()
 
-    lines.extend(append)
+    plugins.extend(append)
 
     if redirects:
-        lines = [redirects.get(line, line) for line in lines]
 
         cur_date_iso = datetime.now().strftime("%Y-%m-%d")
         with open(deprecated, "r") as f:
             deprecations = json.load(f)
         for old, new in redirects.items():
-            old_plugin = fetch_plugin_from_pluginline(config, old)
+            old_plugin, _ = prefetch_plugin(old)
-            new_plugin = fetch_plugin_from_pluginline(config, new)
+            new_plugin, _ = prefetch_plugin(new)
             if old_plugin.normalized_name != new_plugin.normalized_name:
                 deprecations[old_plugin.normalized_name] = {
                     "new": new_plugin.normalized_name,
@@ -615,10 +630,14 @@ def rewrite_input(
             json.dump(deprecations, f, indent=4, sort_keys=True)
             f.write("\n")
 
-    lines = sorted(lines, key=str.casefold)
-
     with open(input_file, "w") as f:
-        f.writelines(lines)
+        log.debug("Writing into %s", input_file)
+        # fields = dataclasses.fields(PluginDesc)
+        fieldnames = ['repo', 'branch', 'alias']
+        writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
+        writer.writeheader()
+        for plugin in sorted(plugins):
+            writer.writerow(asdict(plugin))
 
 
 def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
@@ -660,9 +679,11 @@ def update_plugins(editor: Editor, args):
             )
 
     for plugin_line in args.add_plugins:
-        editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=(plugin_line + "\n",))
+        pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
+        append = [ pdesc ]
+        editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
         update()
-        plugin = fetch_plugin_from_pluginline(fetch_config, plugin_line)
+        plugin, _ = prefetch_plugin(pdesc, )
         if autocommit:
             commit(
                 nixpkgs_repo,