update-python-libraries: commit updates and specify update kind
This commit introduces two new features: 1. specify with --target whether major, minor or patch updates should be made 2. use --commit to create commits for each of the updates
This commit is contained in:
parent
70faf4f5a2
commit
2c9ecb4f4d
|
@ -1,5 +1,5 @@
|
||||||
#! /usr/bin/env nix-shell
|
#! /usr/bin/env nix-shell
|
||||||
#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])'
|
#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ packaging requests toolz ])' -p git
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Update a Python package expression by passing in the `.nix` file, or the directory containing it.
|
Update a Python package expression by passing in the `.nix` file, or the directory containing it.
|
||||||
|
@ -18,7 +18,12 @@ import os
|
||||||
import re
|
import re
|
||||||
import requests
|
import requests
|
||||||
import toolz
|
import toolz
|
||||||
from concurrent.futures import ThreadPoolExecutor as pool
|
from concurrent.futures import ThreadPoolExecutor as Pool
|
||||||
|
from packaging.version import Version as _Version
|
||||||
|
from packaging.version import InvalidVersion
|
||||||
|
from packaging.specifiers import SpecifierSet
|
||||||
|
import collections
|
||||||
|
import subprocess
|
||||||
|
|
||||||
INDEX = "https://pypi.io/pypi"
|
INDEX = "https://pypi.io/pypi"
|
||||||
"""url of PyPI"""
|
"""url of PyPI"""
|
||||||
|
@ -26,10 +31,30 @@ INDEX = "https://pypi.io/pypi"
|
||||||
EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl']
|
EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl']
|
||||||
"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
|
"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
|
||||||
|
|
||||||
|
PRERELEASES = False
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
class Version(_Version, collections.abc.Sequence):
|
||||||
|
|
||||||
|
def __init__(self, version):
|
||||||
|
super().__init__(version)
|
||||||
|
# We cannot use `str(Version(0.04.21))` because that becomes `0.4.21`
|
||||||
|
# https://github.com/avian2/unidecode/issues/13#issuecomment-354538882
|
||||||
|
self.raw_version = version
|
||||||
|
|
||||||
|
def __getitem__(self, i):
|
||||||
|
return self._version.release[i]
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self._version.release)
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
yield from self._version.release
|
||||||
|
|
||||||
|
|
||||||
def _get_values(attribute, text):
|
def _get_values(attribute, text):
|
||||||
"""Match attribute in text and return all matches.
|
"""Match attribute in text and return all matches.
|
||||||
|
|
||||||
|
@ -82,13 +107,59 @@ def _fetch_page(url):
|
||||||
else:
|
else:
|
||||||
raise ValueError("request for {} failed".format(url))
|
raise ValueError("request for {} failed".format(url))
|
||||||
|
|
||||||
def _get_latest_version_pypi(package, extension):
|
|
||||||
|
SEMVER = {
|
||||||
|
'major' : 0,
|
||||||
|
'minor' : 1,
|
||||||
|
'patch' : 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _determine_latest_version(current_version, target, versions):
|
||||||
|
"""Determine latest version, given `target`.
|
||||||
|
"""
|
||||||
|
current_version = Version(current_version)
|
||||||
|
|
||||||
|
def _parse_versions(versions):
|
||||||
|
for v in versions:
|
||||||
|
try:
|
||||||
|
yield Version(v)
|
||||||
|
except InvalidVersion:
|
||||||
|
pass
|
||||||
|
|
||||||
|
versions = _parse_versions(versions)
|
||||||
|
|
||||||
|
index = SEMVER[target]
|
||||||
|
|
||||||
|
ceiling = list(current_version[0:index])
|
||||||
|
if len(ceiling) == 0:
|
||||||
|
ceiling = None
|
||||||
|
else:
|
||||||
|
ceiling[-1]+=1
|
||||||
|
ceiling = Version(".".join(map(str, ceiling)))
|
||||||
|
|
||||||
|
# We do not want prereleases
|
||||||
|
versions = SpecifierSet(prereleases=PRERELEASES).filter(versions)
|
||||||
|
|
||||||
|
if ceiling is not None:
|
||||||
|
versions = SpecifierSet(f"<{ceiling}").filter(versions)
|
||||||
|
|
||||||
|
return (max(sorted(versions))).raw_version
|
||||||
|
|
||||||
|
|
||||||
|
def _get_latest_version_pypi(package, extension, current_version, target):
|
||||||
"""Get latest version and hash from PyPI."""
|
"""Get latest version and hash from PyPI."""
|
||||||
url = "{}/{}/json".format(INDEX, package)
|
url = "{}/{}/json".format(INDEX, package)
|
||||||
json = _fetch_page(url)
|
json = _fetch_page(url)
|
||||||
|
|
||||||
version = json['info']['version']
|
versions = json['releases'].keys()
|
||||||
for release in json['releases'][version]:
|
version = _determine_latest_version(current_version, target, versions)
|
||||||
|
|
||||||
|
try:
|
||||||
|
releases = json['releases'][version]
|
||||||
|
except KeyError as e:
|
||||||
|
raise KeyError('Could not find version {} for {}'.format(version, package)) from e
|
||||||
|
for release in releases:
|
||||||
if release['filename'].endswith(extension):
|
if release['filename'].endswith(extension):
|
||||||
# TODO: In case of wheel we need to do further checks!
|
# TODO: In case of wheel we need to do further checks!
|
||||||
sha256 = release['digests']['sha256']
|
sha256 = release['digests']['sha256']
|
||||||
|
@ -98,7 +169,7 @@ def _get_latest_version_pypi(package, extension):
|
||||||
return version, sha256
|
return version, sha256
|
||||||
|
|
||||||
|
|
||||||
def _get_latest_version_github(package, extension):
|
def _get_latest_version_github(package, extension, current_version, target):
|
||||||
raise ValueError("updating from GitHub is not yet supported.")
|
raise ValueError("updating from GitHub is not yet supported.")
|
||||||
|
|
||||||
|
|
||||||
|
@ -141,9 +212,9 @@ def _determine_extension(text, fetcher):
|
||||||
"""
|
"""
|
||||||
if fetcher == 'fetchPypi':
|
if fetcher == 'fetchPypi':
|
||||||
try:
|
try:
|
||||||
format = _get_unique_value('format', text)
|
src_format = _get_unique_value('format', text)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
format = None # format was not given
|
src_format = None # format was not given
|
||||||
|
|
||||||
try:
|
try:
|
||||||
extension = _get_unique_value('extension', text)
|
extension = _get_unique_value('extension', text)
|
||||||
|
@ -151,9 +222,11 @@ def _determine_extension(text, fetcher):
|
||||||
extension = None # extension was not given
|
extension = None # extension was not given
|
||||||
|
|
||||||
if extension is None:
|
if extension is None:
|
||||||
if format is None:
|
if src_format is None:
|
||||||
format = 'setuptools'
|
src_format = 'setuptools'
|
||||||
extension = FORMATS[format]
|
elif src_format == 'flit':
|
||||||
|
raise ValueError("Don't know how to update a Flit package.")
|
||||||
|
extension = FORMATS[src_format]
|
||||||
|
|
||||||
elif fetcher == 'fetchurl':
|
elif fetcher == 'fetchurl':
|
||||||
url = _get_unique_value('url', text)
|
url = _get_unique_value('url', text)
|
||||||
|
@ -167,9 +240,7 @@ def _determine_extension(text, fetcher):
|
||||||
return extension
|
return extension
|
||||||
|
|
||||||
|
|
||||||
def _update_package(path):
|
def _update_package(path, target):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Read the expression
|
# Read the expression
|
||||||
with open(path, 'r') as f:
|
with open(path, 'r') as f:
|
||||||
|
@ -186,11 +257,13 @@ def _update_package(path):
|
||||||
|
|
||||||
extension = _determine_extension(text, fetcher)
|
extension = _determine_extension(text, fetcher)
|
||||||
|
|
||||||
new_version, new_sha256 = _get_latest_version_pypi(pname, extension)
|
new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target)
|
||||||
|
|
||||||
if new_version == version:
|
if new_version == version:
|
||||||
logging.info("Path {}: no update available for {}.".format(path, pname))
|
logging.info("Path {}: no update available for {}.".format(path, pname))
|
||||||
return False
|
return False
|
||||||
|
elif new_version <= version:
|
||||||
|
raise ValueError("downgrade for {}.".format(pname))
|
||||||
if not new_sha256:
|
if not new_sha256:
|
||||||
raise ValueError("no file available for {}.".format(pname))
|
raise ValueError("no file available for {}.".format(pname))
|
||||||
|
|
||||||
|
@ -202,10 +275,19 @@ def _update_package(path):
|
||||||
|
|
||||||
logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
|
logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
|
||||||
|
|
||||||
return True
|
result = {
|
||||||
|
'path' : path,
|
||||||
|
'target': target,
|
||||||
|
'pname': pname,
|
||||||
|
'old_version' : version,
|
||||||
|
'new_version' : new_version,
|
||||||
|
#'fetcher' : fetcher,
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _update(path):
|
def _update(path, target):
|
||||||
|
|
||||||
# We need to read and modify a Nix expression.
|
# We need to read and modify a Nix expression.
|
||||||
if os.path.isdir(path):
|
if os.path.isdir(path):
|
||||||
|
@ -222,24 +304,58 @@ def _update(path):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return _update_package(path)
|
return _update_package(path, target)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.warning("Path {}: {}".format(path, e))
|
logging.warning("Path {}: {}".format(path, e))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _commit(path, pname, old_version, new_version, **kwargs):
|
||||||
|
"""Commit result.
|
||||||
|
"""
|
||||||
|
|
||||||
|
msg = f'python: {pname}: {old_version} -> {new_version}'
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.check_call(['git', 'add', path])
|
||||||
|
subprocess.check_call(['git', 'commit', '-m', msg])
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
subprocess.check_call(['git', 'checkout', path])
|
||||||
|
raise subprocess.CalledProcessError(f'Could not commit {path}') from e
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument('package', type=str, nargs='+')
|
parser.add_argument('package', type=str, nargs='+')
|
||||||
|
parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')
|
||||||
|
parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
target = args.target
|
||||||
|
|
||||||
packages = map(os.path.abspath, args.package)
|
packages = list(map(os.path.abspath, args.package))
|
||||||
|
|
||||||
|
logging.info("Updating packages...")
|
||||||
|
|
||||||
|
# Use threads to update packages concurrently
|
||||||
|
with Pool() as p:
|
||||||
|
results = list(p.map(lambda pkg: _update(pkg, target), packages))
|
||||||
|
|
||||||
|
logging.info("Finished updating packages.")
|
||||||
|
|
||||||
|
# Commits are created sequentially.
|
||||||
|
if args.commit:
|
||||||
|
logging.info("Committing updates...")
|
||||||
|
list(map(lambda x: _commit(**x), filter(bool, results)))
|
||||||
|
logging.info("Finished committing updates")
|
||||||
|
|
||||||
|
count = sum(map(bool, results))
|
||||||
|
logging.info("{} package(s) updated".format(count))
|
||||||
|
|
||||||
with pool() as p:
|
|
||||||
count = list(p.map(_update, packages))
|
|
||||||
|
|
||||||
logging.info("{} package(s) updated".format(sum(count)))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
Loading…
Reference in a new issue