Merge pull request #166941 from teto/vim-plugins-toml

This commit is contained in:
Matthieu Coudron 2022-04-05 09:33:40 +02:00 committed by GitHub
commit d396c7d316
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 2041 additions and 1996 deletions

View file

@ -8,6 +8,7 @@
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
import argparse
import csv
import functools
import http
import json
@ -28,7 +29,7 @@ from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
from urllib.parse import urljoin, urlparse
from tempfile import NamedTemporaryFile
from dataclasses import dataclass
from dataclasses import dataclass, asdict
import git
@ -85,21 +86,30 @@ def make_request(url: str, token=None) -> urllib.request.Request:
headers["Authorization"] = f"token {token}"
return urllib.request.Request(url, headers=headers)
Redirects = Dict['Repo', 'Repo']
class Repo:
def __init__(
self, uri: str, branch: str, alias: Optional[str]
self, uri: str, branch: str
) -> None:
self.uri = uri
'''Url to the repo'''
self.branch = branch
self.alias = alias
self.redirect: Dict[str, str] = {}
self._branch = branch
# {old_uri: new_uri}
self.redirect: Redirects = {}
self.token = "dummy_token"
@property
def name(self):
return self.uri.split('/')[-1]
@property
def branch(self):
return self._branch or "HEAD"
def __str__(self) -> str:
return f"{self.uri}"
def __repr__(self) -> str:
return f"Repo({self.name}, {self.uri})"
@ -109,6 +119,7 @@ class Repo:
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
def latest_commit(self) -> Tuple[str, datetime]:
log.debug("Latest commit")
loaded = self._prefetch(None)
updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
@ -124,6 +135,7 @@ class Repo:
return loaded
def prefetch(self, ref: Optional[str]) -> str:
print("Prefetching")
loaded = self._prefetch(ref)
return loaded["sha256"]
@ -137,21 +149,22 @@ class Repo:
class RepoGitHub(Repo):
def __init__(
self, owner: str, repo: str, branch: str, alias: Optional[str]
self, owner: str, repo: str, branch: str
) -> None:
self.owner = owner
self.repo = repo
self.token = None
'''Url to the repo'''
super().__init__(self.url(""), branch, alias)
log.debug("Instantiating github repo %s/%s", self.owner, self.repo)
super().__init__(self.url(""), branch)
log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
@property
def name(self):
return self.repo
def url(self, path: str) -> str:
return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
return res
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
def has_submodules(self) -> bool:
@ -168,6 +181,7 @@ class RepoGitHub(Repo):
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
def latest_commit(self) -> Tuple[str, datetime]:
commit_url = self.url(f"commits/{self.branch}.atom")
log.debug("Sending request to %s", commit_url)
commit_req = make_request(commit_url, self.token)
with urllib.request.urlopen(commit_req, timeout=10) as req:
self._check_for_redirect(commit_url, req)
@ -191,12 +205,9 @@ class RepoGitHub(Repo):
new_owner, new_name = (
urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
)
end_line = "\n" if self.alias is None else f" as {self.alias}\n"
plugin_line = "{owner}/{name}" + end_line
old_plugin = plugin_line.format(owner=self.owner, name=self.name)
new_plugin = plugin_line.format(owner=new_owner, name=new_name)
self.redirect[old_plugin] = new_plugin
new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
self.redirect[self] = new_repo
def prefetch(self, commit: str) -> str:
@ -207,9 +218,9 @@ class RepoGitHub(Repo):
return sha256
def prefetch_github(self, ref: str) -> str:
data = subprocess.check_output(
["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
)
cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
log.debug("Running %s", cmd)
data = subprocess.check_output(cmd)
return data.strip().decode("utf-8")
def as_nix(self, plugin: "Plugin") -> str:
@ -239,21 +250,38 @@ class PluginDesc:
else:
return self.alias
def __lt__(self, other):
return self.repo.name < other.repo.name
@staticmethod
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
branch = row["branch"]
repo = make_repo(row['repo'], branch.strip())
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), row["alias"])
@staticmethod
def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
branch = "HEAD"
alias = None
uri = line
if " as " in uri:
uri, alias = uri.split(" as ")
alias = alias.strip()
if "@" in uri:
uri, branch = uri.split("@")
repo = make_repo(uri.strip(), branch.strip())
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), alias)
@dataclass
class Plugin:
def __init__(
self,
name: str,
commit: str,
has_submodules: bool,
sha256: str,
date: Optional[datetime] = None,
) -> None:
self.name = name
self.commit = commit
self.has_submodules = has_submodules
self.sha256 = sha256
self.date = date
name: str
commit: str
has_submodules: bool
sha256: str
date: Optional[datetime] = None
@property
def normalized_name(self) -> str:
@ -270,6 +298,17 @@ class Plugin:
return copy
def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
log.debug("Load plugins from csv %s", input_file)
plugins = []
with open(input_file, newline='') as csvfile:
log.debug("Writing into %s", input_file)
reader = csv.DictReader(csvfile,)
for line in reader:
plugin = PluginDesc.load_from_csv(config, line)
plugins.append(plugin)
return plugins
class Editor:
"""The configuration of the update script."""
@ -298,14 +337,8 @@ class Editor:
return get_current_plugins(self)
def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
plugins = []
with open(plugin_file) as f:
for line in f:
if line.startswith("#"):
continue
plugin = parse_plugin_line(config, line)
plugins.append(plugin)
return plugins
'''CSV spec'''
return load_plugins_from_csv(config, plugin_file)
def generate_nix(self, plugins, outfile: str):
'''Returns nothing for now, writes directly to outfile'''
@ -316,11 +349,11 @@ class Editor:
_prefetch = functools.partial(prefetch, cache=cache)
def update() -> dict:
plugin_names = self.load_plugin_spec(config, input_file)
plugins = self.load_plugin_spec(config, input_file)
try:
pool = Pool(processes=config.proc)
results = pool.map(_prefetch, plugin_names)
results = pool.map(_prefetch, plugins)
finally:
cache.store()
@ -423,6 +456,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
data = json.loads(out)
plugins = []
for name, attr in data.items():
print("get_current_plugins: name %s" % name)
p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
plugins.append(p)
return plugins
@ -431,7 +465,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]:
def prefetch_plugin(
p: PluginDesc,
cache: "Optional[Cache]" = None,
) -> Tuple[Plugin, Dict[str, str]]:
) -> Tuple[Plugin, Redirects]:
repo, branch, alias = p.repo, p.branch, p.alias
name = alias or p.repo.name
commit = None
@ -454,11 +488,6 @@ def prefetch_plugin(
)
def fetch_plugin_from_pluginline(config: FetchConfig, plugin_line: str) -> Plugin:
plugin, _ = prefetch_plugin(parse_plugin_line(config, plugin_line))
return plugin
def print_download_error(plugin: str, ex: Exception):
print(f"{plugin}: {ex}", file=sys.stderr)
ex_traceback = ex.__traceback__
@ -468,14 +497,14 @@ def print_download_error(plugin: str, ex: Exception):
]
print("\n".join(tb_lines))
def check_results(
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Dict[str, str]]]
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Dict[str, str]]:
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Redirects]]
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
''' '''
failures: List[Tuple[str, Exception]] = []
plugins = []
redirects: Dict[str, str] = {}
# {old: new} plugindesc
redirects: Dict[Repo, Repo] = {}
for (pdesc, result, redirect) in results:
if isinstance(result, Exception):
failures.append((pdesc.name, result))
@ -495,31 +524,17 @@ def check_results(
sys.exit(1)
def make_repo(uri, branch, alias) -> Repo:
def make_repo(uri: str, branch) -> Repo:
'''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
# dumb check to see if it's of the form owner/repo (=> github) or https://...
res = uri.split('/')
if len(res) <= 2:
repo = RepoGitHub(res[0], res[1], branch, alias)
res = urlparse(uri)
if res.netloc in [ "github.com", ""]:
res = res.path.strip('/').split('/')
repo = RepoGitHub(res[0], res[1], branch)
else:
repo = Repo(uri.strip(), branch, alias)
repo = Repo(uri.strip(), branch)
return repo
def parse_plugin_line(config: FetchConfig, line: str) -> PluginDesc:
branch = "HEAD"
alias = None
uri = line
if " as " in uri:
uri, alias = uri.split(" as ")
alias = alias.strip()
if "@" in uri:
uri, branch = uri.split("@")
repo = make_repo(uri.strip(), branch.strip(), alias)
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), alias)
def get_cache_path(cache_file_name: str) -> Optional[Path]:
xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
@ -585,27 +600,27 @@ def prefetch(
return (pluginDesc, e, {})
def rewrite_input(
config: FetchConfig,
input_file: Path,
deprecated: Path,
redirects: Dict[str, str] = None,
append: Tuple = (),
# old pluginDesc and the new
redirects: Dict[PluginDesc, PluginDesc] = {},
append: List[PluginDesc] = [],
):
with open(input_file, "r") as f:
lines = f.readlines()
plugins = load_plugins_from_csv(config, input_file,)
lines.extend(append)
plugins.extend(append)
if redirects:
lines = [redirects.get(line, line) for line in lines]
cur_date_iso = datetime.now().strftime("%Y-%m-%d")
with open(deprecated, "r") as f:
deprecations = json.load(f)
for old, new in redirects.items():
old_plugin = fetch_plugin_from_pluginline(config, old)
new_plugin = fetch_plugin_from_pluginline(config, new)
old_plugin, _ = prefetch_plugin(old)
new_plugin, _ = prefetch_plugin(new)
if old_plugin.normalized_name != new_plugin.normalized_name:
deprecations[old_plugin.normalized_name] = {
"new": new_plugin.normalized_name,
@ -615,10 +630,14 @@ def rewrite_input(
json.dump(deprecations, f, indent=4, sort_keys=True)
f.write("\n")
lines = sorted(lines, key=str.casefold)
with open(input_file, "w") as f:
f.writelines(lines)
log.debug("Writing into %s", input_file)
# fields = dataclasses.fields(PluginDesc)
fieldnames = ['repo', 'branch', 'alias']
writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
writer.writeheader()
for plugin in sorted(plugins):
writer.writerow(asdict(plugin))
def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
@ -660,9 +679,11 @@ def update_plugins(editor: Editor, args):
)
for plugin_line in args.add_plugins:
editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=(plugin_line + "\n",))
pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
append = [ pdesc ]
editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
update()
plugin = fetch_plugin_from_pluginline(fetch_config, plugin_line)
plugin, _ = prefetch_plugin(pdesc, )
if autocommit:
commit(
nixpkgs_repo,

View file

@ -866,6 +866,14 @@
package.
</para>
</listitem>
<listitem>
<para>
The vim/kakoune plugin updater now reads from a CSV file:
check
<literal>pkgs/applications/editors/vim/plugins/vim-plugin-names</literal>
out to see the new format
</para>
</listitem>
</itemizedlist>
</section>
</section>

View file

@ -282,3 +282,5 @@ When upgrading from a previous release, please be aware of the following incompa
- The NixOS test driver supports user services declared by `systemd.user.services`. The methods `waitForUnit`, `getUnitInfo`, `startJob` and `stopJob` provide an optional `$user` argument for that purpose.
- Enabling bash completion on NixOS, `programs.bash.enableCompletion`, will now also enable completion for the Nix command line tools by installing the [nix-bash-completions](https://github.com/hedning/nix-bash-completions) package.
- The vim/kakoune plugin updater now reads from a CSV file: check `pkgs/applications/editors/vim/plugins/vim-plugin-names` out to see the new format

View file

@ -1,19 +1,20 @@
alexherbo2/auto-pairs.kak
alexherbo2/replace-mode.kak
alexherbo2/sleuth.kak
andreyorst/fzf.kak
andreyorst/powerline.kak
basbebe/pandoc.kak
danr/kakoune-easymotion
Delapouite/kakoune-buffers
Delapouite/kakoune-registers
enricozb/tabs.kak@main
greenfork/active-window.kak
kakoune-editor/kakoune-extra-filetypes
kakounedotcom/connect.kak
kakounedotcom/prelude.kak
lePerdu/kakboard
listentolist/kakoune-rainbow
mayjs/openscad.kak
occivink/kakoune-buffer-switcher
occivink/kakoune-vertical-selection
repo,branch,alias
alexherbo2/auto-pairs.kak,,
alexherbo2/replace-mode.kak,,
alexherbo2/sleuth.kak,,
andreyorst/fzf.kak,,
andreyorst/powerline.kak,,
basbebe/pandoc.kak,,
danr/kakoune-easymotion,,
Delapouite/kakoune-buffers,,
Delapouite/kakoune-registers,,
enricozb/tabs.kak@main,,
greenfork/active-window.kak,,
kakoune-editor/kakoune-extra-filetypes,,
kakounedotcom/connect.kak,,
kakounedotcom/prelude.kak,,
lePerdu/kakboard,,
listentolist/kakoune-rainbow,,
mayjs/openscad.kak,,
occivink/kakoune-buffer-switcher,,
occivink/kakoune-vertical-selection,,

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff