f3eb063ecf
- Update GitLab to 12.3.4 - Update update.py to cope with the new upstream repository structure - Refactor gitlab-shell to use buildGoPackage and bundlerEnv for dependencies - Refactor gitlab-workhorse to use buildGoPackage for dependencies - Make update.py able to update gitlab-shell and gitlab-workhorse dependencies - Various fixes necessary for update to work
276 lines
9.4 KiB
Python
Executable file
276 lines
9.4 KiB
Python
Executable file
#!/usr/bin/env nix-shell
|
|
#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix
|
|
|
|
import click
|
|
import click_log
|
|
import os
|
|
import re
|
|
import logging
|
|
import subprocess
|
|
import json
|
|
import pathlib
|
|
from distutils.version import LooseVersion
|
|
from typing import Iterable
|
|
|
|
import requests
|
|
from xml.etree import ElementTree
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GitLabRepo:
|
|
version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?")
|
|
def __init__(self, owner: str, repo: str):
|
|
self.owner = owner
|
|
self.repo = repo
|
|
|
|
@property
|
|
def url(self):
|
|
return f"https://gitlab.com/{self.owner}/{self.repo}"
|
|
|
|
@property
|
|
def tags(self) -> Iterable[str]:
|
|
r = requests.get(self.url + "/tags?format=atom", stream=True)
|
|
|
|
tree = ElementTree.fromstring(r.content)
|
|
versions = [e.text for e in tree.findall('{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')]
|
|
# filter out versions not matching version_regex
|
|
versions = list(filter(self.version_regex.match, versions))
|
|
|
|
# sort, but ignore v and -ee for sorting comparisons
|
|
versions.sort(key=lambda x: LooseVersion(x.replace("v", "").replace("-ee", "")), reverse=True)
|
|
return versions
|
|
|
|
def get_git_hash(self, rev: str):
|
|
out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
|
|
j = json.loads(out)
|
|
return j['sha256']
|
|
|
|
def get_deb_url(self, flavour: str, version: str, arch: str = 'amd64') -> str:
|
|
"""
|
|
gitlab builds debian packages, which we currently need as we don't build the frontend on our own
|
|
this returns the url of a given flavour, version and arch
|
|
:param flavour: 'ce' or 'ee'
|
|
:param version: a version, without 'v' prefix and '-ee' suffix
|
|
:param arch: amd64
|
|
:return: url of the debian package
|
|
"""
|
|
if flavour not in ['ce', 'ee']:
|
|
raise Exception(f"don't know how to get deb_url for flavour {flavour}")
|
|
return f"https://packages.gitlab.com/gitlab/gitlab-{flavour}/packages" + \
|
|
f"/debian/stretch/gitlab-{flavour}_{version}-{flavour}.0_{arch}.deb/download.deb"
|
|
|
|
def get_deb_hash(self, flavour: str, version: str) -> str:
|
|
out = subprocess.check_output(['nix-prefetch-url', self.get_deb_url(flavour, version)])
|
|
return out.decode('utf-8').strip()
|
|
|
|
@staticmethod
|
|
def rev2version(tag: str) -> str:
|
|
"""
|
|
normalize a tag to a version number.
|
|
This obviously isn't very smart if we don't pass something that looks like a tag
|
|
:param tag: the tag to normalize
|
|
:return: a normalized version number
|
|
"""
|
|
# strip v prefix
|
|
version = re.sub(r"^v", '', tag)
|
|
# strip -ee suffix
|
|
return re.sub(r"-ee$", '', version)
|
|
|
|
def get_file(self, filepath, rev):
|
|
"""
|
|
returns file contents at a given rev
|
|
:param filepath: the path to the file, relative to the repo root
|
|
:param rev: the rev to fetch at
|
|
:return:
|
|
"""
|
|
return requests.get(self.url + f"/raw/{rev}/{filepath}").text
|
|
|
|
def get_data(self, rev, flavour):
|
|
version = self.rev2version(rev)
|
|
|
|
passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
|
|
'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
|
|
return dict(version=self.rev2version(rev),
|
|
repo_hash=self.get_git_hash(rev),
|
|
deb_hash=self.get_deb_hash(flavour, version),
|
|
deb_url=self.get_deb_url(flavour, version),
|
|
owner=self.owner,
|
|
repo=self.repo,
|
|
rev=rev,
|
|
passthru=passthru)
|
|
|
|
|
|
def _flavour2gitlabrepo(flavour: str):
|
|
if flavour not in ['ce', 'ee']:
|
|
raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
|
|
|
|
owner = 'gitlab-org'
|
|
if flavour == 'ce':
|
|
repo = 'gitlab-foss'
|
|
else:
|
|
repo = 'gitlab'
|
|
|
|
return GitLabRepo(owner, repo)
|
|
|
|
|
|
def _update_data_json(filename: str, repo: GitLabRepo, rev: str, flavour: str):
|
|
flavour_data = repo.get_data(rev, flavour)
|
|
|
|
if not os.path.exists(filename):
|
|
with open(filename, 'w') as f:
|
|
json.dump({flavour: flavour_data}, f, indent=2)
|
|
else:
|
|
with open(filename, 'r+') as f:
|
|
data = json.load(f)
|
|
data[flavour] = flavour_data
|
|
f.seek(0)
|
|
f.truncate()
|
|
json.dump(data, f, indent=2)
|
|
|
|
|
|
def _get_data_json():
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
with open(data_file_path, 'r') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def _call_update_source_version(pkg, version):
|
|
"""calls update-source-version from nixpkgs root dir"""
|
|
nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
|
|
return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)
|
|
|
|
|
|
@click_log.simple_verbosity_option(logger)
|
|
@click.group()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command('update-data')
|
|
@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
|
|
@click.argument('flavour')
|
|
def update_data(rev: str, flavour: str):
|
|
"""Update data.nix for a selected flavour"""
|
|
r = _flavour2gitlabrepo(flavour)
|
|
|
|
if rev == 'latest':
|
|
# filter out pre and re releases
|
|
rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), r.tags))
|
|
logger.debug(f"Using rev {rev}")
|
|
|
|
version = r.rev2version(rev)
|
|
logger.debug(f"Using version {version}")
|
|
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
|
|
_update_data_json(filename=data_file_path.as_posix(),
|
|
repo=r,
|
|
rev=rev,
|
|
flavour=flavour)
|
|
|
|
|
|
@cli.command('update-rubyenv')
|
|
@click.argument('flavour')
|
|
def update_rubyenv(flavour):
|
|
"""Update rubyEnv-${flavour}"""
|
|
if flavour not in ['ce', 'ee']:
|
|
raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
|
|
|
|
r = _flavour2gitlabrepo(flavour)
|
|
rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv-{flavour}"
|
|
|
|
# load rev from data.json
|
|
data = _get_data_json()
|
|
rev = data[flavour]['rev']
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(rubyenv_dir / fn, 'w') as f:
|
|
f.write(r.get_file(fn, rev))
|
|
|
|
subprocess.check_output(['bundix'], cwd=rubyenv_dir)
|
|
|
|
|
|
@cli.command('update-gitaly')
|
|
def update_gitaly():
|
|
"""Update gitaly"""
|
|
data = _get_data_json()
|
|
gitaly_server_version = data['ce']['passthru']['GITALY_SERVER_VERSION']
|
|
r = GitLabRepo('gitlab-org', 'gitaly')
|
|
gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(gitaly_dir / fn, 'w') as f:
|
|
f.write(r.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitaly_dir / fn, 'w') as f:
|
|
f.write(r.get_file(fn, f"v{gitaly_server_version}"))
|
|
|
|
subprocess.check_output(['bundix'], cwd=gitaly_dir)
|
|
subprocess.check_output(['vgo2nix'], cwd=gitaly_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitaly_dir / fn)
|
|
|
|
_call_update_source_version('gitaly', gitaly_server_version)
|
|
|
|
|
|
@cli.command('update-gitlab-shell')
|
|
def update_gitlab_shell():
|
|
"""Update gitlab-shell"""
|
|
data = _get_data_json()
|
|
gitlab_shell_version = data['ce']['passthru']['GITLAB_SHELL_VERSION']
|
|
_call_update_source_version('gitlab-shell', gitlab_shell_version)
|
|
|
|
r = GitLabRepo('gitlab-org', 'gitlab-shell')
|
|
gitlab_shell_dir = pathlib.Path(__file__).parent / 'gitlab-shell'
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(gitlab_shell_dir / fn, 'w') as f:
|
|
f.write(r.get_file(fn, f"v{gitlab_shell_version}"))
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitlab_shell_dir / fn, 'w') as f:
|
|
f.write(r.get_file(f"go/{fn}", f"v{gitlab_shell_version}"))
|
|
|
|
subprocess.check_output(['bundix'], cwd=gitlab_shell_dir)
|
|
subprocess.check_output(['vgo2nix'], cwd=gitlab_shell_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitlab_shell_dir / fn)
|
|
|
|
|
|
@cli.command('update-gitlab-workhorse')
|
|
def update_gitlab_workhorse():
|
|
"""Update gitlab-workhorse"""
|
|
data = _get_data_json()
|
|
gitlab_workhorse_version = data['ce']['passthru']['GITLAB_WORKHORSE_VERSION']
|
|
_call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)
|
|
|
|
r = GitLabRepo('gitlab-org', 'gitlab-workhorse')
|
|
gitlab_workhorse_dir = pathlib.Path(__file__).parent / 'gitlab-workhorse'
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitlab_workhorse_dir / fn, 'w') as f:
|
|
f.write(r.get_file(fn, f"v{gitlab_workhorse_version}"))
|
|
|
|
subprocess.check_output(['vgo2nix'], cwd=gitlab_workhorse_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitlab_workhorse_dir / fn)
|
|
|
|
@cli.command('update-all')
|
|
@click.pass_context
|
|
def update_all(ctx):
|
|
"""Update gitlab ce and ee data.nix and rubyenvs to the latest stable release"""
|
|
for flavour in ['ce', 'ee']:
|
|
ctx.invoke(update_data, rev='latest', flavour=flavour)
|
|
ctx.invoke(update_rubyenv, flavour=flavour)
|
|
ctx.invoke(update_gitaly)
|
|
ctx.invoke(update_gitlab_shell)
|
|
ctx.invoke(update_gitlab_workhorse)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cli()
|