nixpkgs-suyu/pkgs/applications/version-management/gitlab/update.py

#!/usr/bin/env nix-shell
#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix yarn2nix-moretea.yarn2nix

import click
import click_log
import os
import re
import logging
import subprocess
import json
import pathlib
from distutils.version import LooseVersion
from typing import Iterable

import requests
from xml.etree import ElementTree

logger = logging.getLogger(__name__)


class GitLabRepo:
    version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?")
    def __init__(self, owner: str, repo: str):
        self.owner = owner
        self.repo = repo

    @property
    def url(self):
        return f"https://gitlab.com/{self.owner}/{self.repo}"

    @property
    def tags(self) -> Iterable[str]:
        r = requests.get(self.url + "/tags?format=atom", stream=True)

        tree = ElementTree.fromstring(r.content)
        versions = [e.text for e in tree.findall('{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')]
        # filter out versions not matching version_regex
        versions = list(filter(self.version_regex.match, versions))

        # sort, but ignore v and -ee for sorting comparisons
        versions.sort(key=lambda x: LooseVersion(x.replace("v", "").replace("-ee", "")), reverse=True)
        return versions

    def get_git_hash(self, rev: str):
        out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
        j = json.loads(out)
        return j['sha256']

    @staticmethod
    def rev2version(tag: str) -> str:
        """
        normalize a tag to a version number.
        This obviously isn't very smart if we don't pass something that looks like a tag
        :param tag: the tag to normalize
        :return: a normalized version number
        """
        # strip v prefix
        version = re.sub(r"^v", '', tag)
        # strip -ee suffix
        return re.sub(r"-ee$", '', version)

    def get_file(self, filepath, rev):
        """
        returns file contents at a given rev
        :param filepath: the path to the file, relative to the repo root
        :param rev: the rev to fetch at
        :return:
        """
        return requests.get(self.url + f"/raw/{rev}/{filepath}").text

    def get_data(self, rev, flavour):
        version = self.rev2version(rev)

        passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
                                                               'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
        return dict(version=self.rev2version(rev),
                    repo_hash=self.get_git_hash(rev),
                    owner=self.owner,
                    repo=self.repo,
                    rev=rev,
                    passthru=passthru)


def _flavour2gitlabrepo(flavour: str):
    if flavour not in ['ce', 'ee']:
        raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")

    owner = 'gitlab-org'
    if flavour == 'ce':
        repo = 'gitlab-foss'
    else:
        repo = 'gitlab'

    return GitLabRepo(owner, repo)


def _update_data_json(filename: str, repo: GitLabRepo, rev: str, flavour: str):
    flavour_data = repo.get_data(rev, flavour)

    if not os.path.exists(filename):
        with open(filename, 'w') as f:
            json.dump({flavour: flavour_data}, f, indent=2)
    else:
        with open(filename, 'r+') as f:
            data = json.load(f)
            data[flavour] = flavour_data
            f.seek(0)
            f.truncate()
            json.dump(data, f, indent=2)


def _get_data_json():
    data_file_path = pathlib.Path(__file__).parent / 'data.json'
    with open(data_file_path, 'r') as f:
        return json.load(f)


def _call_update_source_version(pkg, version):
    """calls update-source-version from nixpkgs root dir"""
    nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
    return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)


@click_log.simple_verbosity_option(logger)
@click.group()
def cli():
    pass


@cli.command('update-data')
@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
@click.argument('flavour')
def update_data(rev: str, flavour: str):
    """Update data.nix for a selected flavour"""
    r = _flavour2gitlabrepo(flavour)

    if rev == 'latest':
        # filter out pre and re releases
        rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), r.tags))
    logger.debug(f"Using rev {rev}")

    version = r.rev2version(rev)
    logger.debug(f"Using version {version}")

    data_file_path = pathlib.Path(__file__).parent / 'data.json'

    _update_data_json(filename=data_file_path.as_posix(),
                      repo=r,
                      rev=rev,
                      flavour=flavour)


@cli.command('update-rubyenv')
@click.argument('flavour')
def update_rubyenv(flavour):
    """Update rubyEnv-${flavour}"""
    if flavour not in ['ce', 'ee']:
        raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")

    r = _flavour2gitlabrepo(flavour)
    rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv-{flavour}"

    # load rev from data.json
    data = _get_data_json()
    rev = data[flavour]['rev']

    for fn in ['Gemfile.lock', 'Gemfile']:
        with open(rubyenv_dir / fn, 'w') as f:
            f.write(r.get_file(fn, rev))

    subprocess.check_output(['bundix'], cwd=rubyenv_dir)


@cli.command('update-yarnpkgs')
def update_yarnpkgs():
    """Update yarnPkgs"""

    # yarn.lock is identical between the repos
    repo = _flavour2gitlabrepo('ee')
    yarnpkgs_dir = pathlib.Path(__file__).parent

    # load rev from data.json
    data = _get_data_json()
    rev = data['ee']['rev']

    with open(yarnpkgs_dir / 'yarn.lock', 'w') as f:
        f.write(repo.get_file('yarn.lock', rev))

    with open(yarnpkgs_dir / 'yarnPkgs.nix', 'w') as f:
        subprocess.run(['yarn2nix'], cwd=yarnpkgs_dir, check=True, stdout=f)

    os.unlink(yarnpkgs_dir / 'yarn.lock')


@cli.command('update-gitaly')
def update_gitaly():
    """Update gitaly"""
    data = _get_data_json()
    gitaly_server_version = data['ce']['passthru']['GITALY_SERVER_VERSION']
    r = GitLabRepo('gitlab-org', 'gitaly')
    gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'

    for fn in ['Gemfile.lock', 'Gemfile']:
        with open(gitaly_dir / fn, 'w') as f:
            f.write(r.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))

    for fn in ['go.mod', 'go.sum']:
        with open(gitaly_dir / fn, 'w') as f:
            f.write(r.get_file(fn, f"v{gitaly_server_version}"))

    subprocess.check_output(['bundix'], cwd=gitaly_dir)
    subprocess.check_output(['vgo2nix'], cwd=gitaly_dir)

    for fn in ['go.mod', 'go.sum']:
        os.unlink(gitaly_dir / fn)

    _call_update_source_version('gitaly', gitaly_server_version)


@cli.command('update-gitlab-shell')
def update_gitlab_shell():
    """Update gitlab-shell"""
    data = _get_data_json()
    gitlab_shell_version = data['ce']['passthru']['GITLAB_SHELL_VERSION']
    _call_update_source_version('gitlab-shell', gitlab_shell_version)

    r = GitLabRepo('gitlab-org', 'gitlab-shell')
    gitlab_shell_dir = pathlib.Path(__file__).parent / 'gitlab-shell'

    for fn in ['Gemfile.lock', 'Gemfile']:
        with open(gitlab_shell_dir / fn, 'w') as f:
            f.write(r.get_file(fn, f"v{gitlab_shell_version}"))

    for fn in ['go.mod', 'go.sum']:
        with open(gitlab_shell_dir / fn, 'w') as f:
            f.write(r.get_file(f"go/{fn}", f"v{gitlab_shell_version}"))

    subprocess.check_output(['bundix'], cwd=gitlab_shell_dir)
    subprocess.check_output(['vgo2nix'], cwd=gitlab_shell_dir)

    for fn in ['go.mod', 'go.sum']:
        os.unlink(gitlab_shell_dir / fn)


@cli.command('update-gitlab-workhorse')
def update_gitlab_workhorse():
    """Update gitlab-workhorse"""
    data = _get_data_json()
    gitlab_workhorse_version = data['ce']['passthru']['GITLAB_WORKHORSE_VERSION']
    _call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)

    r = GitLabRepo('gitlab-org', 'gitlab-workhorse')
    gitlab_workhorse_dir = pathlib.Path(__file__).parent / 'gitlab-workhorse'

    for fn in ['go.mod', 'go.sum']:
        with open(gitlab_workhorse_dir / fn, 'w') as f:
            f.write(r.get_file(fn, f"v{gitlab_workhorse_version}"))

    subprocess.check_output(['vgo2nix'], cwd=gitlab_workhorse_dir)

    for fn in ['go.mod', 'go.sum']:
        os.unlink(gitlab_workhorse_dir / fn)

@cli.command('update-all')
@click.pass_context
def update_all(ctx):
    """Update gitlab ce and ee data.nix and rubyenvs to the latest stable release"""
    for flavour in ['ce', 'ee']:
        ctx.invoke(update_data, rev='latest', flavour=flavour)
        ctx.invoke(update_rubyenv, flavour=flavour)
    ctx.invoke(update_yarnpkgs)
    ctx.invoke(update_gitaly)
    ctx.invoke(update_gitlab_shell)
    ctx.invoke(update_gitlab_workhorse)


if __name__ == '__main__':
    cli()