lemmy: Rewrite updater in Python
When trying to refactor the updater to also support release candidates I found it to become very messy when I wanted reusable sections and accessing data members returned by the Github API. Python is a more competent programming language that more people in the nix community know how to use.
This commit is contained in:
adisbladis
parent
ea382e4e67
commit
061d5e208c
5 changed files with 183 additions and 57 deletions
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
"version": "0.18.1",
|
||||
"serverVersion": "0.18.1",
|
||||
"uiVersion": "0.18.1",
|
||||
"serverSha256": "sha256-jYbrbIRyXo2G113ReG32oZ56ed2FEB/ZBcqYAxoxzGQ=",
|
||||
"serverCargoSha256": "sha256-7DNMNPSjzYY45DlR6Eo2q6QdwrMrRb51cFOnXfOuub0=",
|
||||
"uiSha256": "sha256-tc7fGA4okIv+3kq5t6I+EN+owdekCgAdk0EtkDgodIU=",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
}:
|
||||
let
|
||||
pinData = lib.importJSON ./pin.json;
|
||||
version = pinData.version;
|
||||
version = pinData.serverVersion;
|
||||
in
|
||||
rustPlatform.buildRustPackage rec {
|
||||
inherit version;
|
||||
|
|
@ -46,7 +46,7 @@ rustPlatform.buildRustPackage rec {
|
|||
PROTOC_INCLUDE = "${protobuf}/include";
|
||||
nativeBuildInputs = [ protobuf rustfmt ];
|
||||
|
||||
passthru.updateScript = ./update.sh;
|
||||
passthru.updateScript = ./update.py;
|
||||
passthru.tests.lemmy-server = nixosTests.lemmy;
|
||||
|
||||
meta = with lib; {
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ let
|
|||
};
|
||||
|
||||
name = "lemmy-ui";
|
||||
version = pinData.version;
|
||||
version = pinData.uiVersion;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "LemmyNet";
|
||||
|
|
@ -77,7 +77,7 @@ mkYarnPackage {
|
|||
|
||||
distPhase = "true";
|
||||
|
||||
passthru.updateScript = ./update.sh;
|
||||
passthru.updateScript = ./update.py;
|
||||
passthru.tests.lemmy-ui = nixosTests.lemmy;
|
||||
|
||||
meta = with lib; {
|
||||
|
|
|
|||
177
pkgs/servers/web-apps/lemmy/update.py
Executable file
177
pkgs/servers/web-apps/lemmy/update.py
Executable file
|
|
@ -0,0 +1,177 @@
|
|||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i python3 -p python3 python3.pkgs.semver nix-prefetch-github
|
||||
from urllib.request import Request, urlopen
|
||||
import dataclasses
|
||||
import subprocess
|
||||
import hashlib
|
||||
import os.path
|
||||
import semver
|
||||
import base64
|
||||
from typing import (
|
||||
Optional,
|
||||
Dict,
|
||||
List,
|
||||
)
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
NIXPKGS = os.path.abspath(os.path.join(SCRIPT_DIR, "../../../../"))
|
||||
|
||||
|
||||
OWNER = "LemmyNet"
|
||||
UI_REPO = "lemmy-ui"
|
||||
SERVER_REPO = "lemmy"
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Pin:
|
||||
serverVersion: str
|
||||
uiVersion: str
|
||||
serverSha256: str = ""
|
||||
serverCargoSha256: str = ""
|
||||
uiSha256: str = ""
|
||||
uiYarnDepsSha256: str = ""
|
||||
|
||||
filename: Optional[str] = None
|
||||
|
||||
def write(self) -> None:
|
||||
if not self.filename:
|
||||
raise ValueError("No filename set")
|
||||
|
||||
with open(self.filename, "w") as fd:
|
||||
pin = dataclasses.asdict(self)
|
||||
del pin["filename"]
|
||||
json.dump(pin, fd, indent=2)
|
||||
fd.write("\n")
|
||||
|
||||
|
||||
def github_get(path: str) -> Dict:
|
||||
"""Send a GET request to Gituhb, optionally adding GITHUB_TOKEN auth header"""
|
||||
url = f"https://api.github.com/{path.lstrip('/')}"
|
||||
print(f"Retreiving {url}")
|
||||
|
||||
req = Request(url)
|
||||
|
||||
if "GITHUB_TOKEN" in os.environ:
|
||||
req.add_header("authorization", f"Bearer {os.environ['GITHUB_TOKEN']}")
|
||||
|
||||
with urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def get_latest_release(owner: str, repo: str) -> str:
|
||||
return github_get(f"/repos/{owner}/{repo}/releases/latest")["tag_name"]
|
||||
|
||||
|
||||
def sha256_url(url: str) -> str:
|
||||
sha256 = hashlib.sha256()
|
||||
with urlopen(url) as resp:
|
||||
while data := resp.read(1024):
|
||||
sha256.update(data)
|
||||
return "sha256-" + base64.urlsafe_b64encode(sha256.digest()).decode()
|
||||
|
||||
|
||||
def prefetch_github(owner: str, repo: str, rev: str) -> str:
|
||||
"""Prefetch github rev and return sha256 hash"""
|
||||
print(f"Prefetching {owner}/{repo}({rev})")
|
||||
|
||||
proc = subprocess.run(
|
||||
["nix-prefetch-github", owner, repo, "--rev", rev, "--fetch-submodules"],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
sha256 = json.loads(proc.stdout)["sha256"]
|
||||
if not sha256.startswith("sha256-"): # Work around bug in nix-prefetch-github
|
||||
return "sha256-" + sha256
|
||||
|
||||
return sha256
|
||||
|
||||
|
||||
def get_latest_tag(owner: str, repo: str, prerelease: bool = False) -> str:
|
||||
"""Get the latest tag from a Github Repo"""
|
||||
tags: List[str] = []
|
||||
|
||||
# As the Github API doesn't have any notion of "latest" for tags we need to
|
||||
# collect all of them and sort so we can figure out the latest one.
|
||||
i = 0
|
||||
while i <= 100: # Prevent infinite looping
|
||||
i += 1
|
||||
resp = github_get(f"/repos/{owner}/{repo}/tags?page={i}")
|
||||
if not resp:
|
||||
break
|
||||
|
||||
# Filter out unparseable tags
|
||||
for tag in resp:
|
||||
try:
|
||||
parsed = semver.Version.parse(tag["name"])
|
||||
if (
|
||||
semver.Version.parse(tag["name"])
|
||||
and not prerelease
|
||||
and parsed.prerelease
|
||||
): # Filter out release candidates
|
||||
continue
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
tags.append(tag["name"])
|
||||
|
||||
# Sort and return latest
|
||||
return sorted(tags, key=lambda name: semver.Version.parse(name))[-1]
|
||||
|
||||
|
||||
def get_fod_hash(attr: str) -> str:
|
||||
"""
|
||||
Get fixed output hash for attribute.
|
||||
This depends on a fixed output derivation with an empty hash.
|
||||
"""
|
||||
|
||||
print(f"Getting fixed output hash for {attr}")
|
||||
|
||||
proc = subprocess.run(["nix-build", NIXPKGS, "-A", attr], stderr=subprocess.PIPE)
|
||||
if proc.returncode != 1:
|
||||
raise ValueError("Expected nix-build to fail")
|
||||
|
||||
# Iterate list in reverse order so we get the "got:" line early
|
||||
for line in proc.stderr.decode().split("\n")[::-1]:
|
||||
cols = line.split()
|
||||
if cols and cols[0] == "got:":
|
||||
return cols[1]
|
||||
|
||||
raise ValueError("No fixed output hash found")
|
||||
|
||||
|
||||
def make_server_pin(pin: Pin, attr: str) -> None:
|
||||
pin.serverSha256 = prefetch_github(OWNER, SERVER_REPO, pin.serverVersion)
|
||||
pin.write()
|
||||
pin.serverCargoSha256 = get_fod_hash(attr)
|
||||
pin.write()
|
||||
|
||||
|
||||
def make_ui_pin(pin: Pin, package_json: str, attr: str) -> None:
|
||||
# Save a copy of package.json
|
||||
print("Getting package.json")
|
||||
with urlopen(
|
||||
f"https://raw.githubusercontent.com/{OWNER}/{UI_REPO}/{pin.uiVersion}/package.json"
|
||||
) as resp:
|
||||
with open(os.path.join(SCRIPT_DIR, package_json), "wb") as fd:
|
||||
fd.write(resp.read())
|
||||
|
||||
pin.uiSha256 = prefetch_github(OWNER, UI_REPO, pin.uiVersion)
|
||||
pin.write()
|
||||
pin.uiYarnDepsSha256 = get_fod_hash(attr)
|
||||
pin.write()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get server version
|
||||
server_version = get_latest_release(OWNER, SERVER_REPO)
|
||||
|
||||
# Get UI version (not always the same as lemmy-server)
|
||||
ui_version = get_latest_tag(OWNER, UI_REPO)
|
||||
|
||||
pin = Pin(server_version, ui_version, filename=os.path.join(SCRIPT_DIR, "pin.json"))
|
||||
make_server_pin(pin, "lemmy-server")
|
||||
make_ui_pin(pin, "package.json", "lemmy-ui")
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i oil -p oil jq sd nix-prefetch-github ripgrep moreutils
|
||||
|
||||
# TODO set to `verbose` or `extdebug` once implemented in oil
|
||||
shopt --set xtrace
|
||||
# we need failures inside of command subs to get the correct dependency sha256
|
||||
shopt --unset inherit_errexit
|
||||
|
||||
const directory = $(dirname $0 | xargs realpath)
|
||||
const owner = "LemmyNet"
|
||||
const ui_repo = "lemmy-ui"
|
||||
const server_repo = "lemmy"
|
||||
const latest_rev = $(curl -q https://api.github.com/repos/${owner}/${server_repo}/releases/latest | \
|
||||
jq -r '.tag_name')
|
||||
const latest_version = $(echo $latest_rev)
|
||||
const current_version = $(jq -r '.version' $directory/pin.json)
|
||||
echo "latest version: $latest_version, current version: $current_version"
|
||||
if ("$latest_version" === "$current_version") {
|
||||
echo "lemmy is already up-to-date"
|
||||
return 0
|
||||
} else {
|
||||
# for some strange reason, hydra fails on reading upstream package.json directly
|
||||
const source = "https://raw.githubusercontent.com/$owner/$ui_repo/$latest_version"
|
||||
const package_json = $(curl -qf $source/package.json)
|
||||
echo $package_json > $directory/package.json
|
||||
|
||||
const server_tarball_meta = $(nix-prefetch-github $owner $server_repo --rev $latest_rev --fetch-submodules)
|
||||
const server_tarball_hash = "sha256-$(echo $server_tarball_meta | jq -r '.sha256')"
|
||||
const ui_tarball_meta = $(nix-prefetch-github $owner $ui_repo --rev $latest_rev --fetch-submodules)
|
||||
const ui_tarball_hash = "sha256-$(echo $ui_tarball_meta | jq -r '.sha256')"
|
||||
|
||||
jq ".version = \"$latest_version\" | \
|
||||
.\"serverSha256\" = \"$server_tarball_hash\" | \
|
||||
.\"uiSha256\" = \"$ui_tarball_hash\" | \
|
||||
.\"serverCargoSha256\" = \"\" | \
|
||||
.\"uiYarnDepsSha256\" = \"\"" $directory/pin.json | sponge $directory/pin.json
|
||||
|
||||
const new_cargo_sha256 = $(nix-build $directory/../../../.. -A lemmy-server 2>&1 | \
|
||||
tail -n 2 | \
|
||||
head -n 1 | \
|
||||
sd '\s+got:\s+' '')
|
||||
|
||||
const new_offline_cache_sha256 = $(nix-build $directory/../../../.. -A lemmy-ui 2>&1 | \
|
||||
tail -n 2 | \
|
||||
head -n 1 | \
|
||||
sd '\s+got:\s+' '')
|
||||
|
||||
jq ".\"serverCargoSha256\" = \"$new_cargo_sha256\" | \
|
||||
.\"uiYarnDepsSha256\" = \"$new_offline_cache_sha256\"" \
|
||||
$directory/pin.json | sponge $directory/pin.json
|
||||
}
|
||||
|
||||
Loading…
Reference in a new issue