diff --git a/pkgs/servers/web-apps/lemmy/pin.json b/pkgs/servers/web-apps/lemmy/pin.json index 85fef5a4f9fe..d7af8379f6c7 100644 --- a/pkgs/servers/web-apps/lemmy/pin.json +++ b/pkgs/servers/web-apps/lemmy/pin.json @@ -1,5 +1,6 @@ { - "version": "0.18.1", + "serverVersion": "0.18.1", + "uiVersion": "0.18.1", "serverSha256": "sha256-jYbrbIRyXo2G113ReG32oZ56ed2FEB/ZBcqYAxoxzGQ=", "serverCargoSha256": "sha256-7DNMNPSjzYY45DlR6Eo2q6QdwrMrRb51cFOnXfOuub0=", "uiSha256": "sha256-tc7fGA4okIv+3kq5t6I+EN+owdekCgAdk0EtkDgodIU=", diff --git a/pkgs/servers/web-apps/lemmy/server.nix b/pkgs/servers/web-apps/lemmy/server.nix index 894c0ca01c07..652dfbe7b18a 100644 --- a/pkgs/servers/web-apps/lemmy/server.nix +++ b/pkgs/servers/web-apps/lemmy/server.nix @@ -12,7 +12,7 @@ }: let pinData = lib.importJSON ./pin.json; - version = pinData.version; + version = pinData.serverVersion; in rustPlatform.buildRustPackage rec { inherit version; @@ -46,7 +46,7 @@ rustPlatform.buildRustPackage rec { PROTOC_INCLUDE = "${protobuf}/include"; nativeBuildInputs = [ protobuf rustfmt ]; - passthru.updateScript = ./update.sh; + passthru.updateScript = ./update.py; passthru.tests.lemmy-server = nixosTests.lemmy; meta = with lib; { diff --git a/pkgs/servers/web-apps/lemmy/ui.nix b/pkgs/servers/web-apps/lemmy/ui.nix index 3fa1e445d801..06cceef3aa4d 100644 --- a/pkgs/servers/web-apps/lemmy/ui.nix +++ b/pkgs/servers/web-apps/lemmy/ui.nix @@ -33,7 +33,7 @@ let }; name = "lemmy-ui"; - version = pinData.version; + version = pinData.uiVersion; src = fetchFromGitHub { owner = "LemmyNet"; @@ -77,7 +77,7 @@ mkYarnPackage { distPhase = "true"; - passthru.updateScript = ./update.sh; + passthru.updateScript = ./update.py; passthru.tests.lemmy-ui = nixosTests.lemmy; meta = with lib; { diff --git a/pkgs/servers/web-apps/lemmy/update.py b/pkgs/servers/web-apps/lemmy/update.py new file mode 100755 index 000000000000..076f7e754f64 --- /dev/null +++ b/pkgs/servers/web-apps/lemmy/update.py @@ -0,0 +1,177 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 python3.pkgs.semver nix-prefetch-github +from urllib.request import Request, urlopen +import dataclasses +import subprocess +import hashlib +import os.path +import semver +import base64 +from typing import ( + Optional, + Dict, + List, +) +import json +import os + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +NIXPKGS = os.path.abspath(os.path.join(SCRIPT_DIR, "../../../../")) + + +OWNER = "LemmyNet" +UI_REPO = "lemmy-ui" +SERVER_REPO = "lemmy" + + +@dataclasses.dataclass +class Pin: + serverVersion: str + uiVersion: str + serverSha256: str = "" + serverCargoSha256: str = "" + uiSha256: str = "" + uiYarnDepsSha256: str = "" + + filename: Optional[str] = None + + def write(self) -> None: + if not self.filename: + raise ValueError("No filename set") + + with open(self.filename, "w") as fd: + pin = dataclasses.asdict(self) + del pin["filename"] + json.dump(pin, fd, indent=2) + fd.write("\n") + + +def github_get(path: str) -> Dict: + """Send a GET request to Gituhb, optionally adding GITHUB_TOKEN auth header""" + url = f"https://api.github.com/{path.lstrip('/')}" + print(f"Retreiving {url}") + + req = Request(url) + + if "GITHUB_TOKEN" in os.environ: + req.add_header("authorization", f"Bearer {os.environ['GITHUB_TOKEN']}") + + with urlopen(req) as resp: + return json.loads(resp.read()) + + +def get_latest_release(owner: str, repo: str) -> str: + return github_get(f"/repos/{owner}/{repo}/releases/latest")["tag_name"] + + +def sha256_url(url: str) -> str: + sha256 = hashlib.sha256() + with urlopen(url) as resp: + while data := resp.read(1024): + sha256.update(data) + return "sha256-" + base64.urlsafe_b64encode(sha256.digest()).decode() + + +def prefetch_github(owner: str, repo: str, rev: str) -> str: + """Prefetch github rev and return sha256 hash""" + print(f"Prefetching {owner}/{repo}({rev})") + + proc = subprocess.run( + ["nix-prefetch-github", owner, repo, "--rev", rev, "--fetch-submodules"], + check=True, + stdout=subprocess.PIPE, + ) + + sha256 = json.loads(proc.stdout)["sha256"] + if not sha256.startswith("sha256-"): # Work around bug in nix-prefetch-github + return "sha256-" + sha256 + + return sha256 + + +def get_latest_tag(owner: str, repo: str, prerelease: bool = False) -> str: + """Get the latest tag from a Github Repo""" + tags: List[str] = [] + + # As the Github API doesn't have any notion of "latest" for tags we need to + # collect all of them and sort so we can figure out the latest one. + i = 0 + while i <= 100: # Prevent infinite looping + i += 1 + resp = github_get(f"/repos/{owner}/{repo}/tags?page={i}") + if not resp: + break + + # Filter out unparseable tags + for tag in resp: + try: + parsed = semver.Version.parse(tag["name"]) + if ( + semver.Version.parse(tag["name"]) + and not prerelease + and parsed.prerelease + ): # Filter out release candidates + continue + except ValueError: + continue + else: + tags.append(tag["name"]) + + # Sort and return latest + return sorted(tags, key=lambda name: semver.Version.parse(name))[-1] + + +def get_fod_hash(attr: str) -> str: + """ + Get fixed output hash for attribute. + This depends on a fixed output derivation with an empty hash. + """ + + print(f"Getting fixed output hash for {attr}") + + proc = subprocess.run(["nix-build", NIXPKGS, "-A", attr], stderr=subprocess.PIPE) + if proc.returncode != 1: + raise ValueError("Expected nix-build to fail") + + # Iterate list in reverse order so we get the "got:" line early + for line in proc.stderr.decode().split("\n")[::-1]: + cols = line.split() + if cols and cols[0] == "got:": + return cols[1] + + raise ValueError("No fixed output hash found") + + +def make_server_pin(pin: Pin, attr: str) -> None: + pin.serverSha256 = prefetch_github(OWNER, SERVER_REPO, pin.serverVersion) + pin.write() + pin.serverCargoSha256 = get_fod_hash(attr) + pin.write() + + +def make_ui_pin(pin: Pin, package_json: str, attr: str) -> None: + # Save a copy of package.json + print("Getting package.json") + with urlopen( + f"https://raw.githubusercontent.com/{OWNER}/{UI_REPO}/{pin.uiVersion}/package.json" + ) as resp: + with open(os.path.join(SCRIPT_DIR, package_json), "wb") as fd: + fd.write(resp.read()) + + pin.uiSha256 = prefetch_github(OWNER, UI_REPO, pin.uiVersion) + pin.write() + pin.uiYarnDepsSha256 = get_fod_hash(attr) + pin.write() + + +if __name__ == "__main__": + # Get server version + server_version = get_latest_release(OWNER, SERVER_REPO) + + # Get UI version (not always the same as lemmy-server) + ui_version = get_latest_tag(OWNER, UI_REPO) + + pin = Pin(server_version, ui_version, filename=os.path.join(SCRIPT_DIR, "pin.json")) + make_server_pin(pin, "lemmy-server") + make_ui_pin(pin, "package.json", "lemmy-ui") diff --git a/pkgs/servers/web-apps/lemmy/update.sh b/pkgs/servers/web-apps/lemmy/update.sh deleted file mode 100755 index 8de7a8640d8f..000000000000 --- a/pkgs/servers/web-apps/lemmy/update.sh +++ /dev/null @@ -1,52 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i oil -p oil jq sd nix-prefetch-github ripgrep moreutils - -# TODO set to `verbose` or `extdebug` once implemented in oil -shopt --set xtrace -# we need failures inside of command subs to get the correct dependency sha256 -shopt --unset inherit_errexit - -const directory = $(dirname $0 | xargs realpath) -const owner = "LemmyNet" -const ui_repo = "lemmy-ui" -const server_repo = "lemmy" -const latest_rev = $(curl -q https://api.github.com/repos/${owner}/${server_repo}/releases/latest | \ - jq -r '.tag_name') -const latest_version = $(echo $latest_rev) -const current_version = $(jq -r '.version' $directory/pin.json) -echo "latest version: $latest_version, current version: $current_version" -if ("$latest_version" === "$current_version") { - echo "lemmy is already up-to-date" - return 0 -} else { - # for some strange reason, hydra fails on reading upstream package.json directly - const source = "https://raw.githubusercontent.com/$owner/$ui_repo/$latest_version" - const package_json = $(curl -qf $source/package.json) - echo $package_json > $directory/package.json - - const server_tarball_meta = $(nix-prefetch-github $owner $server_repo --rev $latest_rev --fetch-submodules) - const server_tarball_hash = "sha256-$(echo $server_tarball_meta | jq -r '.sha256')" - const ui_tarball_meta = $(nix-prefetch-github $owner $ui_repo --rev $latest_rev --fetch-submodules) - const ui_tarball_hash = "sha256-$(echo $ui_tarball_meta | jq -r '.sha256')" - - jq ".version = \"$latest_version\" | \ - .\"serverSha256\" = \"$server_tarball_hash\" | \ - .\"uiSha256\" = \"$ui_tarball_hash\" | \ - .\"serverCargoSha256\" = \"\" | \ - .\"uiYarnDepsSha256\" = \"\"" $directory/pin.json | sponge $directory/pin.json - - const new_cargo_sha256 = $(nix-build $directory/../../../.. -A lemmy-server 2>&1 | \ - tail -n 2 | \ - head -n 1 | \ - sd '\s+got:\s+' '') - - const new_offline_cache_sha256 = $(nix-build $directory/../../../.. -A lemmy-ui 2>&1 | \ - tail -n 2 | \ - head -n 1 | \ - sd '\s+got:\s+' '') - - jq ".\"serverCargoSha256\" = \"$new_cargo_sha256\" | \ - .\"uiYarnDepsSha256\" = \"$new_offline_cache_sha256\"" \ - $directory/pin.json | sponge $directory/pin.json -} -