From aab14dc76955382e5f76cae40c292c39dbcfc506 Mon Sep 17 00:00:00 2001 From: Daniel Nagy Date: Thu, 24 Nov 2022 13:00:00 +0100 Subject: [PATCH] sapling: simplify dependency generation script --- .gitattributes | 1 + .../version-management/sapling/default.nix | 8 +- .../version-management/sapling/deps.json | 78 +++++++++++++ .../version-management/sapling/deps.nix | 104 ------------------ .../version-management/sapling/gen-deps | 87 --------------- .../version-management/sapling/gen-deps.py | 42 +++++++ 6 files changed, 123 insertions(+), 197 deletions(-) create mode 100644 pkgs/applications/version-management/sapling/deps.json delete mode 100644 pkgs/applications/version-management/sapling/deps.nix delete mode 100755 pkgs/applications/version-management/sapling/gen-deps create mode 100755 pkgs/applications/version-management/sapling/gen-deps.py diff --git a/.gitattributes b/.gitattributes index 4862e0eab93c..36ea9338880b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ **/deps.nix linguist-generated +**/deps.json linguist-generated **/node-packages.nix linguist-generated pkgs/applications/editors/emacs-modes/*-generated.nix linguist-generated diff --git a/pkgs/applications/version-management/sapling/default.nix b/pkgs/applications/version-management/sapling/default.nix index c51454d5c5c0..d964a9b5167c 100644 --- a/pkgs/applications/version-management/sapling/default.nix +++ b/pkgs/applications/version-management/sapling/default.nix @@ -1,9 +1,7 @@ { lib, stdenv, python3Packages, fetchFromGitHub, fetchurl, sd, curl, pkg-config, openssl, rustPlatform, fetchYarnDeps, yarn, nodejs, fixup_yarn_lock }: let - deps = import ./deps.nix { inherit lib fetchurl; }; - version = deps.version; - versionHash = deps.versionHash; + inherit (lib.importJSON ./deps.json) links version versionHash; src = fetchFromGitHub { owner = "facebook"; @@ -83,9 +81,7 @@ let # with filesystem paths for the curl calls. postUnpack = '' mkdir $sourceRoot/hack_pydeps - ( cd $sourceRoot/hack_pydeps - ${deps.links} - ) + ${lib.concatStrings (map (li: "ln -s ${fetchurl li} $sourceRoot/hack_pydeps/${baseNameOf li.url}\n") links)} sed -i "s|https://files.pythonhosted.org/packages/[[:alnum:]]*/[[:alnum:]]*/[[:alnum:]]*/|file://$NIX_BUILD_TOP/$sourceRoot/hack_pydeps/|g" $sourceRoot/setup.py ''; diff --git a/pkgs/applications/version-management/sapling/deps.json b/pkgs/applications/version-management/sapling/deps.json new file mode 100644 index 000000000000..06751b51b15e --- /dev/null +++ b/pkgs/applications/version-management/sapling/deps.json @@ -0,0 +1,78 @@ +{ + "links": [ + { + "sha256": "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab", + "url": "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl" + }, + { + "sha256": "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b", + "url": "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl" + }, + { + "sha256": "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d", + "url": "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl" + }, + { + "sha256": "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw", + "url": "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip" + }, + { + "sha256": "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0", + "url": "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl" + }, + { + "sha256": "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz", + "url": "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl" + }, + { + "sha256": "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v", + "url": "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl" + }, + { + "sha256": "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8", + "url": "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl" + }, + { + "sha256": "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y", + "url": "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl" + }, + { + "sha256": "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26", + "url": "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl" + }, + { + "sha256": "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f", + "url": "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl" + }, + { + "sha256": "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn", + "url": "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl" + }, + { + "sha256": "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn", + "url": "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl" + }, + { + "sha256": "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h", + "url": "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl" + }, + { + "sha256": "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa", + "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + }, + { + "sha256": "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj", + "url": "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl" + }, + { + "sha256": "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm", + "url": "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz" + }, + { + "sha256": "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg", + "url": "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz" + } + ], + "version": "0.1.20221118-210929-cfbb68aa", + "versionHash": "5535144625961033752" +} diff --git a/pkgs/applications/version-management/sapling/deps.nix b/pkgs/applications/version-management/sapling/deps.nix deleted file mode 100644 index 8cf89ec39288..000000000000 --- a/pkgs/applications/version-management/sapling/deps.nix +++ /dev/null @@ -1,104 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# Generated by `gen-deps` - -{ lib, fetchurl }: -let - ptyprocess = fetchurl { - url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl"; - sha256 = "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab"; - }; - pexpect = fetchurl { - url = "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl"; - sha256 = "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b"; - }; - ipython = fetchurl { - url = "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl"; - sha256 = "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d"; - }; - simplegeneric = fetchurl { - url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"; - sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw"; - }; - toml = fetchurl { - url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl"; - sha256 = "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0"; - }; - colorama = fetchurl { - url = "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl"; - sha256 = "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz"; - }; - backcall = fetchurl { - url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl"; - sha256 = "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v"; - }; - setuptools = fetchurl { - url = "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl"; - sha256 = "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8"; - }; - wcwidth = fetchurl { - url = "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl"; - sha256 = "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y"; - }; - prompt_toolkit = fetchurl { - url = "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl"; - sha256 = "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26"; - }; - decorator = fetchurl { - url = "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl"; - sha256 = "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f"; - }; - pickleshare = fetchurl { - url = "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl"; - sha256 = "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn"; - }; - Pygments = fetchurl { - url = "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl"; - sha256 = "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn"; - }; - traitlets = fetchurl { - url = "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl"; - sha256 = "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h"; - }; - six = fetchurl { - url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl"; - sha256 = "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa"; - }; - ipython_genutils = fetchurl { - url = "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl"; - sha256 = "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj"; - }; - ipdb = fetchurl { - url = "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz"; - sha256 = "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm"; - }; - Cython = fetchurl { - url = "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz"; - sha256 = "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg"; - }; -in -{ - links = '' - ln -s ${ptyprocess} ptyprocess-0.7.0-py2.py3-none-any.whl - ln -s ${pexpect} pexpect-4.8.0-py2.py3-none-any.whl - ln -s ${ipython} ipython-7.16.1-py3-none-any.whl - ln -s ${simplegeneric} simplegeneric-0.8.1.zip - ln -s ${toml} toml-0.10.2-py2.py3-none-any.whl - ln -s ${colorama} colorama-0.4.4-py2.py3-none-any.whl - ln -s ${backcall} backcall-0.2.0-py2.py3-none-any.whl - ln -s ${setuptools} setuptools-57.0.0-py3-none-any.whl - ln -s ${wcwidth} wcwidth-0.2.5-py2.py3-none-any.whl - ln -s ${prompt_toolkit} prompt_toolkit-2.0.10-py3-none-any.whl - ln -s ${decorator} decorator-5.0.9-py3-none-any.whl - ln -s ${pickleshare} pickleshare-0.7.5-py2.py3-none-any.whl - ln -s ${Pygments} Pygments-2.9.0-py3-none-any.whl - ln -s ${traitlets} traitlets-4.3.3-py2.py3-none-any.whl - ln -s ${six} six-1.16.0-py2.py3-none-any.whl - ln -s ${ipython_genutils} ipython_genutils-0.2.0-py2.py3-none-any.whl - ln -s ${ipdb} ipdb-0.13.9.tar.gz - ln -s ${Cython} Cython-0.29.26.tar.gz - ''; - - version = "0.1.20221118-210929-cfbb68aa"; - - versionHash = "5535144625961033752"; -} diff --git a/pkgs/applications/version-management/sapling/gen-deps b/pkgs/applications/version-management/sapling/gen-deps deleted file mode 100755 index 3ff7e24916c1..000000000000 --- a/pkgs/applications/version-management/sapling/gen-deps +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" nixpkgs-fmt - -# vim: set syntax=python: - -import hashlib -import json -import re -import struct -import subprocess - -import requests - -gen_file = "deps.nix" -repo = "facebook/sapling" -source_file = "eden/scm/setup.py" - -# Fetch the latest stable release metadata from GitHub -latest = requests.get(f"https://api.github.com/repos/{repo}/releases/latest").text -latest = json.loads(latest) - -# Find latest's git tag which corresponds to the Sapling version. Also needed -# is a hash of the version, so calculate that here. Taken from Sapling source -# `$root/eden/scm/setup_with_version.py`. -version = str(latest["tag_name"]) -version_hash = str( - struct.unpack(">Q", hashlib.sha1(version.encode("ascii")).digest()[:8])[0] -) - -# Fetch the `setup.py` source and look for instances of assets being downloaded -# from files.pythonhosted.org. -source = requests.get( - f"https://raw.githubusercontent.com/{repo}/{version}/{source_file}" -).text -matches = re.findall( - r'(https://files\.pythonhosted\.org/packages/.*?/.*?/.*?/)(.*?)(-.*?)"', source -) - -# For each matched asset URL, prefetch said asset and build the corresponding -# Nix code needed to call `fetchurl` and perform symlinks in the actual build. -assets = "" -links = "" -for m in matches: - sha256 = ( - subprocess.Popen( - ["nix-prefetch-url", "--type", "sha256", f"{m[0]}{m[1]}{m[2]}"], - text=True, - stdout=subprocess.PIPE, - ) - .stdout.read() - .strip() - ) - assets += """ {} = fetchurl {{ - url = "{}{}{}"; - sha256 = "{}"; - }};\n""".format( - m[1], m[0], m[1], m[2], sha256 - ) - links += " ln -s ${{{}}} {}{}\n".format(m[1], m[1], m[2]) -assets = assets.strip() -links = links.strip() - -# Render an autogenerated Nix file with the fetched dependency and version -# information for import in the main derivation. -out = f""" -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# Generated by `gen-deps` - -{{ lib, fetchurl }}: -let - {assets} -in -{{ - links = '' - {links} - ''; - - version = "{version}"; - - versionHash = "{version_hash}"; -}} -""" -with open(gen_file, "w+") as f: - f.write(out.strip()) - -# Formamt the generated Nix file, which also serves as a syntax sanity check. -subprocess.run(["nixpkgs-fmt", gen_file], check=True) diff --git a/pkgs/applications/version-management/sapling/gen-deps.py b/pkgs/applications/version-management/sapling/gen-deps.py new file mode 100755 index 000000000000..e3fe56f73221 --- /dev/null +++ b/pkgs/applications/version-management/sapling/gen-deps.py @@ -0,0 +1,42 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" +import json +import re +from hashlib import sha1 +from struct import unpack +from subprocess import run + +from requests import get + +# Fetch the latest stable release metadata from GitHub +latestTag = get("https://api.github.com/repos/facebook/sapling/releases/latest").json()[ + "tag_name" +] + + +def nixPrefetchUrl(url): + return run( + ["nix-prefetch-url", "--type", "sha256", url], + check=True, + text=True, + capture_output=True, + ).stdout.rstrip() + + +# Fetch the `setup.py` source and look for instances of assets being downloaded +# from files.pythonhosted.org. +setupPy = get( + f"https://github.com/facebook/sapling/raw/{latestTag}/eden/scm/setup.py" +).text +foundUrls = re.findall(r'(https://files\.pythonhosted\.org/packages/[^\s]+)"', setupPy) + +dataDeps = { + "links": [{"url": url, "sha256": nixPrefetchUrl(url)} for url in foundUrls], + "version": latestTag, + # Find latest's git tag which corresponds to the Sapling version. Also + # needed is a hash of the version, so calculate that here. Taken from + # Sapling source `$root/eden/scm/setup_with_version.py`. + "versionHash": str(unpack(">Q", sha1(latestTag.encode("ascii")).digest()[:8])[0]), +} + +open("deps.json", "w").write(json.dumps(dataDeps, indent=2, sort_keys=True) + "\n")