Merge pull request #220896 from xworld21/texlive-upgrade-with-nix

texlive: generate "tlpdb.nix" from Nixpkgs attribute

pkgs/test/texlive/default.nix

@@ -2,6 +2,15 @@
 
 {
 
+  tlpdb-nix = runCommand "texlive-test-tlpdb-nix" {
+    nixpkgsTlpdbNix = ../../tools/typesetting/tex/texlive/tlpdb.nix;
+    tlpdbNix = texlive.tlpdb-nix;
+  }
+  ''
+    mkdir -p "$out"
+    diff -u "''${nixpkgsTlpdbNix}" "''${tlpdbNix}" | tee "$out/tlpdb.nix.patch"
+  '';
+
   luaotfload-fonts = runCommand "texlive-test-lualatex" {
     nativeBuildInputs = [
       (with texlive; combine { inherit scheme-medium libertinus-fonts; })

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -33,20 +33,31 @@ See https://tug.org/texlive/acquire-mirror.html for instructions.
 
 ### Upgrade package information from texlive package database
 
+First, edit `default.nix` as follows.
+
+If upgrading to a daily snapshot:
+- change `snapshot.year`, `snapshot.month`, `snapshot.day`;
+- ensure `urlPrefixes` uses the https://texlive.info/tlnet-archive mirror;
+- ensure `texlive.extraVersion` uses the `snapshot` info.
+
+If upgrading to a final release:
+- upgrade `texlive.bin` first;
+- ensure `urlPrefixes` uses the historic mirrors;
+- ensure `texlive.extraVersion` is `"-final"`.
+
+Then upgrade `tlpdb.hash` to match the new hash of `texlive.tlpdb.xz` and run
 
 ```bash
-curl -L https://texlive.info/tlnet-archive/$YEAR/$MONTH/$DAY/tlnet/tlpkg/texlive.tlpdb.xz \
-         | xzcat | sed -rn -f ./tl2nix.sed | uniq > ./pkgs.nix
+nix-build ../../../../.. -A texlive.tlpdb-nix --no-out-link
 ```
 
-This will download the daily snapshot of the CTAN package database `texlive.tlpdb.xz`
-and regenerate all of the sha512 hashes for the current upstream distribution in `pkgs.nix`.
+This will download the daily snapshot of the CTAN package database
+`texlive.tlpdb.xz` and extract the relevant package info (including version
+numbers and sha512 hashes) for the selected upstream distribution. Then replace
+`tlpdb.nix` with the generated file.
 
-Use the url
-
-https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/$YEAR/tlnet-final/tlpkg/texlive.tlpdb.xz
-
-for the final TeX Live release.
+The test `pkgs.tests.texlive.tlpdb-nix` verifies that the file `tlpdb.nix`
+in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.
 
 ### Build packages locally and generate fix hashes
 

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -30,7 +30,7 @@ let
 
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    orig = import ./pkgs.nix;
+    orig = import ./tlpdb.nix;
     removeSelfDep = lib.mapAttrs
       (n: p: if p ? deps then p // { deps = lib.filter (dn: n != dn) p.deps; }
                          else p);
@@ -126,16 +126,34 @@ let
     day = "27";
   };
 
+  # The tarballs on CTAN mirrors for the current release are constantly
+  # receiving updates, so we can't use those directly. Stable snapshots
+  # need to be used instead. Ideally, for the release branches of NixOS we
+  # should be switching to the tlnet-final versions
+  # (https://tug.org/historic/).
+  urlPrefixes = [
+    # tlnet-final snapshot
+    #"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive"
+    #"ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive"
+
+    # Daily snapshots hosted by one of the texlive release managers
+    "https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive"
+  ];
+
   tlpdb = fetchurl {
     # use the same mirror(s) as urlPrefixes below
-    urls = [
-      #"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/tlpkg/texlive.tlpdb.xz"
-      #"ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/tlpkg/texlive.tlpdb.xz"
-      "https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/tlpkg/texlive.tlpdb.xz"
-    ];
+    urls = map (up: "${up}/../tlpkg/texlive.tlpdb.xz") urlPrefixes;
     hash = "sha256-i8DE3/rZmtp+gODJWeHV1VcCK5cgHUgmywf3Q/agTOA=";
   };
 
+  tlpdb-nix = runCommand "tlpdb.nix" {
+    inherit tlpdb;
+    tl2nix = ./tl2nix.sed;
+  }
+  ''
+    xzcat "$tlpdb" | sed -rn -f "$tl2nix" | uniq > "$out"
+  '';
+
   # create a derivation that contains an unpacked upstream TL package
   mkPkg = { pname, tlType, revision, version, sha512, postUnpack ? "", stripPrefix ? 1, ... }@args:
     let
@@ -145,21 +163,7 @@ let
       fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
-        map (up: "${up}/${urlName}.r${toString revision}.tar.xz") urlPrefixes);
-
-      # The tarballs on CTAN mirrors for the current release are constantly
-      # receiving updates, so we can't use those directly. Stable snapshots
-      # need to be used instead. Ideally, for the release branches of NixOS we
-      # should be switching to the tlnet-final versions
-      # (https://tug.org/historic/).
-      urlPrefixes = args.urlPrefixes or [
-        # tlnet-final snapshot
-        #"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive"
-        #"ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive"
-
-        # Daily snapshots hosted by one of the texlive release managers
-        "https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive"
-      ];
+        map (up: "${up}/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
 
     in runCommand "texlive-${tlName}"
       ( {
@@ -191,7 +195,7 @@ let
 
 in
   tl // {
-    inherit bin combine;
+    inherit bin combine tlpdb-nix;
 
     # Pre-defined combined packages for TeX Live schemes,
     # to make nix-env usage more comfortable and build selected on Hydra.