Merge pull request #225503 from xworld21/texlive-binaries-from-tlpdb

texlive: generate bin containers from tlpdb

doc/languages-frameworks/texlive.section.md

@@ -22,7 +22,7 @@ Since release 15.09 there is a new TeX Live packaging that lives entirely under
   texlive.combine {
     # inherit (texlive) whatever-you-want;
     pkgFilter = pkg:
-      pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "cm-super";
+      pkg.tlType == "run" || pkg.tlType == "bin" || pkg.hasManpages || pkg.pname == "cm-super";
     # elem tlType [ "run" "bin" "doc" "source" ]
     # there are also other attributes: version, name
   }

pkgs/test/texlive/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, runCommand, fetchurl, file, texlive, writeShellScript, writeText }:
+{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText }:
 
 {
 
@@ -210,6 +210,16 @@
     mkdir "$out"
   '';
 
+  # verify that the restricted mode gets enabled when
+  # needed (detected by checking if it disallows --gscmd)
+  rpdfcrop = runCommand "texlive-test-rpdfcrop" {
+    nativeBuildInputs = [ (texlive.combine { inherit (texlive) scheme-infraonly pdfcrop; }) ];
+  } ''
+    ! (pdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null
+    (rpdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null
+    mkdir "$out"
+  '';
+
   # check that all binaries run successfully, in the following sense:
   # (1) run --version, -v, --help, -h successfully; or
   # (2) run --help, -h, or no argument with error code but show help text; or
@@ -218,58 +228,140 @@
   # compiled binaries or trivial shell wrappers
   binaries = let
       # TODO known broken binaries
-      broken = [ "albatross" "arara" "bbl2bib" "bib2gls" "bibdoiadd" "bibmradd" "bibzbladd" "citeproc" "convbkmk"
+      broken = [
-        "convertgls2bib" "ctan-o-mat" "ctanify" "ctanupload" "dtxgen" "ebong" "epspdftk" "exceltex" "gsx" "htcontext"
+        # *.inc files in source container rather than run
-        "installfont-tl" "kanji-fontmap-creator" "ketcindy" "latex-git-log" "latex2nemeth" "ltxfileinfo" "match_parens"
+        "texaccents"
-        "pdfannotextractor" "purifyeps" "pythontex" "svn-multi" "texexec" "texosquery" "texosquery-jre5"
+
-        "texosquery-jre8" "texplate" "tlcockpit" "tlmgr" "tlshell" "ulqda" "xhlatex" ];
+        # 'Error initialising QuantumRenderer: no suitable pipeline found'
+        "tlcockpit"
+
+        # 'tlmgr: config.guess script does not exist, goodbye'
+        "tlshell"
+      ] ++ lib.optional stdenv.isDarwin "epspdftk";  # wish shebang is a script, not a binary!
+
       # (1) binaries requiring -v
       shortVersion = [ "devnag" "diadia" "pmxchords" "ptex2pdf" "simpdftex" "ttf2afm" ];
       # (1) binaries requiring --help or -h
       help = [ "arlatex" "bundledoc" "cachepic" "checklistings" "dvipos" "extractres" "fig4latex" "fragmaster"
-        "kpsewhere" "mendex" "pn2pdf" "psbook" "psnup" "psresize" "simpdftex" "tex2xindy" "texluac" "texluajitc"
+        "kpsewhere" "latex-git-log" "ltxfileinfo" "mendex" "perltex" "pn2pdf" "psbook" "psnup" "psresize" "purifyeps"
-        "urlbst" "yplan" ];
+        "simpdftex" "tex2xindy" "texluac" "texluajitc" "urlbst" "yplan" ];
-      shortHelp = [ "adhocfilelist" "authorindex" "biburl2doi" "disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man"
+      shortHelp = [ "adhocfilelist" "authorindex" "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "ctanupload"
-        "lprsetup.sh" "pygmentex" ];
+        "disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man" "listings-ext.sh" "pygmentex" ];
       # (2) binaries that return non-zero exit code even if correctly asked for help
       ignoreExitCode = [ "authorindex" "dvibook" "dviconcat" "dvipos" "extractres" "fig4latex" "fragmaster" "latex2man"
-        "lprsetup.sh" "pdf2dsc" "psbook" "psnup" "psresize" "tex2xindy" "texluac" "texluajitc" ];
+        "latex-git-log" "listings-ext.sh" "psbook" "psnup" "psresize" "purifyeps" "tex2xindy"  "texluac"
+        "texluajitc" ];
       # (2) binaries that print help on no argument, returning non-zero exit code
       noArg = [ "a2ping" "bg5+latex" "bg5+pdflatex" "bg5latex" "bg5pdflatex" "cef5latex" "cef5pdflatex" "ceflatex"
-        "cefpdflatex" "cefslatex" "cefspdflatex" "chkdvifont" "dvi2fax" "dvipdf" "dvired" "dviselect"
+        "cefpdflatex" "cefslatex" "cefspdflatex" "chkdvifont" "dvi2fax" "dvired" "dviselect" "dvitodvi" "epsffit"
-        "dvitodvi" "eps2eps" "epsffit" "findhyph" "gbklatex" "gbkpdflatex" "komkindex" "kpsepath" "listbib"
+        "findhyph" "gbklatex" "gbkpdflatex" "komkindex" "kpsepath" "listbib" "listings-ext" "mag" "mathspic" "mf2pt1"
-        "listings-ext" "mag" "mathspic" "mf2pt1" "mk4ht" "mkt1font" "mkgrkindex" "musixflx" "pdf2ps" "pdftosrc"
+        "mk4ht" "mkt1font" "mkgrkindex" "musixflx" "pdf2ps" "pdftosrc" "pdfxup" "pedigree" "pfb2pfa" "pk2bm" "prepmx"
-        "pdfxup" "pedigree" "pfb2pfa" "pfbtopfa" "pk2bm" "pphs" "prepmx" "ps2pk" "ps2pdf*" "ps2ps*" "psselect" "pstops"
+        "ps2pk" "psselect" "pstops" "rubibtex" "rubikrotation" "sjislatex" "sjispdflatex" "srcredact" "t4ht" "tex4ht"
-        "rubibtex" "rubikrotation" "sjislatex" "sjispdflatex" "srcredact" "t4ht" "tex4ht" "texdiff" "texdirflatten"
+        "texdiff" "texdirflatten" "texplate" "tie" "ttf2kotexfont" "ttfdump" "vlna" "vpl2ovp" "vpl2vpl" "yplan" ];
-        "texplate" "tie" "ttf2kotexfont" "ttfdump" "vlna" "vpl2ovp" "vpl2vpl" "yplan" ];
+      # (3) binaries requiring a .tex file
-      # (3) binary requiring a .tex file
+      contextTest = [ "htcontext" ];
-      tex = [ "de-macro" "e2pall" "makeindex" "pslatex" "rumakeindex" "tpic2pdftex" "wordcount" ];
+      latexTest = [ "de-macro" "e2pall" "htlatex" "htxelatex" "makeindex" "pslatex" "rumakeindex" "tpic2pdftex"
+        "wordcount" "xhlatex" ];
+      texTest = [ "fontinst" "htmex" "httex" "httexi" "htxetex" ];
       # tricky binaries or scripts that are obviously working but are hard to test
       # (e.g. because they expect user input no matter the arguments)
       # (printafm comes from ghostscript, not texlive)
-      ignored = [ "dt2dv" "dv2dt" "dvi2tty" "dvidvi" "dvispc" "fontinst" "ht" "htlatex" "htmex" "httex" "httexi"
+      ignored = [
-        "htxelatex" "htxetex" "otp2ocp" "outocp" "pmxab" "printafm" ];
+        # compiled binaries
-      testTex = writeText "test.tex" ''
+        "dt2dv" "dv2dt" "dvi2tty" "dvidvi" "dvispc" "otp2ocp" "outocp" "pmxab"
+
+        # GUI scripts that accept no argument or crash without a graphics server; please test manualy
+        "epspdftk" "texdoctk" "xasy"
+
+        # requires Cinderella, not open source and not distributed via Nixpkgs
+        "ketcindy"
+      ];
+      # binaries that need a combined scheme and cannot work standalone
+      needScheme = [
+        # pfarrei: require working kpse to find lua module
+        "a5toa4"
+
+        # bibexport: requires kpsewhich
+        "bibexport"
+
+        # crossrefware: require bibtexperllibs under TEXMFROOT
+        "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "checkcites" "ltx2crossrefxml"
+
+        # require other texlive binaries in PATH
+        "allcm" "allec" "chkweb" "fontinst" "ht*" "installfont-tl" "kanji-config-updmap-sys" "kanji-config-updmap-user"
+        "kpse*" "latexfileversion" "mkocp" "mkofm" "mtxrunjit" "pdftex-quiet" "pslatex" "rumakeindex" "texconfig"
+        "texconfig-sys" "texexec" "texlinks" "texmfstart" "typeoutfileinfo" "wordcount" "xdvi" "xhlatex"
+
+        # misc luatex binaries searching for luatex in PATH
+        "citeproc-lua" "context" "contextjit" "ctanbib" "digestif" "epspdf" "l3build" "luafindfont" "luaotfload-tool"
+        "luatools" "make4ht" "pmxchords" "tex4ebook" "texdoc" "texlogsieve" "xindex"
+
+        # requires full TEXMFROOT (e.g. for config)
+        "mktexfmt" "mktexmf" "mktexpk" "mktextfm" "psnup" "psresize" "pstops" "tlmgr" "updmap" "webquiz"
+
+        # texlive-scripts: requires texlive.infra's TeXLive::TLUtils under TEXMFROOT
+        "fmtutil" "fmtutil-sys" "fmtutil-user"
+
+        # texlive-scripts: not used in nixpkgs, need updmap in PATH
+        "updmap-sys" "updmap-user"
+      ];
+
+      # simple test files
+      contextTestTex = writeText "context-test.tex" ''
+        \starttext
+          A simple test file.
+        \stoptext
+      '';
+      latexTestTex = writeText "latex-test.tex" ''
         \documentclass{article}
         \begin{document}
           A simple test file.
         \end{document}
       '';
+      texTestTex = writeText "tex-test.tex" ''
+        Hello.
+        \bye
+      '';
+
+      # link all binaries in single derivation
+      allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive)));
+      binPackages = lib.filter (p: p.tlType == "bin") allPackages;
+      binaries = buildEnv { name = "texlive-binaries"; paths = binPackages; };
     in
-    runCommand "texlive-test-binaries" { inherit testTex; }
+    runCommand "texlive-test-binaries"
+      {
+        inherit binaries contextTestTex latexTestTex texTestTex;
+        texliveScheme = texlive.combined.scheme-full;
+      }
       ''
+        loadables="$(command -v bash)"
+        loadables="''${loadables%/bin/bash}/lib/bash"
+        enable -f "$loadables/realpath" realpath
         mkdir -p "$out"
         export HOME="$(mktemp -d)"
         declare -i binCount=0 ignoredCount=0 brokenCount=0 failedCount=0
-        cp "$testTex" test.tex
+        cp "$contextTestTex" context-test.tex
+        cp "$latexTestTex" latex-test.tex
+        cp "$texTestTex" tex-test.tex
 
         testBin () {
+          path="$(realpath "$bin")"
+          path="''${path##*/}"
           if [[ -z "$ignoreExitCode" ]] ; then
-            "$bin" $args >"$out/$base.log" 2>&1
+            PATH="$path" "$bin" $args >"$out/$base.log" 2>&1
-            return $?
-          else
-            "$bin" $args >"$out/$base.log" 2>&1
             ret=$?
+            if [[ $ret == 0 ]] && grep -i 'command not found' "$out/$base.log" >/dev/null ; then
+              echo "command not found when running '$base''${args:+ $args}'"
+              return 1
+            fi
+            return $ret
+          else
+            PATH="$path" "$bin" $args >"$out/$base.log" 2>&1
+            ret=$?
+            if [[ $ret == 0 ]] && grep -i 'command not found' "$out/$base.log" >/dev/null ; then
+              echo "command not found when running '$base''${args:+ $args}'"
+              return 1
+            fi
             if ! grep -Ei '(Example:|Options:|Syntax:|Usage:|improper command|SYNOPSIS)' "$out/$base.log" >/dev/null ; then
               echo "did not find usage info when running '$base''${args:+ $args}'"
               return $ret
@@ -277,7 +369,7 @@
           fi
         }
 
-        for bin in ${texlive.combined.scheme-full}/bin/* ; do
+        for bin in "$binaries"/bin/* ; do
           base="''${bin##*/}"
           args=
           ignoreExitCode=
@@ -295,10 +387,19 @@
               args=-h ;;
             ${lib.concatStringsSep "|" noArg})
               ;;
-            ${lib.concatStringsSep "|" tex})
+            ${lib.concatStringsSep "|" contextTest})
-              args=test.tex ;;
+              args=context-test.tex ;;
+            ${lib.concatStringsSep "|" latexTest})
+              args=latex-test.tex ;;
+            ${lib.concatStringsSep "|" texTest})
+              args=tex-test.tex ;;
             ${lib.concatStringsSep "|" shortVersion})
               args=-v ;;
+            ebong)
+              touch empty
+              args=empty ;;
+            ht)
+              args='latex latex-test.tex' ;;
             pdf2dsc)
               args='--help --help --help' ;;
             typeoutfileinfo)
@@ -312,16 +413,57 @@
               ignoreExitCode=1 ;;
           esac
 
+          case "$base" in
+            ${lib.concatStringsSep "|" needScheme})
+              bin="$texliveScheme/bin/$base"
+              if [[ ! -f "$bin" ]] ; then
+                ignoredCount=$((ignoredCount + 1))
+                continue
+              fi ;;
+          esac
+
           if testBin ; then : ; else # preserve exit code
             echo "failed '$base''${args:+ $args}' (exit code: $?)"
+            sed 's/^/  > /' < "$out/$base.log"
             failedCount=$((failedCount + 1))
           fi
         done
 
-        echo "tested $binCount binCount: $ignoredCount ignored, $brokenCount broken, $failedCount failed"
+        echo "tested $binCount binaries: $ignoredCount ignored, $brokenCount broken, $failedCount failed"
         [[ $failedCount = 0 ]]
       '';
 
+  # check that all scripts have a Nix shebang
+  shebangs = let
+      allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive)));
+      binPackages = lib.filter (p: p.tlType == "bin") allPackages;
+    in
+    runCommand "texlive-test-shebangs" { }
+      (''
+        echo "checking that all texlive scripts shebangs are in '$NIX_STORE'"
+        declare -i scriptCount=0 invalidCount=0
+      '' +
+      (lib.concatMapStrings
+        (pkg: ''
+          for bin in '${pkg.outPath}'/bin/* ; do
+            grep -I -q . "$bin" || continue  # ignore binary files
+            scriptCount=$((scriptCount + 1))
+            read -r cmdline < "$bin"
+            read -r interp <<< "$cmdline"
+            if [[ "$interp" != "#!$NIX_STORE"/* && "$interp" != "#! $NIX_STORE"/* ]] ; then
+              echo "error: non-nix shebang '$interp' in script '$bin'"
+              invalidCount=$((invalidCount + 1))
+            fi
+          done
+        '')
+        binPackages)
+      + ''
+        echo "checked $scriptCount scripts, found $invalidCount non-nix shebangs"
+        [[ $invalidCount -gt 0 ]] && exit 1
+        mkdir -p "$out"
+      ''
+      );
+
   # verify that the precomputed licensing information in default.nix
   # does indeed match the metadata of the individual packages.
   #

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -98,3 +98,34 @@ a message like
 
 Please make sure to follow the [CONTRIBUTING](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md)
 guidelines.
+
+## Reviewing the bin containers
+
+Most `tlType == "bin"` containers consist of links to scripts distributed in
+`$TEXMFDIST/scripts` with a number of patches applied within `default.nix`.
+
+At each upgrade, please run the tests `tests.texlive.shebangs` to verify that
+all shebangs have been patched and in case add the relevant interpreters, and
+use `tests.texlive.binaries` to check if basic execution of all binaries works.
+
+Please review manually all binaries in the `broken` and `ignored` lists of
+`tests.texlive.binaries` at least once for major TeX Live release.
+
+Since the tests cannot catch all runtime dependencies, you should grep the
+`$TEXMFDIST/scripts` folder for common cases, for instance (where `$scripts`
+points to the relevant folder of `scheme-full`):
+- Calls to `exec $interpreter`
+  ```
+  grep -IRS 'exec ' "$TEXMFDIST/scripts" | cut -d: -f2 | sort -u | less -S
+  ```
+- Calls to Ghostscripts (see `needsGhostscript` in `combine.nix`)
+  ```
+  grep -IR '\([^a-zA-Z]\|^\)gs\( \|$\|"\)' "$TEXMFDIST"/scripts
+  grep -IR 'rungs' "$TEXMFDIST"
+  ```
+
+As a general rule, if a runtime dependency as above is essential for the core
+functionality of the package, then it should be made available in the bin
+containers (by patching `PATH`), or in `texlive.combine` (as we do for
+Ghostscript). Non-essential runtime dependencies should be ignored if they
+increase the closure substantially.

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -5,7 +5,7 @@
 , perl, perlPackages, python3Packages, pkg-config
 , libpaper, graphite2, zziplib, harfbuzz, potrace, gmp, mpfr
 , brotli, cairo, pixman, xorg, clisp, biber, woff2, xxHash
-, makeWrapper, shortenPerlShebang, useFixedHashes
+, makeWrapper, shortenPerlShebang, useFixedHashes, asymptote
 }:
 
 # Useful resource covering build options:
@@ -387,38 +387,6 @@ dvipng = stdenv.mkDerivation {
   enableParallelBuilding = true;
 };
 
-
-latexindent = perlPackages.buildPerlPackage rec {
-  pname = "latexindent";
-  inherit (src) version;
-
-  src = assertFixedHash pname (lib.head (builtins.filter (p: p.tlType == "run") texlive.latexindent.pkgs));
-
-  outputs = [ "out" ];
-
-  nativeBuildInputs = lib.optional stdenv.isDarwin shortenPerlShebang;
-  propagatedBuildInputs = with perlPackages; [ FileHomeDir LogDispatch LogLog4perl UnicodeLineBreak YAMLTiny ];
-
-  postPatch = ''
-    substituteInPlace scripts/latexindent/LatexIndent/GetYamlSettings.pm \
-      --replace '$FindBin::RealBin/defaultSettings.yaml' ${src}/scripts/latexindent/defaultSettings.yaml
-  '';
-
-  # Dirty hack to apply perlFlags, but do no build
-  preConfigure = ''
-    touch Makefile.PL
-  '';
-  dontBuild = true;
-  installPhase = ''
-    install -D ./scripts/latexindent/latexindent.pl "$out"/bin/latexindent
-    mkdir -p "$out"/${perl.libPrefix}
-    cp -r ./scripts/latexindent/LatexIndent "$out"/${perl.libPrefix}/
-  '' + lib.optionalString stdenv.isDarwin ''
-    shortenPerlShebang "$out"/bin/latexindent
-  '';
-};
-
-
 pygmentex = python3Packages.buildPythonApplication rec {
   pname = "pygmentex";
   inherit (src) version;
@@ -456,27 +424,7 @@ pygmentex = python3Packages.buildPythonApplication rec {
   };
 };
 
-
+inherit asymptote;
-texlinks = stdenv.mkDerivation rec {
-  name = "texlinks";
-
-  src = assertFixedHash name (lib.head (builtins.filter (p: p.tlType == "run") texlive.texlive-scripts-extra.pkgs));
-
-  dontBuild = true;
-  doCheck = false;
-
-  installPhase = ''
-    runHook preInstall
-
-    # Patch texlinks.sh back to 2015 version;
-    # otherwise some bin/ links break, e.g. xe(la)tex.
-    patch --verbose -R scripts/texlive-extra/texlinks.sh < '${./texlinks.diff}'
-    install -Dm555 scripts/texlive-extra/texlinks.sh "$out"/bin/texlinks
-
-    runHook postInstall
-  '';
-};
-
 
 inherit biber;
 bibtexu = bibtex8;

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -1,39 +1,26 @@
 params: with params;
 # combine =
 args@{
-  pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core")
+  pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core"
+                    || pkg.hasManpages or false)
 , extraName ? "combined"
 , extraVersion ? ""
 , ...
 }:
 let
-  pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {
+  pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ];
-    # include a fake "core" package
-    core.pkgs = [
-      (bin.core.out // { pname = "core"; tlType = "bin"; })
-      (bin.core.doc // { pname = "core"; tlType = "doc"; })
-    ];
-  };
   pkgList = rec {
     combined = combinePkgs (lib.attrValues pkgSet);
     all = lib.filter pkgFilter combined;
     splitBin = builtins.partition (p: p.tlType == "bin") all;
-    bin = splitBin.right
+    bin = splitBin.right;
-      ++ lib.optional
-          (lib.any (p: p.tlType == "run" && p.pname == "pdfcrop") splitBin.wrong)
-          (lib.getBin ghostscript);
     nonbin = splitBin.wrong;
     tlpkg = lib.filter (pkg: pkg.tlType == "tlpkg") combined;
-
-    # extra interpreters needed for shebangs, based on 2015 schemes "medium" and "tetex"
-    # (omitted tk needed in pname == "epspdf", bin/epspdftk)
-    pkgNeedsPython = pkg: pkg.tlType == "run" && lib.elem pkg.pname
-      [ "de-macro" "pythontex" "dviasm" "texliveonfly" ];
-    pkgNeedsRuby = pkg: pkg.tlType == "run" && pkg.pname == "match-parens";
-    extraInputs =
-      lib.optional (lib.any pkgNeedsPython splitBin.wrong) python3
-      ++ lib.optional (lib.any pkgNeedsRuby splitBin.wrong) ruby;
   };
+  # list generated by inspecting `grep -IR '\([^a-zA-Z]\|^\)gs\( \|$\|"\)' "$TEXMFDIST"/scripts`
+  # and `grep -IR rungs "$TEXMFDIST"`
+  # and ignoring luatex, perl, and shell scripts (those must be patched using postFixup)
+  needsGhostscript = lib.any (p: lib.elem p.pname [ "context" "dvipdfmx" "latex-papersize" "lyluatex" ]) pkgList.bin;
 
   name = "texlive-${extraName}-${bin.texliveYear}${extraVersion}";
 
@@ -43,11 +30,11 @@ let
     # remove fake derivations (without 'outPath') to avoid undesired build dependencies
     paths = lib.catAttrs "outPath" pkgList.nonbin;
 
-    nativeBuildInputs = [ perl bin.core.out ];
+    nativeBuildInputs = [ (lib.last tl.texlive-scripts.pkgs) ];
 
     postBuild = # generate ls-R database
     ''
-      perl "$out/scripts/texlive/mktexlsr.pl" --sort "$out"
+      mktexlsr --sort "$out"
     '';
   }).overrideAttrs (_: { allowSubstitutes = true; });
 
@@ -94,11 +81,17 @@ in (buildEnv {
     "/share/texmf-var/scripts"
     "/share/texmf-var/tex/generic/config"
     "/share/texmf-var/web2c"
+    "/share/texmf-config"
     "/bin" # ensure these are writeable directories
   ];
 
-  nativeBuildInputs = [ makeWrapper libfaketime perl bin.texlinks ];
+  nativeBuildInputs = [
-  buildInputs = pkgList.extraInputs;
+    makeWrapper
+    libfaketime
+    (lib.last tl.texlive-scripts.pkgs) # fmtutil, mktexlsr, updmap
+    (lib.last tl.texlive-scripts-extra.pkgs) # texlinks
+    perl
+  ];
 
   passthru = {
     # This is set primarily to help find-tarballs.nix to do its job
@@ -107,13 +100,44 @@ in (buildEnv {
     fonts = "${texmfroot}/texmf-dist/fonts";
   };
 
-  postBuild = ''
+  postBuild =
+    # environment variables (note: only export the ones that are used in the wrappers)
+  ''
     TEXMFROOT="${texmfroot}"
     TEXMFDIST="${texmfdist}"
     export PATH="$out/bin:$PATH"
     TEXMFSYSCONFIG="$out/share/texmf-config"
     TEXMFSYSVAR="$out/share/texmf-var"
     export TEXMFCNF="$TEXMFSYSVAR/web2c"
+  '' +
+    # wrap executables with required env vars as early as possible
+    # 1. we want texlive.combine to use the wrapped binaries, to catch bugs
+    # 2. we do not want to wrap links generated by texlinks
+  ''
+    enable -f '${bash}/lib/bash/realpath' realpath
+    declare -i wrapCount=0
+    for link in "$out"/bin/*; do
+      target="$(realpath "$link")"
+      if [[ "''${target##*/}" != "''${link##*/}" ]] ; then
+        # detected alias with different basename, use immediate target of $link to preserve $0
+        # relevant for mktexfmt, repstopdf, ...
+        target="$(readlink "$link")"
+      fi
+
+      rm "$link"
+      makeWrapper "$target" "$link" \
+        --inherit-argv0 \
+        --prefix PATH : "${
+          # very common dependencies that are not detected by tests.texlive.binaries
+          lib.makeBinPath ([ coreutils gawk gnugrep gnused ] ++ lib.optional needsGhostscript ghostscript)}:$out/bin" \
+        --set-default TEXMFCNF "$TEXMFCNF" \
+        --set-default FONTCONFIG_FILE "${
+          # necessary for XeTeX to find the fonts distributed with texlive
+          makeFontsConf { fontDirectories = [ "${texmfroot}/texmf-dist/fonts" ]; }
+        }"
+      wrapCount=$((wrapCount + 1))
+    done
+    echo "wrapped $wrapCount binaries and scripts"
   '' +
     # patch texmf-dist  -> $TEXMFDIST
     # patch texmf-local -> $out/share/texmf-local
@@ -153,7 +177,7 @@ in (buildEnv {
   (let
     hyphens = lib.filter (p: p.hasHyphens or false && p.tlType == "run") pkgList.splitBin.wrong;
     hyphenPNames = map (p: p.pname) hyphens;
-    formats = lib.filter (p: p.hasFormats or false && p.tlType == "run") pkgList.splitBin.wrong;
+    formats = lib.filter (p: p ? formats && p.tlType == "run") pkgList.splitBin.wrong;
     formatPNames = map (p: p.pname) formats;
     # sed expression that prints the lines in /start/,/end/ except for /end/
     section = start: end: "/${start}/,/${end}/{ /${start}/p; /${end}/!p; };\n";
@@ -199,54 +223,11 @@ in (buildEnv {
     [[ -e "$TEXMFDIST"/web2c/fmtutil.cnf ]] && sed -E -f '${fmtutilSed}' "$TEXMFDIST"/web2c/fmtutil.cnf > "$TEXMFCNF"/fmtutil.cnf
 
     # make new files visible to kpathsea
-    perl "$TEXMFDIST"/scripts/texlive/mktexlsr.pl --sort "$TEXMFSYSVAR"
+    mktexlsr --sort "$TEXMFSYSVAR"
   '') +
-
+    # generate format links (reads fmtutil.cnf to know which ones) *after* the wrappers have been generated
-  # function to wrap created executables with required env vars
   ''
-    wrapBin() {
+    texlinks --quiet "$out/bin"
-    for link in "$out"/bin/*; do
-      [ -L "$link" -a -x "$link" ] || continue # if not link, assume OK
-      local target=$(readlink "$link")
-
-      # skip simple local symlinks; mktexfmt in particular
-      echo "$target" | grep / > /dev/null || continue;
-
-      echo -n "Wrapping '$link'"
-      rm "$link"
-      makeWrapper "$target" "$link" \
-        --prefix PATH : "${gnused}/bin:${gnugrep}/bin:${coreutils}/bin:$out/bin:${perl}/bin" \
-        --set-default TEXMFCNF "$TEXMFCNF" \
-        --set-default FONTCONFIG_FILE "${
-          # neccessary for XeTeX to find the fonts distributed with texlive
-          makeFontsConf { fontDirectories = [ "${texmfroot}/texmf-dist/fonts" ]; }
-        }"
-
-      # avoid using non-nix shebang in $target by calling interpreter
-      if [[ "$(head -c 2 "$target")" = "#!" ]]; then
-        local cmdline="$(head -n 1 "$target" | sed 's/^\#\! *//;s/ *$//')"
-        local relative=`basename "$cmdline" | sed 's/^env //' `
-        local newInterp=`echo "$relative" | cut -d\  -f1`
-        local params=`echo "$relative" | cut -d\  -f2- -s`
-        local newPath="$(type -P "$newInterp")"
-        if [[ -z "$newPath" ]]; then
-          echo " Warning: unknown shebang '$cmdline' in '$target'"
-          continue
-        fi
-        echo " and patching shebang '$cmdline'"
-        sed "s|^exec |exec $newPath $params |" -i "$link"
-
-      elif head -n 1 "$target" | grep -q 'exec perl'; then
-        # see #24343 for details of the problem
-        echo " and patching weird perl shebang"
-        sed "s|^exec |exec '${perl}/bin/perl' -w |" -i "$link"
-
-      else
-        sed 's|^exec |exec -a "$0" |' -i "$link"
-        echo
-      fi
-    done
-    }
   '' +
   # texlive postactions (see TeXLive::TLUtils::_do_postaction_script)
   (lib.concatMapStrings (pkg: ''
@@ -259,20 +240,8 @@ in (buildEnv {
     echo "postaction install script for ${pkg.pname}: ''${postInterp:+$postInterp }$postaction install $TEXMFROOT"
     $postInterp "$TEXMFROOT/$postaction" install "$TEXMFROOT"
   '') (lib.filter (pkg: pkg ? postactionScript) pkgList.tlpkg)) +
-  # texlive post-install actions
-  ''
-    ln -sf "$TEXMFDIST"/scripts/texlive/updmap.pl "$out"/bin/updmap
-    ln -sf "$TEXMFDIST"/scripts/texlive/fmtutil.pl "$out"/bin/fmtutil
-  '' +
-    # now hack to preserve "$0" for mktexfmt
-  ''
-    cp "$TEXMFDIST"/scripts/texlive/fmtutil.pl "$TEXMFSYSVAR"/scripts/mktexfmt
-    ln -sf "$TEXMFSYSVAR"/scripts/mktexfmt "$out"/bin/mktexfmt
-  '' +
     # generate formats
   ''
-    texlinks "$out/bin" && wrapBin
-
     # many formats still ignore SOURCE_DATE_EPOCH even when FORCE_SOURCE_DATE=1
     # libfaketime fixes non-determinism related to timestamps ignoring FORCE_SOURCE_DATE
     # we cannot fix further randomness caused by luatex; for further details, see
@@ -282,41 +251,20 @@ in (buildEnv {
     substitute "$TEXMFDIST"/scripts/texlive/fmtutil.pl fmtutil \
       --replace 'my $cmdline = "$eng -ini ' 'my $cmdline = "faketime -f '"'"'\@1980-01-01 00:00:00 x0.001'"'"' $eng -ini '
     FORCE_SOURCE_DATE=1 TZ= perl fmtutil --sys --all | grep '^fmtutil' # too verbose
-    #texlinks "$out/bin" && wrapBin # do we need to regenerate format links?
 
     # Disable unavailable map files
-    echo y | updmap --sys --syncwithtrees --force
+    echo y | updmap --sys --syncwithtrees --force 2>&1 | grep '^\(updmap\|  /\)'
     # Regenerate the map files (this is optional)
-    updmap --sys --force
+    updmap --sys --force 2>&1 | grep '^\(updmap\|  /\)'
 
     # sort entries to improve reproducibility
     [[ -f "$TEXMFSYSCONFIG"/web2c/updmap.cfg ]] && sort -o "$TEXMFSYSCONFIG"/web2c/updmap.cfg "$TEXMFSYSCONFIG"/web2c/updmap.cfg
 
-    perl "$TEXMFDIST"/scripts/texlive/mktexlsr.pl --sort "$TEXMFSYSCONFIG" "$TEXMFSYSVAR" # to make sure
+    mktexlsr --sort "$TEXMFSYSCONFIG" "$TEXMFSYSVAR" # to make sure (of what?)
   '' +
-    # install (wrappers for) scripts, based on a list from upstream texlive
+    # remove *-sys scripts since /nix/store is readonly
-  ''
-    source '${bin.core.out}/share/texmf-dist/scripts/texlive/scripts.lst'
-    for s in $texmf_scripts; do
-      [[ -x "$TEXMFDIST/scripts/$s" ]] || continue
-      tName="$(basename $s | sed 's/\.[a-z]\+$//')" # remove extension
-      [[ ! -e "$out/bin/$tName" ]] || continue
-      ln -sv "$(realpath "$TEXMFDIST/scripts/$s")" "$out/bin/$tName" # wrapped below
-    done
-  '' +
-    # A hacky way to provide repstopdf
-    #  * Copy is done to have a correct "$0" so that epstopdf enables the restricted mode
-    #  * ./bin/repstopdf needs to be a symlink to be processed by wrapBin
-  ''
-    if [[ -e "$out"/bin/epstopdf ]]; then
-      cp "$out"/bin/epstopdf "$TEXMFSYSVAR"/scripts/repstopdf
-      ln -s "$TEXMFSYSVAR"/scripts/repstopdf "$out"/bin/repstopdf
-    fi
-  '' +
-    # finish up the wrappers
   ''
     rm "$out"/bin/*-sys
-    wrapBin
   '' +
   # TODO: a context trigger https://www.preining.info/blog/2015/06/debian-tex-live-2015-the-new-layout/
     # http://wiki.contextgarden.net/ConTeXt_Standalone#Unix-like_platforms_.28Linux.2FMacOS_X.2FFreeBSD.2FSolaris.29
@@ -332,8 +280,7 @@ in (buildEnv {
         --replace 'uuid=osuuid(),' 'uuid="242be807-d17e-4792-8e39-aa93326fc871",'
       FORCE_SOURCE_DATE=1 TZ= faketime -f '@1980-01-01 00:00:00 x0.001' luatex --luaonly mtxrun.lua --generate
     fi
-  ''
+  '' +
-    + bin.cleanBrokenLinks +
   # Get rid of all log files. They are not needed, but take up space
   # and render the build unreproducible by their embedded timestamps
   # and other non-deterministic diagnostics.

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -4,8 +4,10 @@
 */
 { stdenv, lib, fetchurl, runCommand, writeText, buildEnv
 , callPackage, ghostscript_headless, harfbuzz
-, makeWrapper, python3, ruby, perl, gnused, gnugrep, coreutils
+, makeWrapper
-, libfaketime, makeFontsConf
+, python3, ruby, perl, tk, jdk, bash, snobol4
+, coreutils, findutils, gawk, getopt, gnugrep, gnumake, gnused, gzip, ncurses, zip
+, libfaketime, asymptote, makeFontsConf
 , useFixedHashes ? true
 , recurseIntoAttrs
 }:
@@ -22,7 +24,7 @@ let
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv lib makeWrapper writeText runCommand
-      stdenv python3 ruby perl gnused gnugrep coreutils libfaketime makeFontsConf;
+      stdenv perl libfaketime makeFontsConf bash tl coreutils gawk gnugrep gnused;
     ghostscript = ghostscript_headless;
   };
 
@@ -32,25 +34,341 @@ let
 
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    orig = removeAttrs tlpdb [ "00texlive.config" ];
+    # most format -> engine links are generated by texlinks according to fmtutil.cnf at combine time
+    # so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf)
+    removeFormatLinks = lib.mapAttrs (_: attrs:
+      if (attrs ? formats && attrs ? binfiles)
+      then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine) attrs.formats);
+               binNotFormats = lib.subtractLists formatLinks attrs.binfiles;
+           in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ]
+      else attrs);
 
-    overridden = lib.recursiveUpdate orig {
+    orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]);
-      # overrides of texlive.tlpdb
 
-      # only *.po for tlmgr
+    overridden = lib.recursiveUpdate orig rec {
-      texlive-msg-translations.hasTlpkg = false;
+      #### overrides of texlive.tlpdb
+
+      #### nonstandard script folders
+      context.scriptsFolder = "context/stubs/unix";
+      cyrillic-bin.scriptsFolder = "texlive-extra";
+      fontinst.scriptsFolder = "texlive-extra";
+      mptopdf.scriptsFolder = "context/perl";
+      pdftex.scriptsFolder = "simpdftex";
+      "texlive.infra".scriptsFolder = "texlive";
+      texlive-scripts.scriptsFolder = "texlive";
+      texlive-scripts-extra.scriptsFolder = "texlive-extra";
+      xetex.scriptsFolder = "texlive-extra";
+
+      #### interpreters not detected by looking at the script extensions
+      ctanbib.extraBuildInputs = [ bin.luatex ];
+      de-macro.extraBuildInputs = [ python3 ];
+      match_parens.extraBuildInputs = [ ruby ];
+      optexcount.extraBuildInputs = [ python3 ];
+      pdfbook2.extraBuildInputs = [ python3 ];
+      texlogsieve.extraBuildInputs = [ bin.luatex ];
+
+      #### perl packages
+      crossrefware.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ LWP URI ])) ];
+      ctan-o-mat.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ LWP LWPProtocolHttps ])) ];
+      ctanify.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileCopyRecursive ])) ];
+      ctanupload.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ HTMLFormatter WWWMechanize ])) ];
+      exceltex.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ SpreadsheetParseExcel ])) ];
+      latex-git-log.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ IPCSystemSimple ])) ];
+      latexindent.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileHomeDir LogDispatch LogLog4perl UnicodeLineBreak YAMLTiny ])) ];
+      pax.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileWhich ])) ];
+      ptex-fontmaps.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ Tk ])) ];
+      purifyeps.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileWhich ])) ];
+      svn-multi.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ TimeDate ])) ];
+      texdoctk.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ Tk ])) ];
+      ulqda.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ DigestSHA1 ])) ];
+
+      #### python packages
+      pythontex.extraBuildInputs = [ (python3.withPackages (ps: with ps; [ pygments ])) ];
+
+      #### other runtime PATH dependencies
+      a2ping.extraBuildInputs = [ ghostscript_headless ];
+      bibexport.extraBuildInputs = [ gnugrep ];
+      checklistings.extraBuildInputs = [ coreutils ];
+      cjk-gs-integrate.extraBuildInputs = [ ghostscript_headless ];
+      context.extraBuildInputs = [ coreutils ruby ];
+      cyrillic-bin.extraBuildInputs = [ coreutils gnused ];
+      dtxgen.extraBuildInputs = [ coreutils getopt gnumake zip ];
+      dviljk.extraBuildInputs = [ coreutils ];
+      epspdf.extraBuildInputs = [ ghostscript_headless ];
+      epstopdf.extraBuildInputs = [ ghostscript_headless ];
+      fragmaster.extraBuildInputs = [ ghostscript_headless ];
+      installfont.extraBuildInputs = [ coreutils getopt gnused ];
+      latexfileversion.extraBuildInputs = [ coreutils gnugrep gnused ];
+      listings-ext.extraBuildInputs = [ coreutils getopt ];
+      ltxfileinfo.extraBuildInputs = [ coreutils getopt gnused ];
+      ltximg.extraBuildInputs = [ ghostscript_headless ];
+      luaotfload.extraBuildInputs = [ ncurses ];
+      makeindex.extraBuildInputs = [ coreutils gnused ];
+      pagelayout.extraBuildInputs = [ gnused ncurses ];
+      pdfcrop.extraBuildInputs = [ ghostscript_headless ];
+      pdftex.extraBuildInputs = [ coreutils ghostscript_headless gnused ];
+      pdftex-quiet.extraBuildInputs = [ coreutils ];
+      pdfxup.extraBuildInputs = [ coreutils ghostscript_headless ];
+      pkfix-helper.extraBuildInputs = [ ghostscript_headless ];
+      ps2eps.extraBuildInputs = [ ghostscript_headless ];
+      pst2pdf.extraBuildInputs = [ ghostscript_headless ];
+      tex4ht.extraBuildInputs = [ ruby ];
+      texlive-scripts.extraBuildInputs = [ gnused ];
+      texlive-scripts-extra.extraBuildInputs = [ coreutils findutils ghostscript_headless gnused ];
+      thumbpdf.extraBuildInputs = [ ghostscript_headless ];
+      tpic2pdftex.extraBuildInputs = [ gawk ];
+      wordcount.extraBuildInputs = [ coreutils gnugrep ];
+      xdvi.extraBuildInputs = [ coreutils gnugrep ];
+      xindy.extraBuildInputs = [ gzip ];
+
+      #### adjustments to binaries
+      # TODO patch the scripts from bin.* directly in bin.* instead of here
+
+      # TODO we do not build binaries for the following packages (yet!)
+      biber-ms.binfiles = [];
+      xpdfopen.binfiles = [];
+
+      # mptopdf is a format link, but not generated by texlinks
+      # so we add it back to binfiles to generate it from mkPkgBin
+      mptopdf.binfiles = (orig.mptopdf.binfiles or []) ++ [ "mptopdf" ];
+
+      # mktexlsr distributed by texlive.infra has implicit dependencies (e.g. kpsestat)
+      # the perl one hidden in texlive-scripts is better behaved
+      "texlive.infra".binfiles = lib.remove "mktexlsr" orig."texlive.infra".binfiles;
+
+      # remove man, add mktexlsr
+      texlive-scripts.binfiles = (lib.remove "man" orig.texlive-scripts.binfiles) ++ [ "mktexlsr" ];
+
+      # upmendex is "TODO" in bin.nix
+      uptex.binfiles = lib.remove "upmendex" orig.uptex.binfiles;
+
+      # teckit_compile seems to be missing from bin.core{,-big}
+      # TODO find it!
+      xetex.binfiles = lib.remove "teckit_compile" orig.xetex.binfiles;
+
+      # xindy is broken on some platforms unfortunately
+      xindy.binfiles = if bin ? xindy
+        then lib.subtractLists [ "xindy.mem" "xindy.run" ] orig.xindy.binfiles
+        else [];
+
+      #### additional symlinks
+      cluttex.binlinks = {
+        cllualatex = "cluttex";
+        clxelatex = "cluttex";
+      };
+
+      epstopdf.binlinks.repstopdf = "epstopdf";
+      pdfcrop.binlinks.rpdfcrop = "pdfcrop";
+
+      ptex.binlinks = {
+        pdvitomp = bin.metapost + "/bin/pdvitomp";
+        pmpost = bin.metapost + "/bin/pmpost";
+        r-pmpost = bin.metapost + "/bin/r-pmpost";
+      };
+
+      texdef.binlinks = {
+        latexdef = "texdef";
+      };
+
+      texlive-scripts.binlinks = {
+        mktexfmt = "fmtutil";
+        texhash = "mktexlsr";
+      };
+
+      texlive-scripts-extra.binlinks = {
+        allec = "allcm";
+        kpsepath = "kpsetool";
+        kpsexpand = "kpsetool";
+      };
+
+      # metapost binaries are in bin.metapost instead of bin.core
+      uptex.binlinks = {
+        r-upmpost = bin.metapost + "/bin/r-upmpost";
+        updvitomp = bin.metapost + "/bin/updvitomp";
+        upmpost = bin.metapost + "/bin/upmpost";
+      };
+
+      #### add PATH dependencies without wrappers
+      # TODO deduplicate this code
+      a2ping.postFixup = ''
+        sed -i '6i$ENV{PATH}='"'"'${lib.makeBinPath a2ping.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/a2ping
+      '';
+
+      bibexport.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath bibexport.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/bibexport
+      '';
+
+      checklistings.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath checklistings.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/checklistings
+      '';
+
+      cjk-gs-integrate.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath cjk-gs-integrate.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/cjk-gs-integrate
+      '';
+
+      context.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath [ coreutils ]}''${PATH:+:$PATH}"' "$out"/bin/{contextjit,mtxrunjit}
+        sed -i '2iPATH="${lib.makeBinPath [ ruby ]}''${PATH:+:$PATH}"' "$out"/bin/texexec
+      '';
+
+      cyrillic-bin.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath cyrillic-bin.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/rumakeindex
+      '';
+
+      dtxgen.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath dtxgen.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/dtxgen
+      '';
+
+      dviljk.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath dviljk.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/dvihp
+      '';
+
+      epstopdf.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath epstopdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/epstopdf
+      '';
+
+      fragmaster.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath fragmaster.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/fragmaster
+      '';
+
+      installfont.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath installfont.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/installfont-tl
+      '';
+
+      latexfileversion.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath latexfileversion.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/latexfileversion
+      '';
+
+      listings-ext.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath listings-ext.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/listings-ext.sh
+      '';
+
+      ltxfileinfo.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath ltxfileinfo.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/ltxfileinfo
+      '';
+
+      ltximg.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath ltximg.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/ltximg
+      '';
+
+      luaotfload.postFixup = ''
+        sed -i '2ios.setenv("PATH","${lib.makeBinPath luaotfload.extraBuildInputs}" .. (os.getenv("PATH") and ":" .. os.getenv("PATH") or ""))' "$out"/bin/luaotfload-tool
+      '';
+
+      makeindex.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath makeindex.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/mkindex
+      '';
+
+      pagelayout.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath [ gnused ]}''${PATH:+:$PATH}"' "$out"/bin/pagelayoutapi
+        sed -i '2iPATH="${lib.makeBinPath [ ncurses ]}''${PATH:+:$PATH}"' "$out"/bin/textestvis
+      '';
+
+      pdfcrop.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pdfcrop.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pdfcrop
+      '';
+
+      pdftex.postFixup = ''
+        sed -i -e '2iPATH="${lib.makeBinPath [ coreutils gnused ]}''${PATH:+:$PATH}"' \
+          -e 's!^distillerpath="/usr/local/bin"$!distillerpath="${lib.makeBinPath [ ghostscript_headless ]}"!' \
+          "$out"/bin/simpdftex
+      '';
+
+      pdftex-quiet.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath pdftex-quiet.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/pdftex-quiet
+      '';
+
+      pdfxup.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath pdfxup.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/pdfxup
+      '';
+
+      pkfix-helper.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pkfix-helper.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pkfix-helper
+      '';
+
+      ps2eps.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath ps2eps.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/ps2eps
+      '';
+
+      pst2pdf.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pst2pdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pst2pdf
+      '';
+
+      tex4ht.postFixup = ''
+        sed -i -e '2iPATH="${lib.makeBinPath tex4ht.extraBuildInputs}''${PATH:+:$PATH}"' -e 's/\\rubyCall//g;' "$out"/bin/htcontext
+      '';
+
+      texlive-scripts.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath texlive-scripts.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/{fmtutil-user,mktexmf,mktexpk,mktextfm,updmap-user}
+      '';
+
+      thumbpdf.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath thumbpdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/thumbpdf
+      '';
+
+      tpic2pdftex.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath tpic2pdftex.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/tpic2pdftex
+      '';
+
+      wordcount.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath wordcount.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/wordcount
+      '';
+
+      # TODO patch in bin.xdvi
+      xdvi.postFixup = ''
+        sed -i '2iPATH="${lib.makeBinPath xdvi.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/xdvi
+      '';
+
+      xindy.postFixup = ''
+        sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath xindy.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/{texindy,xindy}
+      '';
+
+      #### other script fixes
+      # misc tab and python3 fixes
+      ebong.postFixup = ''
+        sed -Ei 's/import sre/import re/; s/file\(/open(/g; s/\t/        /g; s/print +(.*)$/print(\1)/g' "$out"/bin/ebong
+      '';
+
+      # find files in script directory, not binary directory
+      # add runtime dependencies to PATH
+      epspdf.postFixup = ''
+        sed -i '2ios.setenv("PATH","${lib.makeBinPath epspdf.extraBuildInputs}" .. (os.getenv("PATH") and ":" .. os.getenv("PATH") or ""))' "$out"/bin/epspdf
+        substituteInPlace "$out"/bin/epspdftk --replace '[info script]' "\"$scriptsFolder/epspdftk.tcl\""
+      '';
+
+      # find files in script directory, not in binary directory
+      latexindent.postFixup = ''
+        substituteInPlace "$out"/bin/latexindent --replace 'use FindBin;' "BEGIN { \$0 = '$scriptsFolder' . '/latexindent.pl'; }; use FindBin;"
+      '';
+
+      # make tlmgr believe it can use kpsewhich to evaluate TEXMFROOT
+      "texlive.infra".postFixup = ''
+        substituteInPlace "$out"/bin/tlmgr \
+          --replace 'if (-r "$bindir/$kpsewhichname")' 'if (1)'
+      '';
+
+      # Patch texlinks.sh back to 2015 version;
+      # otherwise some bin/ links break, e.g. xe(la)tex.
+      # add runtime dependencies to PATH
+      texlive-scripts-extra.postFixup = ''
+        patch -R "$out"/bin/texlinks < '${./texlinks.diff}'
+        sed -i '2iPATH="${lib.makeBinPath [ coreutils ]}''${PATH:+:$PATH}"' "$out"/bin/{allcm,dvired,mkocp,ps2frag}
+        sed -i '2iPATH="${lib.makeBinPath [ coreutils findutils ]}''${PATH:+:$PATH}"' "$out"/bin/allneeded
+        sed -i '2iPATH="${lib.makeBinPath [ coreutils ghostscript_headless ]}''${PATH:+:$PATH}"' "$out"/bin/dvi2fax
+        sed -i '2iPATH="${lib.makeBinPath [ gnused ]}''${PATH:+:$PATH}"' "$out"/bin/{kpsetool,texconfig,texconfig-sys}
+        sed -i '2iPATH="${lib.makeBinPath [ coreutils gnused ]}''${PATH:+:$PATH}"' "$out"/bin/texconfig-dialog
+      '';
+
+      # patch interpreter
+      texosquery.postFixup = ''
+        substituteInPlace "$out"/bin/* --replace java "$interpJava"
+      '';
+
+      #### dependency changes
 
       # it seems to need it to transform fonts
       xdvi.deps = (orig.xdvi.deps or []) ++  [ "metafont" ];
 
-      # TODO: remove when updating to texlive-2023, metadata has been corrected in the TeX catalogue
-      # tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README
-      arabi-add.license = [  "lppl13c" ];
-
-      # TODO: remove this when updating to texlive-2023, npp-for-context is no longer in texlive
-      # tlpdb lists license as "noinfo", but it's gpl3: https://github.com/luigiScarso/context-npp
-      npp-for-context.license = [  "gpl3Only" ];
-
       # remove dependency-heavy packages from the basic collections
       collection-basic.deps = lib.subtractLists [ "metafont" "xdvi" ] orig.collection-basic.deps;
 
@@ -58,6 +376,15 @@ let
       collection-metapost.deps = orig.collection-metapost.deps ++ [ "metafont" ];
       collection-plaingeneric.deps = orig.collection-plaingeneric.deps ++ [ "xdvi" ];
 
+      #### misc
+
+      # tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README
+      arabi-add.license = [  "lppl13c" ];
+
+      # TODO: remove this when updating to texlive-2023, npp-for-context is no longer in texlive
+      # tlpdb lists license as "noinfo", but it's gpl3: https://github.com/luigiScarso/context-npp
+      npp-for-context.license = [  "gpl3Only" ];
+
       texdoc = {
         extraRevision = ".tlpdb${toString tlpdbVersion.revision}";
         extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";
@@ -91,27 +418,27 @@ let
         pkg = attrs // {
           sha512 = attrs.sha512.${if tlType == "tlpkg" then "run" else tlType};
           inherit pname tlType version;
+        } // lib.optionalAttrs (attrs.hasManpages or false) {
+          hasManpages = true;
         };
         in mkPkg pkg;
+      run = if (attrs.hasRunfiles or false) then mkPkgV "run"
+            # the fake derivations are used for filtering of hyphenation patterns and formats
+          else ({
+            inherit pname version;
+            tlType = "run";
+            hasHyphens = attrs.hasHyphens or false;
+            tlDeps = map (n: tl.${n}) (attrs.deps or []);
+          } // lib.optionalAttrs (attrs ? formats) { inherit (attrs) formats; });
     in {
       # TL pkg contains lists of packages: runtime files, docs, sources, tlpkg, binaries
       pkgs =
         # tarball of a collection/scheme itself only contains a tlobj file
-        [( if (attrs.hasRunfiles or false) then mkPkgV "run"
+        [ run ]
-            # the fake derivations are used for filtering of hyphenation patterns and formats
-          else {
-            inherit pname version;
-            tlType = "run";
-            hasFormats = attrs.hasFormats or false;
-            hasHyphens = attrs.hasHyphens or false;
-            tlDeps = map (n: tl.${n}) (attrs.deps or []);
-          }
-        )]
         ++ lib.optional (attrs.sha512 ? doc) (mkPkgV "doc")
         ++ lib.optional (attrs.sha512 ? source) (mkPkgV "source")
         ++ lib.optional (attrs.hasTlpkg or false) (mkPkgV "tlpkg")
-        ++ lib.optional (bin ? ${pname})
+        ++ lib.optional (attrs ? binfiles && attrs.binfiles != []) (mkPkgBin pname version run attrs);
-            ( bin.${pname} // { tlType = "bin"; } );
     };
 
   version = {
@@ -167,8 +494,46 @@ let
   # name + version for the derivation
   mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
 
+  # build tlType == "bin" containers based on `binfiles` in TLPDB
+  # see UPGRADING.md for how to keep the list of shebangs up to date
+  mkPkgBin = let extToInput = {
+      jar = jdk;
+      lua = bin.luatex;
+      py = python3;
+      rb = ruby;
+      sno = snobol4;
+      tcl = tk;
+      texlua = bin.luatex;
+      tlu = bin.luatex;
+    }; in pname: version: run:
+    { binfiles, scriptsFolder ? pname, postFixup ? "", scriptExts ? [], extraBuildInputs ? [], binlinks ? {}, ... }@args:
+    runCommand "texlive-${pname}.bin-${version}"
+      {
+        # metadata for texlive.combine
+        passthru = {
+          inherit pname version;
+          tlType = "bin";
+        };
+        # shebang interpreters
+        buildInputs = extraBuildInputs ++ [ bash perl ] ++ (lib.attrVals scriptExts extToInput);
+        # absolute scripts folder
+        scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + scriptsFolder);
+        # binaries info
+        inherit binfiles;
+        binlinks = builtins.attrNames binlinks;
+        bintargets = builtins.attrValues binlinks;
+        binfolders = [ (lib.getBin bin.core) ] ++ lib.optional (bin ? ${pname}) (lib.getBin bin.${pname});
+        # build scripts
+        patchScripts = ./patch-scripts.sed;
+        makeBinContainers = ./make-bin-containers.sh;
+      }
+      ''
+        . "$makeBinContainers"
+        ${postFixup}
+      '';
+
   # create a derivation that contains an unpacked upstream TL package
-  mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, ... }@args:
+  mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, hasManpages ? false, ... }@args:
     let
       # the basename used by upstream (without ".tar.xz" suffix)
       urlName = mkURLName args;
@@ -191,11 +556,12 @@ let
           } // lib.optionalAttrs (tlType == "run" && args ? deps) {
             tlDeps = map (n: tl.${n}) args.deps;
           } // lib.optionalAttrs (tlType == "run") {
-            hasFormats = args.hasFormats or false;
             hasHyphens = args.hasHyphens or false;
           } // lib.optionalAttrs (tlType == "tlpkg" && args ? postactionScript) {
             postactionScript = args.postactionScript;
-          };
+          }
+          // lib.optionalAttrs (tlType == "run" && args ? formats) { inherit (args) formats; }
+          // lib.optionalAttrs (tlType == "doc" && hasManpages) { hasManpages = true; };
         } // lib.optionalAttrs (fixedHash != null) {
           outputHash = fixedHash;
           outputHashAlgo = "sha256";
@@ -250,7 +616,10 @@ in
       xz = tlpdbxz;
     };
 
-    bin = assert assertions; bin;
+    bin = assert assertions; bin // {
+      # for backward compatibility
+      latexindent = lib.findFirst (p: p.tlType == "bin") tl.latexindent.pkgs;
+    };
     combine = assert assertions; combine;
 
     # Pre-defined combined packages for TeX Live schemes,

pkgs/tools/typesetting/tex/texlive/fixed-hashes.nix

@@ -8629,6 +8629,7 @@
 "texlive-fr.doc.r63071"="120jahzjmak3shjhiy81gv6nk3c1hv4rrxyi1mmzi1xklxjlhl4b";
 "texlive-it.doc.r58653"="0vmwn6n8bxpzcfrzic5qg5k2vklbm6rhl9861zxsli0rd9396qn6";
 "texlive-ja.doc.r62817"="1h2rv13ip3bgmfz8q64abqx3ajla9mc02a50xz1nzmng0rmgz04r";
+"texlive-msg-translations.tlpkg.r65889"="03bshv5xacw01ssbpd5wmz4ryym3dc27l8mwyh652i8gd2lg0scm";
 "texlive-pl.doc.r62841"="19qab4nd9z87v9dpx7gh2a87rw8k4x6kqzg5yc7wqmf46pfjmpcw";
 "texlive-ru.doc.r58426"="0c77yyfj87fajran8jppj0x8krk6b5195iscpn8z2n94spz2fcc9";
 "texlive-scripts.r66570"="0hl1vjr4hr7q7s2pvizicmabb185df5rl84cjsz0ki7vai5mh7pq";

pkgs/tools/typesetting/tex/texlive/make-bin-containers.sh

@@ -0,0 +1,62 @@
+# load realpath
+loadables="$(command -v bash)"
+loadables="${loadables%/bin/bash}/lib/bash"
+enable -f "$loadables/realpath" realpath
+mkdir -p "$out/bin"
+
+# find interpreters
+export interpPerl="$(PATH="$HOST_PATH" command -v perl)"
+export interpJava="$(PATH="$HOST_PATH" command -v java || :)"
+export interpWish="$(PATH="$HOST_PATH" command -v wish || :)"
+
+# prepare sed script
+substituteAll "$patchScripts" patch-scripts.sed
+
+for binname in $binfiles ; do
+  # binlinks to be created last, after the other binaries are in place
+  if [[ " $binlinks " == *" $binname "* ]] ; then
+    continue
+  fi
+
+  output="$out/bin/$binname"
+
+  # look for existing binary from bin.core or bin.${pname}
+  for folder in $binfolders ; do
+    target="$folder"/bin/"$binname"
+    if [[ -f "$target" && -x "$target" ]] ; then
+      ln -s "$(realpath "$target")" "$output"
+      continue 2
+    fi
+  done
+
+  # look for scripts
+  # the explicit list of extensions avoid non-scripts such as $binname.cmd, $binname.jar, $binname.pm
+  # the order is relevant: $binname.sh is preferred to other $binname.*
+  if [[ -n "$scriptsFolder" ]] ; then
+    for script in "$scriptsFolder/$binname"{,.sh,.lua,.pl,.py,.rb,.sno,.tcl,.texlua,.tlu}; do
+      if [[ -f "$script" ]] ; then
+        sed -f patch-scripts.sed \
+          -e 's/^scriptname=`basename "\$0"`$/'"scriptname='$(basename "$binname")'/" \
+          -e 's/^scriptname=`basename "\$0" .sh`$'"/scriptname='$(basename "$binname" .sh)'/" \
+          "$script" > "$output"
+        chmod +x "$output"
+        continue 2
+      fi
+    done
+  fi
+
+  echo "error: could not find source for 'bin/$binname'" >&2
+  exit 1
+done
+
+# patch shebangs
+patchShebangs "$out/bin"
+
+# generate links
+# we canonicalise the source to avoid symlink chains, and to check that it exists
+cd "$out"/bin
+for alias in $binlinks ; do
+  target="${bintargets%% *}"
+  bintargets="${bintargets#* }"
+  ln -s "$(realpath "$target")" "$out/bin/$alias"
+done

pkgs/tools/typesetting/tex/texlive/patch-scripts.sed

@@ -0,0 +1,57 @@
+1{
+  /python/{
+    N;
+    # add script folder to path, unless we interfere with a docstring
+    /\nr"""/b skip-python-path-patch
+    s!\n!\nimport sys; sys.path.insert(0,'@scriptsFolder@')\n!
+    :skip-python-path-patch
+  }
+
+  /^#!.*perl/{
+    # add script folder to @INC
+    s!$! -I@scriptsFolder@!
+  }
+
+  /^eval/{
+    # most likely the weird perl shebang
+    N
+    /^eval '(exit \$?0)' && eval 'exec perl -S \$0 \${1+"\$@"}' && eval 'exec perl -S \$0 \$argv:q'\n *if 0;$/{
+      x; s/.*/patching weird perl shebang/; w /dev/stderr
+      x; s|^.*$|#!@interpPerl@ -I@scriptsFolder@|
+    }
+  }
+}
+
+# patch 'exec interpreter'
+/exec java /{
+  x; s/.*/patching exec java/; w /dev/stderr
+  x; s|exec java |exec '@interpJava@' |g
+  /exec ''/{
+    x; s/^.*$/error: java missing from PATH/; w /dev/stderr
+    q 1
+  }
+}
+
+/exec perl /{
+  x; s/.*/patching exec perl/; w /dev/stderr
+  x; s|exec perl |exec @interpPerl@ -I@scriptsFolder@ |g
+  /exec ''/{
+    x; s/^.*$/error: perl missing from PATH/; w /dev/stderr
+    q 1
+  }
+}
+
+/exec wish /{
+  x; s/.*/patching exec wish/; w /dev/stderr
+  x; s|exec wish |exec '@interpWish@' |g
+  /exec ''/{
+    x; s/^.*$/error: wish missing from PATH/; w /dev/stderr
+    q 1
+  }
+}
+
+# make jar wrappers work without kpsewhich
+s!^jarpath=`kpsewhich --progname=[^ ]* --format=texmfscripts \([^ ]*\)`$!jarpath=@scriptsFolder@/\1!g
+
+# replace CYGWIN grep test with bash builtin
+s!echo "$kernel" | grep CYGWIN >/dev/null;![[ "$kernel" == *CYGWIN* ]]!g

pkgs/tools/typesetting/tex/texlive/tl2nix.sed

@@ -81,6 +81,20 @@ $a}
   }
 
   # detect presence of notable files
+  /^docfiles /{
+    s/^.*$//  # ignore the first line
+
+    # read all files
+    :next-doc
+      N
+      s/\n / /   # remove newline
+      t next-doc # loop if the previous lines matched
+
+    / (texmf-dist|RELOC)\/doc\/man\//i\  hasManpages = true;
+
+    D # restart cycle
+  }
+
   /^runfiles /{
     s/^.*$//  # ignore the first line
 
@@ -89,9 +103,25 @@ $a}
       N
       s/\n / /    # remove newline
       t next-file # loop if previous line matched
+    s/\n/ \n/     # add space before last newline for accurate matching below
 
     / (RELOC|texmf-dist)\//i\  hasRunfiles = true;
     / tlpkg\//i\  hasTlpkg = true;
+
+    # extract script extensions
+    / texmf-dist\/scripts\/.*\.(jar|lua|py|rb|sno|tcl|texlua|tlu) /{
+      i\  scriptExts = [
+        / texmf-dist\/scripts\/.*\.jar /i\    "jar"
+        / texmf-dist\/scripts\/.*\.lua /i\    "lua"
+        / texmf-dist\/scripts\/.*\.py /i\    "py"
+        / texmf-dist\/scripts\/.*\.rb /i\    "rb"
+        / texmf-dist\/scripts\/.*\.sno /i\    "sno"
+        / texmf-dist\/scripts\/.*\.tcl /i\    "tcl"
+        / texmf-dist\/scripts\/.*\.texlua /i\    "texlua"
+        / texmf-dist\/scripts\/.*\.tlu /i\    "tlu"
+      i\  ];
+    }
+
     D # restart cycle from the current line
   }
 
@@ -101,8 +131,50 @@ $a}
   # extract hyphenation patterns and formats
   # (this may create duplicate lines, use uniq to remove them)
   /^execute\sAddHyphen/i\  hasHyphens = true;
-  /^execute\sAddFormat/i\  hasFormats = true;
+
+  # extract format details
+  /^execute\sAddFormat\s/{
+    # open a list
+    i\  formats = [
+
+    # create one attribute set per format
+    # note that format names are not unique
+
+    # plain keys: name, engine, patterns
+    # optionally double quoted key: options
+    # boolean key: mode (enabled/disabled)
+    # comma-separated lists: fmttriggers, patterns
+    :next-fmt
+      s/(^|\n)execute\sAddFormat/    {/
+      s/\s+options="([^"]+)"/\n      options = "\1";/
+      s/\s+(name|engine|options)=([^ \t\n]+)/\n      \1 = "\2";/g
+      s/\s+mode=enabled//
+      s/\s+mode=disabled/\n      enabled = false;/
+      s/\s+(fmttriggers|patterns)=([^ \t\n]+)/\n      \1 = [ "\2" ];/g
+      s/$/\n    }/
+
+      :split-triggers
+        s/"([^,]+),([^"]+)" ]/"\1" "\2" ]/;
+        t split-triggers   # repeat until there are no commas
+
+      p
+      s/^.*$// # clear pattern space
+      N
+      /^\nexecute\sAddFormat\s/b next-fmt
+
+    # close the list
+    i\  ];
+    D # restart cycle from the current line
+  }
 
   # close attrmap
   /^$/i};
 }
+
+# add list of binaries from one of the architecture-specific packages
+/^name ([^.]+|texlive\.infra)\.x86_64-linux$/,/^$/{
+  s/^name ([0-9].*|texlive\.infra)\.x86_64-linux$/"\1".binfiles = [/p
+  s/^name (.*)\.x86_64-linux$/\1.binfiles = [/p
+  s!^ bin/x86_64-linux/(.+)$!  "\1"!p
+  /^$/i];
+}