Merge pull request #225503 from xworld21/texlive-binaries-from-tlpdb

texlive: generate bin containers from tlpdb
This commit is contained in:
Dmitry Kalinkin 2023-08-04 22:31:30 -04:00 committed by GitHub
commit aa725d5a5b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 2562 additions and 262 deletions

View file

@ -22,7 +22,7 @@ Since release 15.09 there is a new TeX Live packaging that lives entirely under
texlive.combine {
# inherit (texlive) whatever-you-want;
pkgFilter = pkg:
pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "cm-super";
pkg.tlType == "run" || pkg.tlType == "bin" || pkg.hasManpages || pkg.pname == "cm-super";
# elem tlType [ "run" "bin" "doc" "source" ]
# there are also other attributes: version, name
}

View file

@ -1,4 +1,4 @@
{ lib, stdenv, runCommand, fetchurl, file, texlive, writeShellScript, writeText }:
{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText }:
{
@ -210,6 +210,16 @@
mkdir "$out"
'';
# verify that the restricted mode gets enabled when
# needed (detected by checking if it disallows --gscmd)
rpdfcrop = runCommand "texlive-test-rpdfcrop" {
nativeBuildInputs = [ (texlive.combine { inherit (texlive) scheme-infraonly pdfcrop; }) ];
} ''
! (pdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null
(rpdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null
mkdir "$out"
'';
# check that all binaries run successfully, in the following sense:
# (1) run --version, -v, --help, -h successfully; or
# (2) run --help, -h, or no argument with error code but show help text; or
@ -218,58 +228,140 @@
# compiled binaries or trivial shell wrappers
binaries = let
# TODO known broken binaries
broken = [ "albatross" "arara" "bbl2bib" "bib2gls" "bibdoiadd" "bibmradd" "bibzbladd" "citeproc" "convbkmk"
"convertgls2bib" "ctan-o-mat" "ctanify" "ctanupload" "dtxgen" "ebong" "epspdftk" "exceltex" "gsx" "htcontext"
"installfont-tl" "kanji-fontmap-creator" "ketcindy" "latex-git-log" "latex2nemeth" "ltxfileinfo" "match_parens"
"pdfannotextractor" "purifyeps" "pythontex" "svn-multi" "texexec" "texosquery" "texosquery-jre5"
"texosquery-jre8" "texplate" "tlcockpit" "tlmgr" "tlshell" "ulqda" "xhlatex" ];
broken = [
# *.inc files in source container rather than run
"texaccents"
# 'Error initialising QuantumRenderer: no suitable pipeline found'
"tlcockpit"
# 'tlmgr: config.guess script does not exist, goodbye'
"tlshell"
] ++ lib.optional stdenv.isDarwin "epspdftk"; # wish shebang is a script, not a binary!
# (1) binaries requiring -v
shortVersion = [ "devnag" "diadia" "pmxchords" "ptex2pdf" "simpdftex" "ttf2afm" ];
# (1) binaries requiring --help or -h
help = [ "arlatex" "bundledoc" "cachepic" "checklistings" "dvipos" "extractres" "fig4latex" "fragmaster"
"kpsewhere" "mendex" "pn2pdf" "psbook" "psnup" "psresize" "simpdftex" "tex2xindy" "texluac" "texluajitc"
"urlbst" "yplan" ];
shortHelp = [ "adhocfilelist" "authorindex" "biburl2doi" "disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man"
"lprsetup.sh" "pygmentex" ];
"kpsewhere" "latex-git-log" "ltxfileinfo" "mendex" "perltex" "pn2pdf" "psbook" "psnup" "psresize" "purifyeps"
"simpdftex" "tex2xindy" "texluac" "texluajitc" "urlbst" "yplan" ];
shortHelp = [ "adhocfilelist" "authorindex" "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "ctanupload"
"disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man" "listings-ext.sh" "pygmentex" ];
# (2) binaries that return non-zero exit code even if correctly asked for help
ignoreExitCode = [ "authorindex" "dvibook" "dviconcat" "dvipos" "extractres" "fig4latex" "fragmaster" "latex2man"
"lprsetup.sh" "pdf2dsc" "psbook" "psnup" "psresize" "tex2xindy" "texluac" "texluajitc" ];
"latex-git-log" "listings-ext.sh" "psbook" "psnup" "psresize" "purifyeps" "tex2xindy" "texluac"
"texluajitc" ];
# (2) binaries that print help on no argument, returning non-zero exit code
noArg = [ "a2ping" "bg5+latex" "bg5+pdflatex" "bg5latex" "bg5pdflatex" "cef5latex" "cef5pdflatex" "ceflatex"
"cefpdflatex" "cefslatex" "cefspdflatex" "chkdvifont" "dvi2fax" "dvipdf" "dvired" "dviselect"
"dvitodvi" "eps2eps" "epsffit" "findhyph" "gbklatex" "gbkpdflatex" "komkindex" "kpsepath" "listbib"
"listings-ext" "mag" "mathspic" "mf2pt1" "mk4ht" "mkt1font" "mkgrkindex" "musixflx" "pdf2ps" "pdftosrc"
"pdfxup" "pedigree" "pfb2pfa" "pfbtopfa" "pk2bm" "pphs" "prepmx" "ps2pk" "ps2pdf*" "ps2ps*" "psselect" "pstops"
"rubibtex" "rubikrotation" "sjislatex" "sjispdflatex" "srcredact" "t4ht" "tex4ht" "texdiff" "texdirflatten"
"texplate" "tie" "ttf2kotexfont" "ttfdump" "vlna" "vpl2ovp" "vpl2vpl" "yplan" ];
# (3) binary requiring a .tex file
tex = [ "de-macro" "e2pall" "makeindex" "pslatex" "rumakeindex" "tpic2pdftex" "wordcount" ];
"cefpdflatex" "cefslatex" "cefspdflatex" "chkdvifont" "dvi2fax" "dvired" "dviselect" "dvitodvi" "epsffit"
"findhyph" "gbklatex" "gbkpdflatex" "komkindex" "kpsepath" "listbib" "listings-ext" "mag" "mathspic" "mf2pt1"
"mk4ht" "mkt1font" "mkgrkindex" "musixflx" "pdf2ps" "pdftosrc" "pdfxup" "pedigree" "pfb2pfa" "pk2bm" "prepmx"
"ps2pk" "psselect" "pstops" "rubibtex" "rubikrotation" "sjislatex" "sjispdflatex" "srcredact" "t4ht" "tex4ht"
"texdiff" "texdirflatten" "texplate" "tie" "ttf2kotexfont" "ttfdump" "vlna" "vpl2ovp" "vpl2vpl" "yplan" ];
# (3) binaries requiring a .tex file
contextTest = [ "htcontext" ];
latexTest = [ "de-macro" "e2pall" "htlatex" "htxelatex" "makeindex" "pslatex" "rumakeindex" "tpic2pdftex"
"wordcount" "xhlatex" ];
texTest = [ "fontinst" "htmex" "httex" "httexi" "htxetex" ];
# tricky binaries or scripts that are obviously working but are hard to test
# (e.g. because they expect user input no matter the arguments)
# (printafm comes from ghostscript, not texlive)
ignored = [ "dt2dv" "dv2dt" "dvi2tty" "dvidvi" "dvispc" "fontinst" "ht" "htlatex" "htmex" "httex" "httexi"
"htxelatex" "htxetex" "otp2ocp" "outocp" "pmxab" "printafm" ];
testTex = writeText "test.tex" ''
ignored = [
# compiled binaries
"dt2dv" "dv2dt" "dvi2tty" "dvidvi" "dvispc" "otp2ocp" "outocp" "pmxab"
# GUI scripts that accept no argument or crash without a graphics server; please test manualy
"epspdftk" "texdoctk" "xasy"
# requires Cinderella, not open source and not distributed via Nixpkgs
"ketcindy"
];
# binaries that need a combined scheme and cannot work standalone
needScheme = [
# pfarrei: require working kpse to find lua module
"a5toa4"
# bibexport: requires kpsewhich
"bibexport"
# crossrefware: require bibtexperllibs under TEXMFROOT
"bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "checkcites" "ltx2crossrefxml"
# require other texlive binaries in PATH
"allcm" "allec" "chkweb" "fontinst" "ht*" "installfont-tl" "kanji-config-updmap-sys" "kanji-config-updmap-user"
"kpse*" "latexfileversion" "mkocp" "mkofm" "mtxrunjit" "pdftex-quiet" "pslatex" "rumakeindex" "texconfig"
"texconfig-sys" "texexec" "texlinks" "texmfstart" "typeoutfileinfo" "wordcount" "xdvi" "xhlatex"
# misc luatex binaries searching for luatex in PATH
"citeproc-lua" "context" "contextjit" "ctanbib" "digestif" "epspdf" "l3build" "luafindfont" "luaotfload-tool"
"luatools" "make4ht" "pmxchords" "tex4ebook" "texdoc" "texlogsieve" "xindex"
# requires full TEXMFROOT (e.g. for config)
"mktexfmt" "mktexmf" "mktexpk" "mktextfm" "psnup" "psresize" "pstops" "tlmgr" "updmap" "webquiz"
# texlive-scripts: requires texlive.infra's TeXLive::TLUtils under TEXMFROOT
"fmtutil" "fmtutil-sys" "fmtutil-user"
# texlive-scripts: not used in nixpkgs, need updmap in PATH
"updmap-sys" "updmap-user"
];
# simple test files
contextTestTex = writeText "context-test.tex" ''
\starttext
A simple test file.
\stoptext
'';
latexTestTex = writeText "latex-test.tex" ''
\documentclass{article}
\begin{document}
A simple test file.
\end{document}
'';
texTestTex = writeText "tex-test.tex" ''
Hello.
\bye
'';
# link all binaries in single derivation
allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive)));
binPackages = lib.filter (p: p.tlType == "bin") allPackages;
binaries = buildEnv { name = "texlive-binaries"; paths = binPackages; };
in
runCommand "texlive-test-binaries" { inherit testTex; }
runCommand "texlive-test-binaries"
{
inherit binaries contextTestTex latexTestTex texTestTex;
texliveScheme = texlive.combined.scheme-full;
}
''
loadables="$(command -v bash)"
loadables="''${loadables%/bin/bash}/lib/bash"
enable -f "$loadables/realpath" realpath
mkdir -p "$out"
export HOME="$(mktemp -d)"
declare -i binCount=0 ignoredCount=0 brokenCount=0 failedCount=0
cp "$testTex" test.tex
cp "$contextTestTex" context-test.tex
cp "$latexTestTex" latex-test.tex
cp "$texTestTex" tex-test.tex
testBin () {
path="$(realpath "$bin")"
path="''${path##*/}"
if [[ -z "$ignoreExitCode" ]] ; then
"$bin" $args >"$out/$base.log" 2>&1
return $?
else
"$bin" $args >"$out/$base.log" 2>&1
PATH="$path" "$bin" $args >"$out/$base.log" 2>&1
ret=$?
if [[ $ret == 0 ]] && grep -i 'command not found' "$out/$base.log" >/dev/null ; then
echo "command not found when running '$base''${args:+ $args}'"
return 1
fi
return $ret
else
PATH="$path" "$bin" $args >"$out/$base.log" 2>&1
ret=$?
if [[ $ret == 0 ]] && grep -i 'command not found' "$out/$base.log" >/dev/null ; then
echo "command not found when running '$base''${args:+ $args}'"
return 1
fi
if ! grep -Ei '(Example:|Options:|Syntax:|Usage:|improper command|SYNOPSIS)' "$out/$base.log" >/dev/null ; then
echo "did not find usage info when running '$base''${args:+ $args}'"
return $ret
@ -277,7 +369,7 @@
fi
}
for bin in ${texlive.combined.scheme-full}/bin/* ; do
for bin in "$binaries"/bin/* ; do
base="''${bin##*/}"
args=
ignoreExitCode=
@ -295,10 +387,19 @@
args=-h ;;
${lib.concatStringsSep "|" noArg})
;;
${lib.concatStringsSep "|" tex})
args=test.tex ;;
${lib.concatStringsSep "|" contextTest})
args=context-test.tex ;;
${lib.concatStringsSep "|" latexTest})
args=latex-test.tex ;;
${lib.concatStringsSep "|" texTest})
args=tex-test.tex ;;
${lib.concatStringsSep "|" shortVersion})
args=-v ;;
ebong)
touch empty
args=empty ;;
ht)
args='latex latex-test.tex' ;;
pdf2dsc)
args='--help --help --help' ;;
typeoutfileinfo)
@ -312,16 +413,57 @@
ignoreExitCode=1 ;;
esac
case "$base" in
${lib.concatStringsSep "|" needScheme})
bin="$texliveScheme/bin/$base"
if [[ ! -f "$bin" ]] ; then
ignoredCount=$((ignoredCount + 1))
continue
fi ;;
esac
if testBin ; then : ; else # preserve exit code
echo "failed '$base''${args:+ $args}' (exit code: $?)"
sed 's/^/ > /' < "$out/$base.log"
failedCount=$((failedCount + 1))
fi
done
echo "tested $binCount binCount: $ignoredCount ignored, $brokenCount broken, $failedCount failed"
echo "tested $binCount binaries: $ignoredCount ignored, $brokenCount broken, $failedCount failed"
[[ $failedCount = 0 ]]
'';
# check that all scripts have a Nix shebang
shebangs = let
allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive)));
binPackages = lib.filter (p: p.tlType == "bin") allPackages;
in
runCommand "texlive-test-shebangs" { }
(''
echo "checking that all texlive scripts shebangs are in '$NIX_STORE'"
declare -i scriptCount=0 invalidCount=0
'' +
(lib.concatMapStrings
(pkg: ''
for bin in '${pkg.outPath}'/bin/* ; do
grep -I -q . "$bin" || continue # ignore binary files
scriptCount=$((scriptCount + 1))
read -r cmdline < "$bin"
read -r interp <<< "$cmdline"
if [[ "$interp" != "#!$NIX_STORE"/* && "$interp" != "#! $NIX_STORE"/* ]] ; then
echo "error: non-nix shebang '$interp' in script '$bin'"
invalidCount=$((invalidCount + 1))
fi
done
'')
binPackages)
+ ''
echo "checked $scriptCount scripts, found $invalidCount non-nix shebangs"
[[ $invalidCount -gt 0 ]] && exit 1
mkdir -p "$out"
''
);
# verify that the precomputed licensing information in default.nix
# does indeed match the metadata of the individual packages.
#

View file

@ -98,3 +98,34 @@ a message like
Please make sure to follow the [CONTRIBUTING](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md)
guidelines.
## Reviewing the bin containers
Most `tlType == "bin"` containers consist of links to scripts distributed in
`$TEXMFDIST/scripts` with a number of patches applied within `default.nix`.
At each upgrade, please run the tests `tests.texlive.shebangs` to verify that
all shebangs have been patched and in case add the relevant interpreters, and
use `tests.texlive.binaries` to check if basic execution of all binaries works.
Please review manually all binaries in the `broken` and `ignored` lists of
`tests.texlive.binaries` at least once for major TeX Live release.
Since the tests cannot catch all runtime dependencies, you should grep the
`$TEXMFDIST/scripts` folder for common cases, for instance (where `$scripts`
points to the relevant folder of `scheme-full`):
- Calls to `exec $interpreter`
```
grep -IRS 'exec ' "$TEXMFDIST/scripts" | cut -d: -f2 | sort -u | less -S
```
- Calls to Ghostscripts (see `needsGhostscript` in `combine.nix`)
```
grep -IR '\([^a-zA-Z]\|^\)gs\( \|$\|"\)' "$TEXMFDIST"/scripts
grep -IR 'rungs' "$TEXMFDIST"
```
As a general rule, if a runtime dependency as above is essential for the core
functionality of the package, then it should be made available in the bin
containers (by patching `PATH`), or in `texlive.combine` (as we do for
Ghostscript). Non-essential runtime dependencies should be ignored if they
increase the closure substantially.

View file

@ -5,7 +5,7 @@
, perl, perlPackages, python3Packages, pkg-config
, libpaper, graphite2, zziplib, harfbuzz, potrace, gmp, mpfr
, brotli, cairo, pixman, xorg, clisp, biber, woff2, xxHash
, makeWrapper, shortenPerlShebang, useFixedHashes
, makeWrapper, shortenPerlShebang, useFixedHashes, asymptote
}:
# Useful resource covering build options:
@ -387,38 +387,6 @@ dvipng = stdenv.mkDerivation {
enableParallelBuilding = true;
};
latexindent = perlPackages.buildPerlPackage rec {
pname = "latexindent";
inherit (src) version;
src = assertFixedHash pname (lib.head (builtins.filter (p: p.tlType == "run") texlive.latexindent.pkgs));
outputs = [ "out" ];
nativeBuildInputs = lib.optional stdenv.isDarwin shortenPerlShebang;
propagatedBuildInputs = with perlPackages; [ FileHomeDir LogDispatch LogLog4perl UnicodeLineBreak YAMLTiny ];
postPatch = ''
substituteInPlace scripts/latexindent/LatexIndent/GetYamlSettings.pm \
--replace '$FindBin::RealBin/defaultSettings.yaml' ${src}/scripts/latexindent/defaultSettings.yaml
'';
# Dirty hack to apply perlFlags, but do no build
preConfigure = ''
touch Makefile.PL
'';
dontBuild = true;
installPhase = ''
install -D ./scripts/latexindent/latexindent.pl "$out"/bin/latexindent
mkdir -p "$out"/${perl.libPrefix}
cp -r ./scripts/latexindent/LatexIndent "$out"/${perl.libPrefix}/
'' + lib.optionalString stdenv.isDarwin ''
shortenPerlShebang "$out"/bin/latexindent
'';
};
pygmentex = python3Packages.buildPythonApplication rec {
pname = "pygmentex";
inherit (src) version;
@ -456,27 +424,7 @@ pygmentex = python3Packages.buildPythonApplication rec {
};
};
texlinks = stdenv.mkDerivation rec {
name = "texlinks";
src = assertFixedHash name (lib.head (builtins.filter (p: p.tlType == "run") texlive.texlive-scripts-extra.pkgs));
dontBuild = true;
doCheck = false;
installPhase = ''
runHook preInstall
# Patch texlinks.sh back to 2015 version;
# otherwise some bin/ links break, e.g. xe(la)tex.
patch --verbose -R scripts/texlive-extra/texlinks.sh < '${./texlinks.diff}'
install -Dm555 scripts/texlive-extra/texlinks.sh "$out"/bin/texlinks
runHook postInstall
'';
};
inherit asymptote;
inherit biber;
bibtexu = bibtex8;

View file

@ -1,39 +1,26 @@
params: with params;
# combine =
args@{
pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core")
pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core"
|| pkg.hasManpages or false)
, extraName ? "combined"
, extraVersion ? ""
, ...
}:
let
pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {
# include a fake "core" package
core.pkgs = [
(bin.core.out // { pname = "core"; tlType = "bin"; })
(bin.core.doc // { pname = "core"; tlType = "doc"; })
];
};
pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ];
pkgList = rec {
combined = combinePkgs (lib.attrValues pkgSet);
all = lib.filter pkgFilter combined;
splitBin = builtins.partition (p: p.tlType == "bin") all;
bin = splitBin.right
++ lib.optional
(lib.any (p: p.tlType == "run" && p.pname == "pdfcrop") splitBin.wrong)
(lib.getBin ghostscript);
bin = splitBin.right;
nonbin = splitBin.wrong;
tlpkg = lib.filter (pkg: pkg.tlType == "tlpkg") combined;
# extra interpreters needed for shebangs, based on 2015 schemes "medium" and "tetex"
# (omitted tk needed in pname == "epspdf", bin/epspdftk)
pkgNeedsPython = pkg: pkg.tlType == "run" && lib.elem pkg.pname
[ "de-macro" "pythontex" "dviasm" "texliveonfly" ];
pkgNeedsRuby = pkg: pkg.tlType == "run" && pkg.pname == "match-parens";
extraInputs =
lib.optional (lib.any pkgNeedsPython splitBin.wrong) python3
++ lib.optional (lib.any pkgNeedsRuby splitBin.wrong) ruby;
};
# list generated by inspecting `grep -IR '\([^a-zA-Z]\|^\)gs\( \|$\|"\)' "$TEXMFDIST"/scripts`
# and `grep -IR rungs "$TEXMFDIST"`
# and ignoring luatex, perl, and shell scripts (those must be patched using postFixup)
needsGhostscript = lib.any (p: lib.elem p.pname [ "context" "dvipdfmx" "latex-papersize" "lyluatex" ]) pkgList.bin;
name = "texlive-${extraName}-${bin.texliveYear}${extraVersion}";
@ -43,11 +30,11 @@ let
# remove fake derivations (without 'outPath') to avoid undesired build dependencies
paths = lib.catAttrs "outPath" pkgList.nonbin;
nativeBuildInputs = [ perl bin.core.out ];
nativeBuildInputs = [ (lib.last tl.texlive-scripts.pkgs) ];
postBuild = # generate ls-R database
''
perl "$out/scripts/texlive/mktexlsr.pl" --sort "$out"
mktexlsr --sort "$out"
'';
}).overrideAttrs (_: { allowSubstitutes = true; });
@ -94,11 +81,17 @@ in (buildEnv {
"/share/texmf-var/scripts"
"/share/texmf-var/tex/generic/config"
"/share/texmf-var/web2c"
"/share/texmf-config"
"/bin" # ensure these are writeable directories
];
nativeBuildInputs = [ makeWrapper libfaketime perl bin.texlinks ];
buildInputs = pkgList.extraInputs;
nativeBuildInputs = [
makeWrapper
libfaketime
(lib.last tl.texlive-scripts.pkgs) # fmtutil, mktexlsr, updmap
(lib.last tl.texlive-scripts-extra.pkgs) # texlinks
perl
];
passthru = {
# This is set primarily to help find-tarballs.nix to do its job
@ -107,13 +100,44 @@ in (buildEnv {
fonts = "${texmfroot}/texmf-dist/fonts";
};
postBuild = ''
postBuild =
# environment variables (note: only export the ones that are used in the wrappers)
''
TEXMFROOT="${texmfroot}"
TEXMFDIST="${texmfdist}"
export PATH="$out/bin:$PATH"
TEXMFSYSCONFIG="$out/share/texmf-config"
TEXMFSYSVAR="$out/share/texmf-var"
export TEXMFCNF="$TEXMFSYSVAR/web2c"
'' +
# wrap executables with required env vars as early as possible
# 1. we want texlive.combine to use the wrapped binaries, to catch bugs
# 2. we do not want to wrap links generated by texlinks
''
enable -f '${bash}/lib/bash/realpath' realpath
declare -i wrapCount=0
for link in "$out"/bin/*; do
target="$(realpath "$link")"
if [[ "''${target##*/}" != "''${link##*/}" ]] ; then
# detected alias with different basename, use immediate target of $link to preserve $0
# relevant for mktexfmt, repstopdf, ...
target="$(readlink "$link")"
fi
rm "$link"
makeWrapper "$target" "$link" \
--inherit-argv0 \
--prefix PATH : "${
# very common dependencies that are not detected by tests.texlive.binaries
lib.makeBinPath ([ coreutils gawk gnugrep gnused ] ++ lib.optional needsGhostscript ghostscript)}:$out/bin" \
--set-default TEXMFCNF "$TEXMFCNF" \
--set-default FONTCONFIG_FILE "${
# necessary for XeTeX to find the fonts distributed with texlive
makeFontsConf { fontDirectories = [ "${texmfroot}/texmf-dist/fonts" ]; }
}"
wrapCount=$((wrapCount + 1))
done
echo "wrapped $wrapCount binaries and scripts"
'' +
# patch texmf-dist -> $TEXMFDIST
# patch texmf-local -> $out/share/texmf-local
@ -153,7 +177,7 @@ in (buildEnv {
(let
hyphens = lib.filter (p: p.hasHyphens or false && p.tlType == "run") pkgList.splitBin.wrong;
hyphenPNames = map (p: p.pname) hyphens;
formats = lib.filter (p: p.hasFormats or false && p.tlType == "run") pkgList.splitBin.wrong;
formats = lib.filter (p: p ? formats && p.tlType == "run") pkgList.splitBin.wrong;
formatPNames = map (p: p.pname) formats;
# sed expression that prints the lines in /start/,/end/ except for /end/
section = start: end: "/${start}/,/${end}/{ /${start}/p; /${end}/!p; };\n";
@ -199,54 +223,11 @@ in (buildEnv {
[[ -e "$TEXMFDIST"/web2c/fmtutil.cnf ]] && sed -E -f '${fmtutilSed}' "$TEXMFDIST"/web2c/fmtutil.cnf > "$TEXMFCNF"/fmtutil.cnf
# make new files visible to kpathsea
perl "$TEXMFDIST"/scripts/texlive/mktexlsr.pl --sort "$TEXMFSYSVAR"
mktexlsr --sort "$TEXMFSYSVAR"
'') +
# function to wrap created executables with required env vars
# generate format links (reads fmtutil.cnf to know which ones) *after* the wrappers have been generated
''
wrapBin() {
for link in "$out"/bin/*; do
[ -L "$link" -a -x "$link" ] || continue # if not link, assume OK
local target=$(readlink "$link")
# skip simple local symlinks; mktexfmt in particular
echo "$target" | grep / > /dev/null || continue;
echo -n "Wrapping '$link'"
rm "$link"
makeWrapper "$target" "$link" \
--prefix PATH : "${gnused}/bin:${gnugrep}/bin:${coreutils}/bin:$out/bin:${perl}/bin" \
--set-default TEXMFCNF "$TEXMFCNF" \
--set-default FONTCONFIG_FILE "${
# neccessary for XeTeX to find the fonts distributed with texlive
makeFontsConf { fontDirectories = [ "${texmfroot}/texmf-dist/fonts" ]; }
}"
# avoid using non-nix shebang in $target by calling interpreter
if [[ "$(head -c 2 "$target")" = "#!" ]]; then
local cmdline="$(head -n 1 "$target" | sed 's/^\#\! *//;s/ *$//')"
local relative=`basename "$cmdline" | sed 's/^env //' `
local newInterp=`echo "$relative" | cut -d\ -f1`
local params=`echo "$relative" | cut -d\ -f2- -s`
local newPath="$(type -P "$newInterp")"
if [[ -z "$newPath" ]]; then
echo " Warning: unknown shebang '$cmdline' in '$target'"
continue
fi
echo " and patching shebang '$cmdline'"
sed "s|^exec |exec $newPath $params |" -i "$link"
elif head -n 1 "$target" | grep -q 'exec perl'; then
# see #24343 for details of the problem
echo " and patching weird perl shebang"
sed "s|^exec |exec '${perl}/bin/perl' -w |" -i "$link"
else
sed 's|^exec |exec -a "$0" |' -i "$link"
echo
fi
done
}
texlinks --quiet "$out/bin"
'' +
# texlive postactions (see TeXLive::TLUtils::_do_postaction_script)
(lib.concatMapStrings (pkg: ''
@ -259,20 +240,8 @@ in (buildEnv {
echo "postaction install script for ${pkg.pname}: ''${postInterp:+$postInterp }$postaction install $TEXMFROOT"
$postInterp "$TEXMFROOT/$postaction" install "$TEXMFROOT"
'') (lib.filter (pkg: pkg ? postactionScript) pkgList.tlpkg)) +
# texlive post-install actions
''
ln -sf "$TEXMFDIST"/scripts/texlive/updmap.pl "$out"/bin/updmap
ln -sf "$TEXMFDIST"/scripts/texlive/fmtutil.pl "$out"/bin/fmtutil
'' +
# now hack to preserve "$0" for mktexfmt
''
cp "$TEXMFDIST"/scripts/texlive/fmtutil.pl "$TEXMFSYSVAR"/scripts/mktexfmt
ln -sf "$TEXMFSYSVAR"/scripts/mktexfmt "$out"/bin/mktexfmt
'' +
# generate formats
''
texlinks "$out/bin" && wrapBin
# many formats still ignore SOURCE_DATE_EPOCH even when FORCE_SOURCE_DATE=1
# libfaketime fixes non-determinism related to timestamps ignoring FORCE_SOURCE_DATE
# we cannot fix further randomness caused by luatex; for further details, see
@ -282,41 +251,20 @@ in (buildEnv {
substitute "$TEXMFDIST"/scripts/texlive/fmtutil.pl fmtutil \
--replace 'my $cmdline = "$eng -ini ' 'my $cmdline = "faketime -f '"'"'\@1980-01-01 00:00:00 x0.001'"'"' $eng -ini '
FORCE_SOURCE_DATE=1 TZ= perl fmtutil --sys --all | grep '^fmtutil' # too verbose
#texlinks "$out/bin" && wrapBin # do we need to regenerate format links?
# Disable unavailable map files
echo y | updmap --sys --syncwithtrees --force
echo y | updmap --sys --syncwithtrees --force 2>&1 | grep '^\(updmap\| /\)'
# Regenerate the map files (this is optional)
updmap --sys --force
updmap --sys --force 2>&1 | grep '^\(updmap\| /\)'
# sort entries to improve reproducibility
[[ -f "$TEXMFSYSCONFIG"/web2c/updmap.cfg ]] && sort -o "$TEXMFSYSCONFIG"/web2c/updmap.cfg "$TEXMFSYSCONFIG"/web2c/updmap.cfg
perl "$TEXMFDIST"/scripts/texlive/mktexlsr.pl --sort "$TEXMFSYSCONFIG" "$TEXMFSYSVAR" # to make sure
mktexlsr --sort "$TEXMFSYSCONFIG" "$TEXMFSYSVAR" # to make sure (of what?)
'' +
# install (wrappers for) scripts, based on a list from upstream texlive
''
source '${bin.core.out}/share/texmf-dist/scripts/texlive/scripts.lst'
for s in $texmf_scripts; do
[[ -x "$TEXMFDIST/scripts/$s" ]] || continue
tName="$(basename $s | sed 's/\.[a-z]\+$//')" # remove extension
[[ ! -e "$out/bin/$tName" ]] || continue
ln -sv "$(realpath "$TEXMFDIST/scripts/$s")" "$out/bin/$tName" # wrapped below
done
'' +
# A hacky way to provide repstopdf
# * Copy is done to have a correct "$0" so that epstopdf enables the restricted mode
# * ./bin/repstopdf needs to be a symlink to be processed by wrapBin
''
if [[ -e "$out"/bin/epstopdf ]]; then
cp "$out"/bin/epstopdf "$TEXMFSYSVAR"/scripts/repstopdf
ln -s "$TEXMFSYSVAR"/scripts/repstopdf "$out"/bin/repstopdf
fi
'' +
# finish up the wrappers
# remove *-sys scripts since /nix/store is readonly
''
rm "$out"/bin/*-sys
wrapBin
'' +
# TODO: a context trigger https://www.preining.info/blog/2015/06/debian-tex-live-2015-the-new-layout/
# http://wiki.contextgarden.net/ConTeXt_Standalone#Unix-like_platforms_.28Linux.2FMacOS_X.2FFreeBSD.2FSolaris.29
@ -332,8 +280,7 @@ in (buildEnv {
--replace 'uuid=osuuid(),' 'uuid="242be807-d17e-4792-8e39-aa93326fc871",'
FORCE_SOURCE_DATE=1 TZ= faketime -f '@1980-01-01 00:00:00 x0.001' luatex --luaonly mtxrun.lua --generate
fi
''
+ bin.cleanBrokenLinks +
'' +
# Get rid of all log files. They are not needed, but take up space
# and render the build unreproducible by their embedded timestamps
# and other non-deterministic diagnostics.

View file

@ -4,8 +4,10 @@
*/
{ stdenv, lib, fetchurl, runCommand, writeText, buildEnv
, callPackage, ghostscript_headless, harfbuzz
, makeWrapper, python3, ruby, perl, gnused, gnugrep, coreutils
, libfaketime, makeFontsConf
, makeWrapper
, python3, ruby, perl, tk, jdk, bash, snobol4
, coreutils, findutils, gawk, getopt, gnugrep, gnumake, gnused, gzip, ncurses, zip
, libfaketime, asymptote, makeFontsConf
, useFixedHashes ? true
, recurseIntoAttrs
}:
@ -22,7 +24,7 @@ let
# function for creating a working environment from a set of TL packages
combine = import ./combine.nix {
inherit bin combinePkgs buildEnv lib makeWrapper writeText runCommand
stdenv python3 ruby perl gnused gnugrep coreutils libfaketime makeFontsConf;
stdenv perl libfaketime makeFontsConf bash tl coreutils gawk gnugrep gnused;
ghostscript = ghostscript_headless;
};
@ -32,25 +34,341 @@ let
# the set of TeX Live packages, collections, and schemes; using upstream naming
tl = let
orig = removeAttrs tlpdb [ "00texlive.config" ];
# most format -> engine links are generated by texlinks according to fmtutil.cnf at combine time
# so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf)
removeFormatLinks = lib.mapAttrs (_: attrs:
if (attrs ? formats && attrs ? binfiles)
then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine) attrs.formats);
binNotFormats = lib.subtractLists formatLinks attrs.binfiles;
in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ]
else attrs);
overridden = lib.recursiveUpdate orig {
# overrides of texlive.tlpdb
orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]);
# only *.po for tlmgr
texlive-msg-translations.hasTlpkg = false;
overridden = lib.recursiveUpdate orig rec {
#### overrides of texlive.tlpdb
#### nonstandard script folders
context.scriptsFolder = "context/stubs/unix";
cyrillic-bin.scriptsFolder = "texlive-extra";
fontinst.scriptsFolder = "texlive-extra";
mptopdf.scriptsFolder = "context/perl";
pdftex.scriptsFolder = "simpdftex";
"texlive.infra".scriptsFolder = "texlive";
texlive-scripts.scriptsFolder = "texlive";
texlive-scripts-extra.scriptsFolder = "texlive-extra";
xetex.scriptsFolder = "texlive-extra";
#### interpreters not detected by looking at the script extensions
ctanbib.extraBuildInputs = [ bin.luatex ];
de-macro.extraBuildInputs = [ python3 ];
match_parens.extraBuildInputs = [ ruby ];
optexcount.extraBuildInputs = [ python3 ];
pdfbook2.extraBuildInputs = [ python3 ];
texlogsieve.extraBuildInputs = [ bin.luatex ];
#### perl packages
crossrefware.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ LWP URI ])) ];
ctan-o-mat.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ LWP LWPProtocolHttps ])) ];
ctanify.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileCopyRecursive ])) ];
ctanupload.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ HTMLFormatter WWWMechanize ])) ];
exceltex.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ SpreadsheetParseExcel ])) ];
latex-git-log.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ IPCSystemSimple ])) ];
latexindent.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileHomeDir LogDispatch LogLog4perl UnicodeLineBreak YAMLTiny ])) ];
pax.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileWhich ])) ];
ptex-fontmaps.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ Tk ])) ];
purifyeps.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ FileWhich ])) ];
svn-multi.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ TimeDate ])) ];
texdoctk.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ Tk ])) ];
ulqda.extraBuildInputs = [ (perl.withPackages (ps: with ps; [ DigestSHA1 ])) ];
#### python packages
pythontex.extraBuildInputs = [ (python3.withPackages (ps: with ps; [ pygments ])) ];
#### other runtime PATH dependencies
a2ping.extraBuildInputs = [ ghostscript_headless ];
bibexport.extraBuildInputs = [ gnugrep ];
checklistings.extraBuildInputs = [ coreutils ];
cjk-gs-integrate.extraBuildInputs = [ ghostscript_headless ];
context.extraBuildInputs = [ coreutils ruby ];
cyrillic-bin.extraBuildInputs = [ coreutils gnused ];
dtxgen.extraBuildInputs = [ coreutils getopt gnumake zip ];
dviljk.extraBuildInputs = [ coreutils ];
epspdf.extraBuildInputs = [ ghostscript_headless ];
epstopdf.extraBuildInputs = [ ghostscript_headless ];
fragmaster.extraBuildInputs = [ ghostscript_headless ];
installfont.extraBuildInputs = [ coreutils getopt gnused ];
latexfileversion.extraBuildInputs = [ coreutils gnugrep gnused ];
listings-ext.extraBuildInputs = [ coreutils getopt ];
ltxfileinfo.extraBuildInputs = [ coreutils getopt gnused ];
ltximg.extraBuildInputs = [ ghostscript_headless ];
luaotfload.extraBuildInputs = [ ncurses ];
makeindex.extraBuildInputs = [ coreutils gnused ];
pagelayout.extraBuildInputs = [ gnused ncurses ];
pdfcrop.extraBuildInputs = [ ghostscript_headless ];
pdftex.extraBuildInputs = [ coreutils ghostscript_headless gnused ];
pdftex-quiet.extraBuildInputs = [ coreutils ];
pdfxup.extraBuildInputs = [ coreutils ghostscript_headless ];
pkfix-helper.extraBuildInputs = [ ghostscript_headless ];
ps2eps.extraBuildInputs = [ ghostscript_headless ];
pst2pdf.extraBuildInputs = [ ghostscript_headless ];
tex4ht.extraBuildInputs = [ ruby ];
texlive-scripts.extraBuildInputs = [ gnused ];
texlive-scripts-extra.extraBuildInputs = [ coreutils findutils ghostscript_headless gnused ];
thumbpdf.extraBuildInputs = [ ghostscript_headless ];
tpic2pdftex.extraBuildInputs = [ gawk ];
wordcount.extraBuildInputs = [ coreutils gnugrep ];
xdvi.extraBuildInputs = [ coreutils gnugrep ];
xindy.extraBuildInputs = [ gzip ];
#### adjustments to binaries
# TODO patch the scripts from bin.* directly in bin.* instead of here
# TODO we do not build binaries for the following packages (yet!)
biber-ms.binfiles = [];
xpdfopen.binfiles = [];
# mptopdf is a format link, but not generated by texlinks
# so we add it back to binfiles to generate it from mkPkgBin
mptopdf.binfiles = (orig.mptopdf.binfiles or []) ++ [ "mptopdf" ];
# mktexlsr distributed by texlive.infra has implicit dependencies (e.g. kpsestat)
# the perl one hidden in texlive-scripts is better behaved
"texlive.infra".binfiles = lib.remove "mktexlsr" orig."texlive.infra".binfiles;
# remove man, add mktexlsr
texlive-scripts.binfiles = (lib.remove "man" orig.texlive-scripts.binfiles) ++ [ "mktexlsr" ];
# upmendex is "TODO" in bin.nix
uptex.binfiles = lib.remove "upmendex" orig.uptex.binfiles;
# teckit_compile seems to be missing from bin.core{,-big}
# TODO find it!
xetex.binfiles = lib.remove "teckit_compile" orig.xetex.binfiles;
# xindy is broken on some platforms unfortunately
xindy.binfiles = if bin ? xindy
then lib.subtractLists [ "xindy.mem" "xindy.run" ] orig.xindy.binfiles
else [];
#### additional symlinks
cluttex.binlinks = {
cllualatex = "cluttex";
clxelatex = "cluttex";
};
epstopdf.binlinks.repstopdf = "epstopdf";
pdfcrop.binlinks.rpdfcrop = "pdfcrop";
ptex.binlinks = {
pdvitomp = bin.metapost + "/bin/pdvitomp";
pmpost = bin.metapost + "/bin/pmpost";
r-pmpost = bin.metapost + "/bin/r-pmpost";
};
texdef.binlinks = {
latexdef = "texdef";
};
texlive-scripts.binlinks = {
mktexfmt = "fmtutil";
texhash = "mktexlsr";
};
texlive-scripts-extra.binlinks = {
allec = "allcm";
kpsepath = "kpsetool";
kpsexpand = "kpsetool";
};
# metapost binaries are in bin.metapost instead of bin.core
uptex.binlinks = {
r-upmpost = bin.metapost + "/bin/r-upmpost";
updvitomp = bin.metapost + "/bin/updvitomp";
upmpost = bin.metapost + "/bin/upmpost";
};
#### add PATH dependencies without wrappers
# TODO deduplicate this code
a2ping.postFixup = ''
sed -i '6i$ENV{PATH}='"'"'${lib.makeBinPath a2ping.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/a2ping
'';
bibexport.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath bibexport.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/bibexport
'';
checklistings.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath checklistings.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/checklistings
'';
cjk-gs-integrate.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath cjk-gs-integrate.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/cjk-gs-integrate
'';
context.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath [ coreutils ]}''${PATH:+:$PATH}"' "$out"/bin/{contextjit,mtxrunjit}
sed -i '2iPATH="${lib.makeBinPath [ ruby ]}''${PATH:+:$PATH}"' "$out"/bin/texexec
'';
cyrillic-bin.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath cyrillic-bin.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/rumakeindex
'';
dtxgen.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath dtxgen.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/dtxgen
'';
dviljk.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath dviljk.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/dvihp
'';
epstopdf.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath epstopdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/epstopdf
'';
fragmaster.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath fragmaster.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/fragmaster
'';
installfont.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath installfont.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/installfont-tl
'';
latexfileversion.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath latexfileversion.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/latexfileversion
'';
listings-ext.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath listings-ext.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/listings-ext.sh
'';
ltxfileinfo.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath ltxfileinfo.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/ltxfileinfo
'';
ltximg.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath ltximg.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/ltximg
'';
luaotfload.postFixup = ''
sed -i '2ios.setenv("PATH","${lib.makeBinPath luaotfload.extraBuildInputs}" .. (os.getenv("PATH") and ":" .. os.getenv("PATH") or ""))' "$out"/bin/luaotfload-tool
'';
makeindex.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath makeindex.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/mkindex
'';
pagelayout.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath [ gnused ]}''${PATH:+:$PATH}"' "$out"/bin/pagelayoutapi
sed -i '2iPATH="${lib.makeBinPath [ ncurses ]}''${PATH:+:$PATH}"' "$out"/bin/textestvis
'';
pdfcrop.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pdfcrop.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pdfcrop
'';
pdftex.postFixup = ''
sed -i -e '2iPATH="${lib.makeBinPath [ coreutils gnused ]}''${PATH:+:$PATH}"' \
-e 's!^distillerpath="/usr/local/bin"$!distillerpath="${lib.makeBinPath [ ghostscript_headless ]}"!' \
"$out"/bin/simpdftex
'';
pdftex-quiet.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath pdftex-quiet.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/pdftex-quiet
'';
pdfxup.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath pdfxup.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/pdfxup
'';
pkfix-helper.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pkfix-helper.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pkfix-helper
'';
ps2eps.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath ps2eps.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/ps2eps
'';
pst2pdf.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath pst2pdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/pst2pdf
'';
tex4ht.postFixup = ''
sed -i -e '2iPATH="${lib.makeBinPath tex4ht.extraBuildInputs}''${PATH:+:$PATH}"' -e 's/\\rubyCall//g;' "$out"/bin/htcontext
'';
texlive-scripts.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath texlive-scripts.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/{fmtutil-user,mktexmf,mktexpk,mktextfm,updmap-user}
'';
thumbpdf.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath thumbpdf.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/thumbpdf
'';
tpic2pdftex.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath tpic2pdftex.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/tpic2pdftex
'';
wordcount.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath wordcount.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/wordcount
'';
# TODO patch in bin.xdvi
xdvi.postFixup = ''
sed -i '2iPATH="${lib.makeBinPath xdvi.extraBuildInputs}''${PATH:+:$PATH}"' "$out"/bin/xdvi
'';
xindy.postFixup = ''
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath xindy.extraBuildInputs}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/{texindy,xindy}
'';
#### other script fixes
# misc tab and python3 fixes
ebong.postFixup = ''
sed -Ei 's/import sre/import re/; s/file\(/open(/g; s/\t/ /g; s/print +(.*)$/print(\1)/g' "$out"/bin/ebong
'';
# find files in script directory, not binary directory
# add runtime dependencies to PATH
epspdf.postFixup = ''
sed -i '2ios.setenv("PATH","${lib.makeBinPath epspdf.extraBuildInputs}" .. (os.getenv("PATH") and ":" .. os.getenv("PATH") or ""))' "$out"/bin/epspdf
substituteInPlace "$out"/bin/epspdftk --replace '[info script]' "\"$scriptsFolder/epspdftk.tcl\""
'';
# find files in script directory, not in binary directory
latexindent.postFixup = ''
substituteInPlace "$out"/bin/latexindent --replace 'use FindBin;' "BEGIN { \$0 = '$scriptsFolder' . '/latexindent.pl'; }; use FindBin;"
'';
# make tlmgr believe it can use kpsewhich to evaluate TEXMFROOT
"texlive.infra".postFixup = ''
substituteInPlace "$out"/bin/tlmgr \
--replace 'if (-r "$bindir/$kpsewhichname")' 'if (1)'
'';
# Patch texlinks.sh back to 2015 version;
# otherwise some bin/ links break, e.g. xe(la)tex.
# add runtime dependencies to PATH
texlive-scripts-extra.postFixup = ''
patch -R "$out"/bin/texlinks < '${./texlinks.diff}'
sed -i '2iPATH="${lib.makeBinPath [ coreutils ]}''${PATH:+:$PATH}"' "$out"/bin/{allcm,dvired,mkocp,ps2frag}
sed -i '2iPATH="${lib.makeBinPath [ coreutils findutils ]}''${PATH:+:$PATH}"' "$out"/bin/allneeded
sed -i '2iPATH="${lib.makeBinPath [ coreutils ghostscript_headless ]}''${PATH:+:$PATH}"' "$out"/bin/dvi2fax
sed -i '2iPATH="${lib.makeBinPath [ gnused ]}''${PATH:+:$PATH}"' "$out"/bin/{kpsetool,texconfig,texconfig-sys}
sed -i '2iPATH="${lib.makeBinPath [ coreutils gnused ]}''${PATH:+:$PATH}"' "$out"/bin/texconfig-dialog
'';
# patch interpreter
texosquery.postFixup = ''
substituteInPlace "$out"/bin/* --replace java "$interpJava"
'';
#### dependency changes
# it seems to need it to transform fonts
xdvi.deps = (orig.xdvi.deps or []) ++ [ "metafont" ];
# TODO: remove when updating to texlive-2023, metadata has been corrected in the TeX catalogue
# tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README
arabi-add.license = [ "lppl13c" ];
# TODO: remove this when updating to texlive-2023, npp-for-context is no longer in texlive
# tlpdb lists license as "noinfo", but it's gpl3: https://github.com/luigiScarso/context-npp
npp-for-context.license = [ "gpl3Only" ];
# remove dependency-heavy packages from the basic collections
collection-basic.deps = lib.subtractLists [ "metafont" "xdvi" ] orig.collection-basic.deps;
@ -58,6 +376,15 @@ let
collection-metapost.deps = orig.collection-metapost.deps ++ [ "metafont" ];
collection-plaingeneric.deps = orig.collection-plaingeneric.deps ++ [ "xdvi" ];
#### misc
# tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README
arabi-add.license = [ "lppl13c" ];
# TODO: remove this when updating to texlive-2023, npp-for-context is no longer in texlive
# tlpdb lists license as "noinfo", but it's gpl3: https://github.com/luigiScarso/context-npp
npp-for-context.license = [ "gpl3Only" ];
texdoc = {
extraRevision = ".tlpdb${toString tlpdbVersion.revision}";
extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";
@ -91,27 +418,27 @@ let
pkg = attrs // {
sha512 = attrs.sha512.${if tlType == "tlpkg" then "run" else tlType};
inherit pname tlType version;
} // lib.optionalAttrs (attrs.hasManpages or false) {
hasManpages = true;
};
in mkPkg pkg;
run = if (attrs.hasRunfiles or false) then mkPkgV "run"
# the fake derivations are used for filtering of hyphenation patterns and formats
else ({
inherit pname version;
tlType = "run";
hasHyphens = attrs.hasHyphens or false;
tlDeps = map (n: tl.${n}) (attrs.deps or []);
} // lib.optionalAttrs (attrs ? formats) { inherit (attrs) formats; });
in {
# TL pkg contains lists of packages: runtime files, docs, sources, tlpkg, binaries
pkgs =
# tarball of a collection/scheme itself only contains a tlobj file
[( if (attrs.hasRunfiles or false) then mkPkgV "run"
# the fake derivations are used for filtering of hyphenation patterns and formats
else {
inherit pname version;
tlType = "run";
hasFormats = attrs.hasFormats or false;
hasHyphens = attrs.hasHyphens or false;
tlDeps = map (n: tl.${n}) (attrs.deps or []);
}
)]
[ run ]
++ lib.optional (attrs.sha512 ? doc) (mkPkgV "doc")
++ lib.optional (attrs.sha512 ? source) (mkPkgV "source")
++ lib.optional (attrs.hasTlpkg or false) (mkPkgV "tlpkg")
++ lib.optional (bin ? ${pname})
( bin.${pname} // { tlType = "bin"; } );
++ lib.optional (attrs ? binfiles && attrs.binfiles != []) (mkPkgBin pname version run attrs);
};
version = {
@ -167,8 +494,46 @@ let
# name + version for the derivation
mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
# build tlType == "bin" containers based on `binfiles` in TLPDB
# see UPGRADING.md for how to keep the list of shebangs up to date
mkPkgBin = let extToInput = {
jar = jdk;
lua = bin.luatex;
py = python3;
rb = ruby;
sno = snobol4;
tcl = tk;
texlua = bin.luatex;
tlu = bin.luatex;
}; in pname: version: run:
{ binfiles, scriptsFolder ? pname, postFixup ? "", scriptExts ? [], extraBuildInputs ? [], binlinks ? {}, ... }@args:
runCommand "texlive-${pname}.bin-${version}"
{
# metadata for texlive.combine
passthru = {
inherit pname version;
tlType = "bin";
};
# shebang interpreters
buildInputs = extraBuildInputs ++ [ bash perl ] ++ (lib.attrVals scriptExts extToInput);
# absolute scripts folder
scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + scriptsFolder);
# binaries info
inherit binfiles;
binlinks = builtins.attrNames binlinks;
bintargets = builtins.attrValues binlinks;
binfolders = [ (lib.getBin bin.core) ] ++ lib.optional (bin ? ${pname}) (lib.getBin bin.${pname});
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;
}
''
. "$makeBinContainers"
${postFixup}
'';
# create a derivation that contains an unpacked upstream TL package
mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, ... }@args:
mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, hasManpages ? false, ... }@args:
let
# the basename used by upstream (without ".tar.xz" suffix)
urlName = mkURLName args;
@ -191,11 +556,12 @@ let
} // lib.optionalAttrs (tlType == "run" && args ? deps) {
tlDeps = map (n: tl.${n}) args.deps;
} // lib.optionalAttrs (tlType == "run") {
hasFormats = args.hasFormats or false;
hasHyphens = args.hasHyphens or false;
} // lib.optionalAttrs (tlType == "tlpkg" && args ? postactionScript) {
postactionScript = args.postactionScript;
};
}
// lib.optionalAttrs (tlType == "run" && args ? formats) { inherit (args) formats; }
// lib.optionalAttrs (tlType == "doc" && hasManpages) { hasManpages = true; };
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha256";
@ -250,7 +616,10 @@ in
xz = tlpdbxz;
};
bin = assert assertions; bin;
bin = assert assertions; bin // {
# for backward compatibility
latexindent = lib.findFirst (p: p.tlType == "bin") tl.latexindent.pkgs;
};
combine = assert assertions; combine;
# Pre-defined combined packages for TeX Live schemes,

View file

@ -8629,6 +8629,7 @@
"texlive-fr.doc.r63071"="120jahzjmak3shjhiy81gv6nk3c1hv4rrxyi1mmzi1xklxjlhl4b";
"texlive-it.doc.r58653"="0vmwn6n8bxpzcfrzic5qg5k2vklbm6rhl9861zxsli0rd9396qn6";
"texlive-ja.doc.r62817"="1h2rv13ip3bgmfz8q64abqx3ajla9mc02a50xz1nzmng0rmgz04r";
"texlive-msg-translations.tlpkg.r65889"="03bshv5xacw01ssbpd5wmz4ryym3dc27l8mwyh652i8gd2lg0scm";
"texlive-pl.doc.r62841"="19qab4nd9z87v9dpx7gh2a87rw8k4x6kqzg5yc7wqmf46pfjmpcw";
"texlive-ru.doc.r58426"="0c77yyfj87fajran8jppj0x8krk6b5195iscpn8z2n94spz2fcc9";
"texlive-scripts.r66570"="0hl1vjr4hr7q7s2pvizicmabb185df5rl84cjsz0ki7vai5mh7pq";

View file

@ -0,0 +1,62 @@
# load realpath
loadables="$(command -v bash)"
loadables="${loadables%/bin/bash}/lib/bash"
enable -f "$loadables/realpath" realpath
mkdir -p "$out/bin"
# find interpreters
export interpPerl="$(PATH="$HOST_PATH" command -v perl)"
export interpJava="$(PATH="$HOST_PATH" command -v java || :)"
export interpWish="$(PATH="$HOST_PATH" command -v wish || :)"
# prepare sed script
substituteAll "$patchScripts" patch-scripts.sed
for binname in $binfiles ; do
# binlinks to be created last, after the other binaries are in place
if [[ " $binlinks " == *" $binname "* ]] ; then
continue
fi
output="$out/bin/$binname"
# look for existing binary from bin.core or bin.${pname}
for folder in $binfolders ; do
target="$folder"/bin/"$binname"
if [[ -f "$target" && -x "$target" ]] ; then
ln -s "$(realpath "$target")" "$output"
continue 2
fi
done
# look for scripts
# the explicit list of extensions avoid non-scripts such as $binname.cmd, $binname.jar, $binname.pm
# the order is relevant: $binname.sh is preferred to other $binname.*
if [[ -n "$scriptsFolder" ]] ; then
for script in "$scriptsFolder/$binname"{,.sh,.lua,.pl,.py,.rb,.sno,.tcl,.texlua,.tlu}; do
if [[ -f "$script" ]] ; then
sed -f patch-scripts.sed \
-e 's/^scriptname=`basename "\$0"`$/'"scriptname='$(basename "$binname")'/" \
-e 's/^scriptname=`basename "\$0" .sh`$'"/scriptname='$(basename "$binname" .sh)'/" \
"$script" > "$output"
chmod +x "$output"
continue 2
fi
done
fi
echo "error: could not find source for 'bin/$binname'" >&2
exit 1
done
# patch shebangs
patchShebangs "$out/bin"
# generate links
# we canonicalise the source to avoid symlink chains, and to check that it exists
cd "$out"/bin
for alias in $binlinks ; do
target="${bintargets%% *}"
bintargets="${bintargets#* }"
ln -s "$(realpath "$target")" "$out/bin/$alias"
done

View file

@ -0,0 +1,57 @@
1{
/python/{
N;
# add script folder to path, unless we interfere with a docstring
/\nr"""/b skip-python-path-patch
s!\n!\nimport sys; sys.path.insert(0,'@scriptsFolder@')\n!
:skip-python-path-patch
}
/^#!.*perl/{
# add script folder to @INC
s!$! -I@scriptsFolder@!
}
/^eval/{
# most likely the weird perl shebang
N
/^eval '(exit \$?0)' && eval 'exec perl -S \$0 \${1+"\$@"}' && eval 'exec perl -S \$0 \$argv:q'\n *if 0;$/{
x; s/.*/patching weird perl shebang/; w /dev/stderr
x; s|^.*$|#!@interpPerl@ -I@scriptsFolder@|
}
}
}
# patch 'exec interpreter'
/exec java /{
x; s/.*/patching exec java/; w /dev/stderr
x; s|exec java |exec '@interpJava@' |g
/exec ''/{
x; s/^.*$/error: java missing from PATH/; w /dev/stderr
q 1
}
}
/exec perl /{
x; s/.*/patching exec perl/; w /dev/stderr
x; s|exec perl |exec @interpPerl@ -I@scriptsFolder@ |g
/exec ''/{
x; s/^.*$/error: perl missing from PATH/; w /dev/stderr
q 1
}
}
/exec wish /{
x; s/.*/patching exec wish/; w /dev/stderr
x; s|exec wish |exec '@interpWish@' |g
/exec ''/{
x; s/^.*$/error: wish missing from PATH/; w /dev/stderr
q 1
}
}
# make jar wrappers work without kpsewhich
s!^jarpath=`kpsewhich --progname=[^ ]* --format=texmfscripts \([^ ]*\)`$!jarpath=@scriptsFolder@/\1!g
# replace CYGWIN grep test with bash builtin
s!echo "$kernel" | grep CYGWIN >/dev/null;![[ "$kernel" == *CYGWIN* ]]!g

View file

@ -81,6 +81,20 @@ $a}
}
# detect presence of notable files
/^docfiles /{
s/^.*$// # ignore the first line
# read all files
:next-doc
N
s/\n / / # remove newline
t next-doc # loop if the previous lines matched
/ (texmf-dist|RELOC)\/doc\/man\//i\ hasManpages = true;
D # restart cycle
}
/^runfiles /{
s/^.*$// # ignore the first line
@ -89,9 +103,25 @@ $a}
N
s/\n / / # remove newline
t next-file # loop if previous line matched
s/\n/ \n/ # add space before last newline for accurate matching below
/ (RELOC|texmf-dist)\//i\ hasRunfiles = true;
/ tlpkg\//i\ hasTlpkg = true;
# extract script extensions
/ texmf-dist\/scripts\/.*\.(jar|lua|py|rb|sno|tcl|texlua|tlu) /{
i\ scriptExts = [
/ texmf-dist\/scripts\/.*\.jar /i\ "jar"
/ texmf-dist\/scripts\/.*\.lua /i\ "lua"
/ texmf-dist\/scripts\/.*\.py /i\ "py"
/ texmf-dist\/scripts\/.*\.rb /i\ "rb"
/ texmf-dist\/scripts\/.*\.sno /i\ "sno"
/ texmf-dist\/scripts\/.*\.tcl /i\ "tcl"
/ texmf-dist\/scripts\/.*\.texlua /i\ "texlua"
/ texmf-dist\/scripts\/.*\.tlu /i\ "tlu"
i\ ];
}
D # restart cycle from the current line
}
@ -101,8 +131,50 @@ $a}
# extract hyphenation patterns and formats
# (this may create duplicate lines, use uniq to remove them)
/^execute\sAddHyphen/i\ hasHyphens = true;
/^execute\sAddFormat/i\ hasFormats = true;
# extract format details
/^execute\sAddFormat\s/{
# open a list
i\ formats = [
# create one attribute set per format
# note that format names are not unique
# plain keys: name, engine, patterns
# optionally double quoted key: options
# boolean key: mode (enabled/disabled)
# comma-separated lists: fmttriggers, patterns
:next-fmt
s/(^|\n)execute\sAddFormat/ {/
s/\s+options="([^"]+)"/\n options = "\1";/
s/\s+(name|engine|options)=([^ \t\n]+)/\n \1 = "\2";/g
s/\s+mode=enabled//
s/\s+mode=disabled/\n enabled = false;/
s/\s+(fmttriggers|patterns)=([^ \t\n]+)/\n \1 = [ "\2" ];/g
s/$/\n }/
:split-triggers
s/"([^,]+),([^"]+)" ]/"\1" "\2" ]/;
t split-triggers # repeat until there are no commas
p
s/^.*$// # clear pattern space
N
/^\nexecute\sAddFormat\s/b next-fmt
# close the list
i\ ];
D # restart cycle from the current line
}
# close attrmap
/^$/i};
}
# add list of binaries from one of the architecture-specific packages
/^name ([^.]+|texlive\.infra)\.x86_64-linux$/,/^$/{
s/^name ([0-9].*|texlive\.infra)\.x86_64-linux$/"\1".binfiles = [/p
s/^name (.*)\.x86_64-linux$/\1.binfiles = [/p
s!^ bin/x86_64-linux/(.+)$! "\1"!p
/^$/i];
}

File diff suppressed because it is too large Load diff