Merge pull request #252012 from xworld21/texlive-split-core

texlive: split outputs & spring cleaning
This commit is contained in:
Dmitry Kalinkin 2023-09-02 22:22:19 -04:00 committed by GitHub
commit ac3d200056
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 122 additions and 176 deletions

View file

@ -373,7 +373,7 @@ rec {
# (1) binaries requiring --help or -h
help = [ "arlatex" "bundledoc" "cachepic" "checklistings" "dvipos" "extractres" "fig4latex" "fragmaster"
"kpsewhere" "latex-git-log" "ltxfileinfo" "mendex" "perltex" "pn2pdf" "psbook" "psnup" "psresize" "purifyeps"
"simpdftex" "tex2xindy" "texluac" "texluajitc" "urlbst" "yplan" ];
"simpdftex" "tex2xindy" "texluac" "texluajitc" "upmendex" "urlbst" "yplan" ];
shortHelp = [ "adhocfilelist" "authorindex" "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "ctanupload"
"disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man" "listings-ext.sh" "pygmentex" ];
# (2) binaries that return non-zero exit code even if correctly asked for help

View file

@ -7,6 +7,7 @@
, brotli, cairo, pixman, xorg, clisp, biber, woff2, xxHash
, makeWrapper, shortenPerlShebang, useFixedHashes, asymptote
, biber-ms
, tlpdb
}:
# Useful resource covering build options:
@ -22,6 +23,20 @@ let
assertFixedHash = name: src:
if ! useFixedHashes || src ? outputHash then src else throw "The TeX Live package '${src.pname}' must have a fixed hash before building '${name}'.";
# list of packages whose binaries are built in core, core-big
# generated manually by inspecting ${core}/bin
corePackages = [ "afm2pl" "aleph" "autosp" "axodraw2" "bibtex" "cjkutils"
"ctie" "cweb" "detex" "dtl" "dvi2tty" "dvicopy" "dvidvi" "dviljk"
"dviout-util" "dvipdfmx" "dvipos" "dvips" "fontware" "gregoriotex"
"gsftopk" "hitex" "kpathsea" "lacheck" "lcdftypetools" "m-tx" "makeindex"
"mfware" "musixtnt" "omegaware" "patgen" "pdftex" "pdftosrc" "pmx"
"ps2eps" "ps2pk" "psutils" "ptex" "seetexk" "synctex" "t1utils" "tex"
"tex4ht" "texlive-scripts-extra" "texware" "tie" "tpic2pdftex" "ttfutils"
"uptex" "velthuis" "vlna" "web" "xml2pmx" ];
coreBigPackages = [ "metafont" "mflua" "metapost" "luatex" "luahbtex"
"xetex" ] ++ lib.optional withLuaJIT "luajittex";
binPackages = lib.getAttrs (corePackages ++ coreBigPackages) tlpdb;
common = {
src = fetchurl {
urls = [
@ -53,21 +68,42 @@ let
configureFlags = [
"--with-banner-add=/nixos.org"
"--disable-missing" "--disable-native-texlive-build"
"--disable-missing" # terminate if a requested program or feature must be
# disabled, e.g., due to missing libraries
"--disable-native-texlive-build" # do not build for the TeX Live binary distribution
"--enable-shared" # "--enable-cxx-runtime-hack" # static runtime
"--enable-tex-synctex"
"--disable-texlive" # do not build the texlive (TeX Live scripts) package
"--disable-linked-scripts" # do not install the linked scripts
"-C" # use configure cache to speed up
]
++ withSystemLibs [
# see "from TL tree" vs. "Using installed" in configure output
"zziplib" "mpfr" "gmp"
"pixman" "potrace" "gd" "freetype2" "libpng" "libpaper" "zlib"
];
] ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform)
"BUILDCC=${buildPackages.stdenv.cc.targetPrefix}cc";
# clean broken links to stuff not built
cleanBrokenLinks = ''
for f in "$out"/bin/*; do
if [[ ! -x "$f" ]]; then rm "$f"; fi
# move binaries to corresponding split outputs, based on content of texlive.tlpdb
binToOutput = lib.listToAttrs
(lib.concatMap
(n: map (v: { name = v; value = builtins.replaceStrings [ "-" ] [ "_" ] n; }) binPackages.${n}.binfiles or [ ])
(builtins.attrNames binPackages));
moveBins = ''
for bin in "$out/bin"/* ; do
bin="''${bin##*/}"
package="''${binToOutput[$bin]}"
if [[ -n "$package" ]] ; then
if [[ -z "''${!package}" ]] ; then
echo "WARNING: missing output '$package' for binary '$bin', leaving in 'out'"
else
mkdir -p "''${!package}"/bin
mv "$out/bin/$bin" "''${!package}"/bin/
fi
else
echo "WARNING: no output known for binary '$bin', leaving in 'out'"
fi
done
'';
};
@ -76,7 +112,6 @@ let
withLuaJIT = !(stdenv.hostPlatform.isPower && stdenv.hostPlatform.is64bit) && !stdenv.hostPlatform.isRiscV;
in rec { # un-indented
inherit (common) cleanBrokenLinks;
texliveYear = year;
@ -84,18 +119,21 @@ core = stdenv.mkDerivation rec {
pname = "texlive-bin";
inherit version;
inherit (common) src prePatch;
__structuredAttrs = true;
outputs = [ "out" "doc" "dev" ];
inherit (common) binToOutput src prePatch;
outputs = [ "out" "dev" "man" "info" ]
++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) corePackages);
nativeBuildInputs = [
pkg-config
] ++ lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) [
] ++ lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) (with texlive.bin.core; [
# configure: error: tangle was not found but is required when cross-compiling.
# dev (himktables) is used when building hitex to generate the additional source file hitables.c
texlive.bin.core
texlive.bin.core.dev
];
web/*tangle*/ cweb/*ctangle*/ omegaware/*otangle*/ tie/*tie*/ # see "Building TeX Live" 6.4.2 Cross problems
dev/*himktables*/
]);
buildInputs = [
/*teckit*/ zziplib mpfr gmp
@ -113,7 +151,9 @@ core = stdenv.mkDerivation rec {
preConfigure = ''
rm -r libs/{cairo,freetype2,gd,gmp,graphite2,harfbuzz,icu,libpaper,libpng} \
libs/{lua53,luajit,mpfr,pixman,zlib,zziplib}
libs/{lua53,luajit,mpfr,pixman,pplib,teckit,zlib,zziplib} \
texk/{bibtex-x,chktex,dvipng,dvisvgm,upmendex,xdvik} \
utils/{asymptote,texdoctk,xindy,xpdfopen}
mkdir WorkDir
cd WorkDir
'';
@ -122,7 +162,6 @@ core = stdenv.mkDerivation rec {
depsBuildBuild = [ buildPackages.stdenv.cc ];
configureFlags = common.configureFlags
++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ "BUILDCC=${buildPackages.stdenv.cc.targetPrefix}cc" ]
++ [ "--without-x" ] # disable xdvik and xpdfopen
++ map (what: "--disable-${what}") [
"chktex"
@ -137,53 +176,18 @@ core = stdenv.mkDerivation rec {
doCheck = false; # triptest fails, likely due to missing TEXMF tree
preCheck = "patchShebangs ../texk/web2c";
installTargets = [ "install" "texlinks" ];
installTargets = [ "install" ];
# TODO: perhaps improve texmf.cnf search locations
postInstall =
/* links format -> engine will be regenerated in texlive.combine
note: for unlinking, the texlinks patch is irrelevant, so we use
the included texlinks.sh to avoid the dependency on bin.texlinks */ ''
PATH="$out/bin:$PATH" sh ../texk/texlive/linked_scripts/texlive-extra/texlinks.sh --cnffile "$out/share/texmf-dist/web2c/fmtutil.cnf" --unlink "$out/bin"
'' + /* a few texmf-dist files are useful; take the rest from pkgs */ ''
mv "$out/share/texmf-dist/web2c/texmf.cnf" .
rm -r "$out/share/texmf-dist"
mkdir -p "$out"/share/texmf-dist/{web2c,scripts/texlive/TeXLive}
mv ./texmf.cnf "$out/share/texmf-dist/web2c/"
cp ../texk/tests/TeXLive/*.pm "$out/share/texmf-dist/scripts/texlive/TeXLive/"
cp ../texk/texlive/linked_scripts/scripts.lst "$out/share/texmf-dist/scripts/texlive/"
'' +
(let extraScripts =
''
tex4ht/ht.sh
tex4ht/htcontext.sh
tex4ht/htcopy.pl
tex4ht/htlatex.sh
tex4ht/htmex.sh
tex4ht/htmove.pl
tex4ht/httex.sh
tex4ht/httexi.sh
tex4ht/htxelatex.sh
tex4ht/htxetex.sh
tex4ht/mk4ht.pl
tex4ht/xhlatex.sh
'';
in
''
echo -e 'texmf_scripts="$texmf_scripts\n${extraScripts}"' \
>> "$out/share/texmf-dist/scripts/texlive/scripts.lst"
'')
+ /* doc location identical with individual TeX pkgs */ ''
mkdir -p "$doc/doc"
mv "$out"/share/{man,info} "$doc"/doc
'' + /* remove manpages for utils that live in texlive.texlive-scripts to avoid a conflict in buildEnv */ ''
(cd "$doc"/doc/man/man1; rm {fmtutil-sys.1,fmtutil.1,mktexfmt.1,mktexmf.1,mktexpk.1,mktextfm.1,texhash.1,updmap-sys.1,updmap.1})
/* remove redundant texmf-dist (content provided by TeX Live packages) */
''
rm -fr "$out"/share/texmf-dist
'' + /* install himktables in separate output for use in cross compilation */ ''
mkdir -p $dev/bin
cp texk/web2c/.libs/himktables $dev/bin/himktables
'' + cleanBrokenLinks;
'' + common.moveBins;
setupHook = ./setup-hook.sh; # TODO: maybe texmf-nix -> texmf (and all references)
passthru = { inherit version buildInputs; };
meta = with lib; {
@ -196,12 +200,15 @@ core = stdenv.mkDerivation rec {
};
inherit (core-big) metafont mflua metapost luatex luahbtex luajittex xetex;
core-big = stdenv.mkDerivation { #TODO: upmendex
inherit (core-big) metafont mflua metapost luatex luahbtex xetex;
luajittex = core.big.luajittex or null;
core-big = stdenv.mkDerivation {
pname = "texlive-core-big.bin";
inherit version;
inherit (common) src prePatch;
__structuredAttrs = true;
inherit (common) binToOutput src prePatch;
patches = [
# improves reproducibility of fmt files. This patch has been proposed upstream,
@ -241,92 +248,46 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
inherit (core) nativeBuildInputs depsBuildBuild;
buildInputs = core.buildInputs ++ [ core cairo harfbuzz icu graphite2 libX11 ];
/* deleting the unused packages speeds up configure by a considerable margin
and ensures we do not rebuild existing libraries by mistake */
preConfigure = ''
rm -r libs/{cairo,freetype2,gd,gmp,graphite2,harfbuzz,icu,libpaper,libpng} \
libs/{mpfr,pixman,xpdf,zlib,zziplib} \
texk/{afm2pl,bibtex-x,chktex,cjkutils,detex,dtl,dvi2tty,dvidvi,dviljk,dviout-util} \
texk/{dvipdfm-x,dvipng,dvipos,dvipsk,dvisvgm,gregorio,gsftopk,kpathsea} \
texk/{lcdf-typetools,makeindexk,makejvf,mendexk,musixtnt,ps2pk,psutils,ptexenc} \
texk/{seetexk,tex4htk,texlive,ttf2pk2,ttfdump,xdvik} \
utils/{asymptote,autosp,axodraw2,devnag,lacheck,m-tx,pmx,ps2eps,t1utils,texdoctk} \
utils/{tpic2pdftex,vlna,xindy,xml2pmx,xpdfopen}
mkdir WorkDir
cd WorkDir
'';
configureFlags = common.configureFlags
++ withSystemLibs [ "kpathsea" "ptexenc" "cairo" "harfbuzz" "icu" "graphite2" ]
++ map (prog: "--disable-${prog}") # don't build things we already have
([ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "pdftex"
# list from texk/web2c/configure
([ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "hitex" "pdftex"
"web-progs" "synctex"
] ++ lib.optionals (!withLuaJIT) [ "luajittex" "luajithbtex" "mfluajit" ]);
] ++ lib.optionals (!withLuaJIT) [ "luajittex" "luajithbtex" "mfluajit" ])
/* disable all packages, re-enable upmendex, web2c packages */
++ [ "--disable-all-pkgs" "--enable-upmendex" "--enable-web2c" ]
/* kpathsea requires specifying the kpathsea location manually */
++ [ "--with-kpathsea-includes=${core.dev}/include" ];
configureScript = ":";
configureScript = "../configure";
# we use static libtexlua, because it's only used by a single binary
postConfigure = let
luajit = lib.optionalString withLuaJIT ",luajit";
in
lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform)
# without this, the native builds attempt to use the binary
# ${target-triple}-gcc, but we need to use the wrapper script.
''
export BUILDCC=${buildPackages.stdenv.cc}/bin/cc
''
+
''
mkdir ./WorkDir && cd ./WorkDir
for path in libs/{pplib,teckit,lua53${luajit}} texk/web2c; do
(
if [[ "$path" =~ "libs/lua" ]]; then
extraConfig="--enable-static --disable-shared"
else
extraConfig=""
fi
'' + lib.optionalString (!stdenv.buildPlatform.canExecute stdenv.hostPlatform)
# results of the tests performed by the configure scripts are
# toolchain-dependent, so native components and cross components cannot use
# the same cached test results.
# Disable the caching for components with native subcomponents.
''
if [[ "$path" =~ "libs/luajit" ]] || [[ "$path" =~ "texk/web2c" ]]; then
extraConfig="$extraConfig --cache-file=/dev/null"
fi
''
+
''
mkdir -p "$path" && cd "$path"
"../../../$path/configure" $configureFlags $extraConfig
if [[ "$path" =~ "libs/luajit" ]] || [[ "$path" =~ "libs/pplib" ]]; then
# ../../../texk/web2c/mfluadir/luapeg/lpeg.h:29:10: fatal error: 'lua.h' file not found
# ../../../texk/web2c/luatexdir/luamd5/md5lib.c:197:10: fatal error: 'utilsha.h' file not found
make ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES}}
fi
)
done
'';
preBuild = "cd texk/web2c";
enableParallelBuilding = true;
doCheck = false; # fails
# now distribute stuff into outputs, roughly as upstream TL
# (uninteresting stuff remains in $out, typically duplicates from `core`)
outputs = [
"out"
"metafont"
"mflua"
"metapost"
"luatex"
"luahbtex"
"luajittex"
"xetex"
];
postInstall = ''
for output in $(getAllOutputNames); do
mkdir -p "''${!output}/bin"
done
mv "$out/bin"/{inimf,mf,mf-nowin} "$metafont/bin/"
mv "$out/bin"/mflua{,-nowin} "$mflua/bin/"
mv "$out/bin"/{*tomp,mfplain,*mpost} "$metapost/bin/"
mv "$out/bin"/{luatex,texlua,texluac} "$luatex/bin/"
mv "$out/bin"/luahbtex "$luahbtex/bin/"
mv "$out/bin"/xetex "$xetex/bin/"
cp ../../libs/teckit/teckit_compile "$xetex/bin/"
'' + lib.optionalString withLuaJIT ''
mv "$out/bin"/mfluajit{,-nowin} "$mflua/bin/"
mv "$out/bin"/{luajittex,luajithbtex,texluajit,texluajitc} "$luajittex/bin/"
'' ;
outputs = [ "out" "dev" "man" "info" ]
++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) coreBigPackages)
# some outputs of metapost, omegaware are for ptex/uptex
++ [ "ptex" "uptex" ]
# unavoidable duplicates from core
++ [ "ctie" "cweb" "omegaware" "texlive_scripts_extra" "tie" "web" ];
postInstall = common.moveBins;
};

View file

@ -118,8 +118,13 @@ let
{
passthru = commonPassthru // { tlType = "bin"; };
inherit meta;
# shebang interpreters
buildInputs = (args.extraBuildInputs or [ ]) ++ [ bash perl ] ++ (lib.attrVals (args.scriptExts or [ ]) extToInput);
# shebang interpreters and compiled binaries
buildInputs = let outName = builtins.replaceStrings [ "-" ] [ "_" ] pname; in
[ texliveBinaries.core.${outName} or null
texliveBinaries.${pname} or null
texliveBinaries.core-big.${outName} or null ]
++ (args.extraBuildInputs or [ ]) ++ [ bash perl ]
++ (lib.attrVals (args.scriptExts or [ ]) extToInput);
nativeBuildInputs = extraNativeBuildInputs;
# absolute scripts folder
scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + args.scriptsFolder or pname);
@ -127,8 +132,6 @@ let
inherit (args) binfiles;
binlinks = builtins.attrNames (args.binlinks or { });
bintargets = builtins.attrValues (args.binlinks or { });
binfolders = [ (lib.getBin texliveBinaries.core) ] ++
lib.optional (texliveBinaries ? ${pname}) (lib.getBin texliveBinaries.${pname});
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;

View file

@ -19,6 +19,7 @@ let
withIcu = true; withGraphite2 = true;
};
inherit useFixedHashes;
tlpdb = overriddenTlpdb;
};
# function for creating a working environment from a set of TL packages
@ -36,7 +37,7 @@ let
overriddenTlpdb = let
overrides = import ./tlpdb-overrides.nix {
inherit
lib bin tlpdb tlpdbxz tl
stdenv lib bin tlpdb tlpdbxz tl
installShellFiles
coreutils findutils gawk getopt ghostscript_headless gnugrep
gnumake gnupg gnused gzip ncurses perl python3 ruby zip;

View file

@ -20,14 +20,12 @@ for binname in $binfiles ; do
output="$out/bin/$binname"
# look for existing binary from bin.core or bin.${pname}
for folder in $binfolders ; do
target="$folder"/bin/"$binname"
if [[ -f "$target" && -x "$target" ]] ; then
ln -s "$(realpath "$target")" "$output"
continue 2
fi
done
# look for existing binary from bin.*
target="$(PATH="$HOST_PATH" command -v "$binname" || :)"
if [[ -n "$target" && -x "$target" ]] ; then
ln -s "$(realpath "$target")" "$output"
continue
fi
# look for scripts
# the explicit list of extensions avoid non-scripts such as $binname.cmd, $binname.jar, $binname.pm

View file

@ -1,7 +0,0 @@
addTeXMFPath () {
if test -d "$1/share/texmf-nix"; then
export TEXINPUTS="${TEXINPUTS}${TEXINPUTS:+:}$1/share/texmf-nix//:"
fi
}
addEnvHooks "$targetOffset" addTeXMFPath

View file

@ -1,4 +1,4 @@
{ lib, tlpdb, bin, tlpdbxz, tl
{ stdenv, lib, tlpdb, bin, tlpdbxz, tl
, installShellFiles
, coreutils, findutils, gawk, getopt, ghostscript_headless, gnugrep
, gnumake, gnupg, gnused, gzip, ncurses, perl, python3, ruby, zip
@ -13,7 +13,9 @@ let
# so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf)
removeFormatLinks = lib.mapAttrs (_: attrs:
if (attrs ? formats && attrs ? binfiles)
then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine) attrs.formats);
# TLPDB reports erroneously that various metafont binaries like "mf" are format links to engines
# like "mf-nowin"; core-big provides both binaries and links so we simply skip them here
then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine && ! lib.hasSuffix "-nowin" f.engine) attrs.formats);
binNotFormats = lib.subtractLists formatLinks attrs.binfiles;
in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ]
else attrs);
@ -104,10 +106,6 @@ in lib.recursiveUpdate orig rec {
# remove man
texlive-scripts.binfiles = lib.remove "man" orig.texlive-scripts.binfiles;
# upmendex is "TODO" in bin.nix
uptex.binfiles = lib.remove "upmendex" orig.uptex.binfiles;
# xindy is broken on some platforms unfortunately
xindy.binfiles = if bin ? xindy
then lib.subtractLists [ "xindy.mem" "xindy.run" ] orig.xindy.binfiles
@ -122,12 +120,6 @@ in lib.recursiveUpdate orig rec {
epstopdf.binlinks.repstopdf = "epstopdf";
pdfcrop.binlinks.rpdfcrop = "pdfcrop";
ptex.binlinks = {
pdvitomp = bin.metapost + "/bin/pdvitomp";
pmpost = bin.metapost + "/bin/pmpost";
r-pmpost = bin.metapost + "/bin/r-pmpost";
};
texdef.binlinks = {
latexdef = "texdef";
};
@ -143,13 +135,6 @@ in lib.recursiveUpdate orig rec {
kpsexpand = "kpsetool";
};
# metapost binaries are in bin.metapost instead of bin.core
uptex.binlinks = {
r-upmpost = bin.metapost + "/bin/r-upmpost";
updvitomp = bin.metapost + "/bin/updvitomp";
upmpost = bin.metapost + "/bin/upmpost";
};
#### add PATH dependencies without wrappers
# TODO deduplicate this code
a2ping.postFixup = ''
@ -340,6 +325,11 @@ in lib.recursiveUpdate orig rec {
#### misc
# RISC-V: https://github.com/LuaJIT/LuaJIT/issues/628
luajittex.binfiles = lib.optionals
(!(stdenv.hostPlatform.isPower && stdenv.hostPlatform.is64bit) && !stdenv.hostPlatform.isRiscV)
orig.luajittex.binfiles;
# tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README
arabi-add.license = [ "lppl13c" ];
@ -362,7 +352,7 @@ in lib.recursiveUpdate orig rec {
mkdir -p support/texdoc
touch support/texdoc/NEWS
TEXMFCNF="${bin.core}"/share/texmf-dist/web2c TEXMF="$out" TEXDOCS=. TEXMFVAR=. \
TEXMFCNF="${lib.head tl.kpathsea.pkgs}/web2c" TEXMF="$out" TEXDOCS=. TEXMFVAR=. \
"${bin.luatex}"/bin/texlua "$out"/scripts/texdoc/texdoc.tlu \
-c texlive_tlpdb=texlive.tlpdb -lM texdoc
@ -372,7 +362,7 @@ in lib.recursiveUpdate orig rec {
# install zsh completion
postFixup = ''
TEXMFCNF="${bin.core}"/share/texmf-dist/web2c TEXMF="$scriptsFolder/../.." \
TEXMFCNF="${lib.head tl.kpathsea.pkgs}"/web2c TEXMF="$scriptsFolder/../.." \
texlua "$out"/bin/texdoc --print-completion zsh > "$TMPDIR"/_texdoc
substituteInPlace "$TMPDIR"/_texdoc \
--replace 'compdef __texdoc texdoc' '#compdef texdoc' \