From 806154fbc745ea9ccad38d7f353a7c96e9c50505 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 01/12] texlive.bin.core: split output according to texlive.tlpdb --- pkgs/tools/typesetting/tex/texlive/bin.nix | 55 ++++++++++++++++--- .../tex/texlive/build-texlive-package.nix | 9 +-- .../tools/typesetting/tex/texlive/default.nix | 1 + .../tex/texlive/make-bin-containers.sh | 14 ++--- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index dd1822de492..cce8812a355 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -7,6 +7,7 @@ , brotli, cairo, pixman, xorg, clisp, biber, woff2, xxHash , makeWrapper, shortenPerlShebang, useFixedHashes, asymptote , biber-ms +, tlpdb }: # Useful resource covering build options: @@ -22,6 +23,18 @@ let assertFixedHash = name: src: if ! useFixedHashes || src ? outputHash then src else throw "The TeX Live package '${src.pname}' must have a fixed hash before building '${name}'."; + # list of packages whose binaries are built in core, core-big + # generated manually by inspecting ${core}/bin + corePackages = [ "afm2pl" "aleph" "autosp" "axodraw2" "bibtex" "cjkutils" + "ctie" "cweb" "detex" "dtl" "dvi2tty" "dvicopy" "dvidvi" "dviljk" + "dviout-util" "dvipdfmx" "dvipos" "dvips" "fontware" "gregoriotex" + "gsftopk" "hitex" "kpathsea" "lacheck" "lcdftypetools" "m-tx" "makeindex" + "mfware" "musixtnt" "omegaware" "patgen" "pdftex" "pdftosrc" "pmx" + "ps2eps" "ps2pk" "psutils" "ptex" "seetexk" "synctex" "t1utils" "tex" + "tex4ht" "texlive-scripts-extra" "texware" "tie" "tpic2pdftex" "ttfutils" + "uptex" "velthuis" "vlna" "web" "xml2pmx" ]; + binPackages = lib.getAttrs corePackages tlpdb; + common = { src = fetchurl { urls = [ @@ -56,6 +69,7 @@ let "--disable-missing" "--disable-native-texlive-build" "--enable-shared" # "--enable-cxx-runtime-hack" # static runtime "--enable-tex-synctex" + "--disable-linked-scripts" "-C" # use configure cache to speed up ] ++ withSystemLibs [ @@ -70,6 +84,30 @@ let if [[ ! -x "$f" ]]; then rm "$f"; fi done ''; + + # move binaries to corresponding split outputs, based on content of texlive.tlpdb + binToOutput = lib.listToAttrs + (lib.concatMap + (n: map (v: { name = v; value = builtins.replaceStrings [ "-" ] [ "_" ] n; }) binPackages.${n}.binfiles or [ ]) + (builtins.attrNames binPackages)); + + # (must be run after cleanBrokenLinks) + moveBins = '' + for bin in "$out/bin"/* ; do + bin="''${bin##*/}" + package="''${binToOutput[$bin]}" + if [[ -n "$package" ]] ; then + if [[ -z "''${!package}" ]] ; then + echo "WARNING: missing output '$package' for binary '$bin', leaving in 'out'" + else + mkdir -p "''${!package}"/bin + mv "$out/bin/$bin" "''${!package}"/bin/ + fi + else + echo "WARNING: no output known for binary '$bin', leaving in 'out'" + fi + done + ''; }; # RISC-V: https://github.com/LuaJIT/LuaJIT/issues/628 @@ -84,18 +122,21 @@ core = stdenv.mkDerivation rec { pname = "texlive-bin"; inherit version; - inherit (common) src prePatch; + __structuredAttrs = true; - outputs = [ "out" "doc" "dev" ]; + inherit (common) binToOutput src prePatch; + + outputs = [ "out" "doc" "dev" ] + ++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) corePackages); nativeBuildInputs = [ pkg-config - ] ++ lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + ] ++ lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) (with texlive.bin.core; [ # configure: error: tangle was not found but is required when cross-compiling. # dev (himktables) is used when building hitex to generate the additional source file hitables.c - texlive.bin.core - texlive.bin.core.dev - ]; + web/*tangle*/ cweb/*ctangle*/ omegaware/*otangle*/ tie/*tie*/ # see "Building TeX Live" 6.4.2 Cross problems + dev/*himktables*/ + ]); buildInputs = [ /*teckit*/ zziplib mpfr gmp @@ -181,7 +222,7 @@ core = stdenv.mkDerivation rec { '' + /* install himktables in separate output for use in cross compilation */ '' mkdir -p $dev/bin cp texk/web2c/.libs/himktables $dev/bin/himktables - '' + cleanBrokenLinks; + '' + cleanBrokenLinks + common.moveBins; setupHook = ./setup-hook.sh; # TODO: maybe texmf-nix -> texmf (and all references) passthru = { inherit version buildInputs; }; diff --git a/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix b/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix index 2e9572f168a..b83075a141e 100644 --- a/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix +++ b/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix @@ -118,8 +118,11 @@ let { passthru = commonPassthru // { tlType = "bin"; }; inherit meta; - # shebang interpreters - buildInputs = (args.extraBuildInputs or [ ]) ++ [ bash perl ] ++ (lib.attrVals (args.scriptExts or [ ]) extToInput); + # shebang interpreters and compiled binaries + buildInputs = let outName = builtins.replaceStrings [ "-" ] [ "_" ] pname; in + [ texliveBinaries.${pname} or texliveBinaries.core.${outName} or null ] + ++ (args.extraBuildInputs or [ ]) ++ [ bash perl ] + ++ (lib.attrVals (args.scriptExts or [ ]) extToInput); nativeBuildInputs = extraNativeBuildInputs; # absolute scripts folder scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + args.scriptsFolder or pname); @@ -127,8 +130,6 @@ let inherit (args) binfiles; binlinks = builtins.attrNames (args.binlinks or { }); bintargets = builtins.attrValues (args.binlinks or { }); - binfolders = [ (lib.getBin texliveBinaries.core) ] ++ - lib.optional (texliveBinaries ? ${pname}) (lib.getBin texliveBinaries.${pname}); # build scripts patchScripts = ./patch-scripts.sed; makeBinContainers = ./make-bin-containers.sh; diff --git a/pkgs/tools/typesetting/tex/texlive/default.nix b/pkgs/tools/typesetting/tex/texlive/default.nix index 96bc6f61e4f..a2984fdd49b 100644 --- a/pkgs/tools/typesetting/tex/texlive/default.nix +++ b/pkgs/tools/typesetting/tex/texlive/default.nix @@ -19,6 +19,7 @@ let withIcu = true; withGraphite2 = true; }; inherit useFixedHashes; + tlpdb = overriddenTlpdb; }; # function for creating a working environment from a set of TL packages diff --git a/pkgs/tools/typesetting/tex/texlive/make-bin-containers.sh b/pkgs/tools/typesetting/tex/texlive/make-bin-containers.sh index c293970ac7f..af49ae22e19 100644 --- a/pkgs/tools/typesetting/tex/texlive/make-bin-containers.sh +++ b/pkgs/tools/typesetting/tex/texlive/make-bin-containers.sh @@ -20,14 +20,12 @@ for binname in $binfiles ; do output="$out/bin/$binname" - # look for existing binary from bin.core or bin.${pname} - for folder in $binfolders ; do - target="$folder"/bin/"$binname" - if [[ -f "$target" && -x "$target" ]] ; then - ln -s "$(realpath "$target")" "$output" - continue 2 - fi - done + # look for existing binary from bin.* + target="$(PATH="$HOST_PATH" command -v "$binname" || :)" + if [[ -n "$target" && -x "$target" ]] ; then + ln -s "$(realpath "$target")" "$output" + continue + fi # look for scripts # the explicit list of extensions avoid non-scripts such as $binname.cmd, $binname.jar, $binname.pm From 1f366170426a2bc8fde5c83b5aa64766779bff28 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 02/12] texlive.bin.core: disable texlive scripts --- pkgs/tools/typesetting/tex/texlive/bin.nix | 44 +++++-------------- .../tex/texlive/tlpdb-overrides.nix | 4 +- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index cce8812a355..615f4b3ed2b 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -66,10 +66,13 @@ let configureFlags = [ "--with-banner-add=/nixos.org" - "--disable-missing" "--disable-native-texlive-build" + "--disable-missing" # terminate if a requested program or feature must be + # disabled, e.g., due to missing libraries + "--disable-native-texlive-build" # do not build for the TeX Live binary distribution "--enable-shared" # "--enable-cxx-runtime-hack" # static runtime "--enable-tex-synctex" - "--disable-linked-scripts" + "--disable-texlive" # do not build the texlive (TeX Live scripts) package + "--disable-linked-scripts" # do not install the linked scripts "-C" # use configure cache to speed up ] ++ withSystemLibs [ @@ -185,40 +188,15 @@ core = stdenv.mkDerivation rec { /* links format -> engine will be regenerated in texlive.combine note: for unlinking, the texlinks patch is irrelevant, so we use the included texlinks.sh to avoid the dependency on bin.texlinks */ '' - PATH="$out/bin:$PATH" sh ../texk/texlive/linked_scripts/texlive-extra/texlinks.sh --cnffile "$out/share/texmf-dist/web2c/fmtutil.cnf" --unlink "$out/bin" - '' + /* a few texmf-dist files are useful; take the rest from pkgs */ '' - mv "$out/share/texmf-dist/web2c/texmf.cnf" . - rm -r "$out/share/texmf-dist" - mkdir -p "$out"/share/texmf-dist/{web2c,scripts/texlive/TeXLive} - mv ./texmf.cnf "$out/share/texmf-dist/web2c/" - cp ../texk/tests/TeXLive/*.pm "$out/share/texmf-dist/scripts/texlive/TeXLive/" - cp ../texk/texlive/linked_scripts/scripts.lst "$out/share/texmf-dist/scripts/texlive/" + PATH="$out/bin:$PATH" sh ../texk/texlive/linked_scripts/texlive-extra/texlinks.sh --cnffile "../texk/texlive/tl_support/fmtutil.cnf" --unlink "$out/bin" '' + - (let extraScripts = - '' - tex4ht/ht.sh - tex4ht/htcontext.sh - tex4ht/htcopy.pl - tex4ht/htlatex.sh - tex4ht/htmex.sh - tex4ht/htmove.pl - tex4ht/httex.sh - tex4ht/httexi.sh - tex4ht/htxelatex.sh - tex4ht/htxetex.sh - tex4ht/mk4ht.pl - tex4ht/xhlatex.sh - ''; - in - '' - echo -e 'texmf_scripts="$texmf_scripts\n${extraScripts}"' \ - >> "$out/share/texmf-dist/scripts/texlive/scripts.lst" - '') - + /* doc location identical with individual TeX pkgs */ '' + /* doc location identical with individual TeX pkgs */ '' mkdir -p "$doc/doc" mv "$out"/share/{man,info} "$doc"/doc - '' + /* remove manpages for utils that live in texlive.texlive-scripts to avoid a conflict in buildEnv */ '' - (cd "$doc"/doc/man/man1; rm {fmtutil-sys.1,fmtutil.1,mktexfmt.1,mktexmf.1,mktexpk.1,mktextfm.1,texhash.1,updmap-sys.1,updmap.1}) + '' + /* remove redundant texmf-dist (content provided by TeX Live packages) */ + '' + rm -fr "$out"/share/texmf-dist + rm "$out"/bin/texdoctk # installed even with --disable-linked-scripts '' + /* install himktables in separate output for use in cross compilation */ '' mkdir -p $dev/bin cp texk/web2c/.libs/himktables $dev/bin/himktables diff --git a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix index 4d2ce46ed47..a9439193dca 100644 --- a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix +++ b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix @@ -362,7 +362,7 @@ in lib.recursiveUpdate orig rec { mkdir -p support/texdoc touch support/texdoc/NEWS - TEXMFCNF="${bin.core}"/share/texmf-dist/web2c TEXMF="$out" TEXDOCS=. TEXMFVAR=. \ + TEXMFCNF="${lib.head tl.kpathsea.pkgs}/web2c" TEXMF="$out" TEXDOCS=. TEXMFVAR=. \ "${bin.luatex}"/bin/texlua "$out"/scripts/texdoc/texdoc.tlu \ -c texlive_tlpdb=texlive.tlpdb -lM texdoc @@ -372,7 +372,7 @@ in lib.recursiveUpdate orig rec { # install zsh completion postFixup = '' - TEXMFCNF="${bin.core}"/share/texmf-dist/web2c TEXMF="$scriptsFolder/../.." \ + TEXMFCNF="${lib.head tl.kpathsea.pkgs}"/web2c TEXMF="$scriptsFolder/../.." \ texlua "$out"/bin/texdoc --print-completion zsh > "$TMPDIR"/_texdoc substituteInPlace "$TMPDIR"/_texdoc \ --replace 'compdef __texdoc texdoc' '#compdef texdoc' \ From 06b73a962bccc54df938d88f495a3f95722c67df Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 03/12] texlive.bin.core-big: simplify configure Use the standard '--disable-all-pkgs' approach to build only certain formats instead of running the configure steps manually, which caused dependenc errors in the past. The change also removes the static build of LuaTeX as it creates an additional copy of Kpathsea. --- pkgs/tools/typesetting/tex/texlive/bin.nix | 77 ++++++++-------------- 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 615f4b3ed2b..dce462072f9 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -79,7 +79,8 @@ let # see "from TL tree" vs. "Using installed" in configure output "zziplib" "mpfr" "gmp" "pixman" "potrace" "gd" "freetype2" "libpng" "libpaper" "zlib" - ]; + ] ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) + "BUILDCC=${buildPackages.stdenv.cc.targetPrefix}cc"; # clean broken links to stuff not built cleanBrokenLinks = '' @@ -166,7 +167,6 @@ core = stdenv.mkDerivation rec { depsBuildBuild = [ buildPackages.stdenv.cc ]; configureFlags = common.configureFlags - ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ "BUILDCC=${buildPackages.stdenv.cc.targetPrefix}cc" ] ++ [ "--without-x" ] # disable xdvik and xpdfopen ++ map (what: "--disable-${what}") [ "chktex" @@ -260,60 +260,35 @@ core-big = stdenv.mkDerivation { #TODO: upmendex inherit (core) nativeBuildInputs depsBuildBuild; buildInputs = core.buildInputs ++ [ core cairo harfbuzz icu graphite2 libX11 ]; + /* deleting the unused packages speeds up configure by a considerable margin + and ensures we do not rebuild existing libraries by mistake */ + preConfigure = '' + rm -r libs/{cairo,freetype2,gd,gmp,graphite2,harfbuzz,icu,libpaper,libpng} \ + libs/{mpfr,pixman,xpdf,zlib,zziplib} \ + texk/{afm2pl,bibtex-x,chktex,cjkutils,detex,dtl,dvi2tty,dvidvi,dviljk,dviout-util} \ + texk/{dvipdfm-x,dvipng,dvipos,dvipsk,dvisvgm,gregorio,gsftopk,kpathsea} \ + texk/{lcdf-typetools,makeindexk,makejvf,mendexk,musixtnt,ps2pk,psutils,ptexenc} \ + texk/{seetexk,tex4htk,texlive,ttf2pk2,ttfdump,upmendex,xdvik} \ + utils/{asymptote,autosp,axodraw2,devnag,lacheck,m-tx,pmx,ps2eps,t1utils,texdoctk} \ + utils/{tpic2pdftex,vlna,xindy,xml2pmx,xpdfopen} + mkdir WorkDir + cd WorkDir + ''; + configureFlags = common.configureFlags ++ withSystemLibs [ "kpathsea" "ptexenc" "cairo" "harfbuzz" "icu" "graphite2" ] ++ map (prog: "--disable-${prog}") # don't build things we already have - ([ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "pdftex" + # list from texk/web2c/configure + ([ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "hitex" "pdftex" "web-progs" "synctex" - ] ++ lib.optionals (!withLuaJIT) [ "luajittex" "luajithbtex" "mfluajit" ]); + ] ++ lib.optionals (!withLuaJIT) [ "luajittex" "luajithbtex" "mfluajit" ]) + /* disable all packages, re-enable web2c package */ + ++ [ "--disable-all-pkgs" "--enable-web2c" ] + /* kpathsea requires specifying the kpathsea location manually */ + ++ [ "--with-kpathsea-includes=${core.dev}/include" ]; - configureScript = ":"; + configureScript = "../configure"; - # we use static libtexlua, because it's only used by a single binary - postConfigure = let - luajit = lib.optionalString withLuaJIT ",luajit"; - in - lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) - # without this, the native builds attempt to use the binary - # ${target-triple}-gcc, but we need to use the wrapper script. - '' - export BUILDCC=${buildPackages.stdenv.cc}/bin/cc - '' - + - '' - mkdir ./WorkDir && cd ./WorkDir - for path in libs/{pplib,teckit,lua53${luajit}} texk/web2c; do - ( - if [[ "$path" =~ "libs/lua" ]]; then - extraConfig="--enable-static --disable-shared" - else - extraConfig="" - fi - '' + lib.optionalString (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) - # results of the tests performed by the configure scripts are - # toolchain-dependent, so native components and cross components cannot use - # the same cached test results. - # Disable the caching for components with native subcomponents. - '' - if [[ "$path" =~ "libs/luajit" ]] || [[ "$path" =~ "texk/web2c" ]]; then - extraConfig="$extraConfig --cache-file=/dev/null" - fi - '' - + - '' - mkdir -p "$path" && cd "$path" - "../../../$path/configure" $configureFlags $extraConfig - - if [[ "$path" =~ "libs/luajit" ]] || [[ "$path" =~ "libs/pplib" ]]; then - # ../../../texk/web2c/mfluadir/luapeg/lpeg.h:29:10: fatal error: 'lua.h' file not found - # ../../../texk/web2c/luatexdir/luamd5/md5lib.c:197:10: fatal error: 'utilsha.h' file not found - make ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES}} - fi - ) - done - ''; - - preBuild = "cd texk/web2c"; enableParallelBuilding = true; doCheck = false; # fails @@ -341,7 +316,7 @@ core-big = stdenv.mkDerivation { #TODO: upmendex mv "$out/bin"/{luatex,texlua,texluac} "$luatex/bin/" mv "$out/bin"/luahbtex "$luahbtex/bin/" mv "$out/bin"/xetex "$xetex/bin/" - cp ../../libs/teckit/teckit_compile "$xetex/bin/" + cp libs/teckit/teckit_compile "$xetex/bin/" '' + lib.optionalString withLuaJIT '' mv "$out/bin"/mfluajit{,-nowin} "$mflua/bin/" mv "$out/bin"/{luajittex,luajithbtex,texluajit,texluajitc} "$luajittex/bin/" From fc04385d8c5fd6edabf2ec21afe171dbb61887ab Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 04/12] texlive.bin.core-big: split output according to texlive.tlpdb --- pkgs/tools/typesetting/tex/texlive/bin.nix | 46 ++++++------------- .../tex/texlive/build-texlive-package.nix | 4 +- .../tex/texlive/tlpdb-overrides.nix | 17 ++----- 3 files changed, 21 insertions(+), 46 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index dce462072f9..a30066bf613 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -33,7 +33,9 @@ let "ps2eps" "ps2pk" "psutils" "ptex" "seetexk" "synctex" "t1utils" "tex" "tex4ht" "texlive-scripts-extra" "texware" "tie" "tpic2pdftex" "ttfutils" "uptex" "velthuis" "vlna" "web" "xml2pmx" ]; - binPackages = lib.getAttrs corePackages tlpdb; + coreBigPackages = [ "metafont" "mflua" "metapost" "luatex" "luahbtex" + "xetex" ] ++ lib.optional withLuaJIT "luajittex"; + binPackages = lib.getAttrs (corePackages ++ coreBigPackages) tlpdb; common = { src = fetchurl { @@ -215,12 +217,15 @@ core = stdenv.mkDerivation rec { }; -inherit (core-big) metafont mflua metapost luatex luahbtex luajittex xetex; +inherit (core-big) metafont mflua metapost luatex luahbtex xetex; +luajittex = core.big.luajittex or null; core-big = stdenv.mkDerivation { #TODO: upmendex pname = "texlive-core-big.bin"; inherit version; - inherit (common) src prePatch; + __structuredAttrs = true; + + inherit (common) binToOutput src prePatch; patches = [ # improves reproducibility of fmt files. This patch has been proposed upstream, @@ -293,34 +298,13 @@ core-big = stdenv.mkDerivation { #TODO: upmendex doCheck = false; # fails - # now distribute stuff into outputs, roughly as upstream TL - # (uninteresting stuff remains in $out, typically duplicates from `core`) - outputs = [ - "out" - "metafont" - "mflua" - "metapost" - "luatex" - "luahbtex" - "luajittex" - "xetex" - ]; - postInstall = '' - for output in $(getAllOutputNames); do - mkdir -p "''${!output}/bin" - done - - mv "$out/bin"/{inimf,mf,mf-nowin} "$metafont/bin/" - mv "$out/bin"/mflua{,-nowin} "$mflua/bin/" - mv "$out/bin"/{*tomp,mfplain,*mpost} "$metapost/bin/" - mv "$out/bin"/{luatex,texlua,texluac} "$luatex/bin/" - mv "$out/bin"/luahbtex "$luahbtex/bin/" - mv "$out/bin"/xetex "$xetex/bin/" - cp libs/teckit/teckit_compile "$xetex/bin/" - '' + lib.optionalString withLuaJIT '' - mv "$out/bin"/mfluajit{,-nowin} "$mflua/bin/" - mv "$out/bin"/{luajittex,luajithbtex,texluajit,texluajitc} "$luajittex/bin/" - '' ; + outputs = [ "out" ] + ++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) coreBigPackages) + # some outputs of metapost, omegaware are for ptex/uptex + ++ [ "ptex" "uptex" ] + # unavoidable duplicates from core + ++ [ "ctie" "cweb" "omegaware" "texlive_scripts_extra" "tie" "web" ]; + postInstall = common.moveBins; }; diff --git a/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix b/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix index b83075a141e..c1e98d710b9 100644 --- a/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix +++ b/pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix @@ -120,7 +120,9 @@ let inherit meta; # shebang interpreters and compiled binaries buildInputs = let outName = builtins.replaceStrings [ "-" ] [ "_" ] pname; in - [ texliveBinaries.${pname} or texliveBinaries.core.${outName} or null ] + [ texliveBinaries.core.${outName} or null + texliveBinaries.${pname} or null + texliveBinaries.core-big.${outName} or null ] ++ (args.extraBuildInputs or [ ]) ++ [ bash perl ] ++ (lib.attrVals (args.scriptExts or [ ]) extToInput); nativeBuildInputs = extraNativeBuildInputs; diff --git a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix index a9439193dca..7930e6f903e 100644 --- a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix +++ b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix @@ -13,7 +13,9 @@ let # so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf) removeFormatLinks = lib.mapAttrs (_: attrs: if (attrs ? formats && attrs ? binfiles) - then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine) attrs.formats); + # TLPDB reports erroneously that various metafont binaries like "mf" are format links to engines + # like "mf-nowin"; core-big provides both binaries and links so we simply skip them here + then let formatLinks = lib.catAttrs "name" (lib.filter (f: f.name != f.engine && ! lib.hasSuffix "-nowin" f.engine) attrs.formats); binNotFormats = lib.subtractLists formatLinks attrs.binfiles; in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ] else attrs); @@ -122,12 +124,6 @@ in lib.recursiveUpdate orig rec { epstopdf.binlinks.repstopdf = "epstopdf"; pdfcrop.binlinks.rpdfcrop = "pdfcrop"; - ptex.binlinks = { - pdvitomp = bin.metapost + "/bin/pdvitomp"; - pmpost = bin.metapost + "/bin/pmpost"; - r-pmpost = bin.metapost + "/bin/r-pmpost"; - }; - texdef.binlinks = { latexdef = "texdef"; }; @@ -143,13 +139,6 @@ in lib.recursiveUpdate orig rec { kpsexpand = "kpsetool"; }; - # metapost binaries are in bin.metapost instead of bin.core - uptex.binlinks = { - r-upmpost = bin.metapost + "/bin/r-upmpost"; - updvitomp = bin.metapost + "/bin/updvitomp"; - upmpost = bin.metapost + "/bin/upmpost"; - }; - #### add PATH dependencies without wrappers # TODO deduplicate this code a2ping.postFixup = '' From 87b1ba03db5f27d9b51d489d2ea62e25845ef350 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 05/12] texlive.luajittex: disable binaries on unsupported platforms --- pkgs/tools/typesetting/tex/texlive/default.nix | 2 +- pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/default.nix b/pkgs/tools/typesetting/tex/texlive/default.nix index a2984fdd49b..60e7043e332 100644 --- a/pkgs/tools/typesetting/tex/texlive/default.nix +++ b/pkgs/tools/typesetting/tex/texlive/default.nix @@ -37,7 +37,7 @@ let overriddenTlpdb = let overrides = import ./tlpdb-overrides.nix { inherit - lib bin tlpdb tlpdbxz tl + stdenv lib bin tlpdb tlpdbxz tl installShellFiles coreutils findutils gawk getopt ghostscript_headless gnugrep gnumake gnupg gnused gzip ncurses perl python3 ruby zip; diff --git a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix index 7930e6f903e..cbf5bb89ffa 100644 --- a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix +++ b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix @@ -1,4 +1,4 @@ -{ lib, tlpdb, bin, tlpdbxz, tl +{ stdenv, lib, tlpdb, bin, tlpdbxz, tl , installShellFiles , coreutils, findutils, gawk, getopt, ghostscript_headless, gnugrep , gnumake, gnupg, gnused, gzip, ncurses, perl, python3, ruby, zip @@ -329,6 +329,11 @@ in lib.recursiveUpdate orig rec { #### misc + # RISC-V: https://github.com/LuaJIT/LuaJIT/issues/628 + luajittex.binfiles = lib.optionals + (!(stdenv.hostPlatform.isPower && stdenv.hostPlatform.is64bit) && !stdenv.hostPlatform.isRiscV) + orig.luajittex.binfiles; + # tlpdb lists license as "unknown", but the README says lppl13: http://mirrors.ctan.org/language/arabic/arabi-add/README arabi-add.license = [ "lppl13c" ]; From c13d1f38ea73470a356c69e4b26b1df2473a0c80 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:21 +0100 Subject: [PATCH 06/12] texlive.bin.core: move man, info pages to standard outputs --- pkgs/tools/typesetting/tex/texlive/bin.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index a30066bf613..ca38175807e 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -132,7 +132,7 @@ core = stdenv.mkDerivation rec { inherit (common) binToOutput src prePatch; - outputs = [ "out" "doc" "dev" ] + outputs = [ "out" "dev" "man" "info" ] ++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) corePackages); nativeBuildInputs = [ @@ -191,10 +191,6 @@ core = stdenv.mkDerivation rec { note: for unlinking, the texlinks patch is irrelevant, so we use the included texlinks.sh to avoid the dependency on bin.texlinks */ '' PATH="$out/bin:$PATH" sh ../texk/texlive/linked_scripts/texlive-extra/texlinks.sh --cnffile "../texk/texlive/tl_support/fmtutil.cnf" --unlink "$out/bin" - '' + - /* doc location identical with individual TeX pkgs */ '' - mkdir -p "$doc/doc" - mv "$out"/share/{man,info} "$doc"/doc '' + /* remove redundant texmf-dist (content provided by TeX Live packages) */ '' rm -fr "$out"/share/texmf-dist From b0b94da0dc983e8ad5b5be55e029616a15de8f2c Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:22 +0100 Subject: [PATCH 07/12] texlive.bin.core-big: split man, info, dev outputs --- pkgs/tools/typesetting/tex/texlive/bin.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index ca38175807e..4ec3cadd870 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -294,7 +294,7 @@ core-big = stdenv.mkDerivation { #TODO: upmendex doCheck = false; # fails - outputs = [ "out" ] + outputs = [ "out" "dev" "man" "info" ] ++ (builtins.map (builtins.replaceStrings [ "-" ] [ "_" ]) coreBigPackages) # some outputs of metapost, omegaware are for ptex/uptex ++ [ "ptex" "uptex" ] From 830ec779587620fcfb2c577a7810ad77ac02832e Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:22 +0100 Subject: [PATCH 08/12] texlive.bin: drop cleanBrokenLinks --- pkgs/tools/typesetting/tex/texlive/bin.nix | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 4ec3cadd870..31720ae8022 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -84,20 +84,12 @@ let ] ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "BUILDCC=${buildPackages.stdenv.cc.targetPrefix}cc"; - # clean broken links to stuff not built - cleanBrokenLinks = '' - for f in "$out"/bin/*; do - if [[ ! -x "$f" ]]; then rm "$f"; fi - done - ''; - # move binaries to corresponding split outputs, based on content of texlive.tlpdb binToOutput = lib.listToAttrs (lib.concatMap (n: map (v: { name = v; value = builtins.replaceStrings [ "-" ] [ "_" ] n; }) binPackages.${n}.binfiles or [ ]) (builtins.attrNames binPackages)); - # (must be run after cleanBrokenLinks) moveBins = '' for bin in "$out/bin"/* ; do bin="''${bin##*/}" @@ -120,7 +112,6 @@ let withLuaJIT = !(stdenv.hostPlatform.isPower && stdenv.hostPlatform.is64bit) && !stdenv.hostPlatform.isRiscV; in rec { # un-indented -inherit (common) cleanBrokenLinks; texliveYear = year; @@ -198,7 +189,7 @@ core = stdenv.mkDerivation rec { '' + /* install himktables in separate output for use in cross compilation */ '' mkdir -p $dev/bin cp texk/web2c/.libs/himktables $dev/bin/himktables - '' + cleanBrokenLinks + common.moveBins; + '' + common.moveBins; setupHook = ./setup-hook.sh; # TODO: maybe texmf-nix -> texmf (and all references) passthru = { inherit version buildInputs; }; From e2e33e611c1b862586c66921e3b7a75709bed517 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:27:22 +0100 Subject: [PATCH 09/12] texlive.bin.core-big: enable upmendex --- pkgs/test/texlive/default.nix | 2 +- pkgs/tools/typesetting/tex/texlive/bin.nix | 8 ++++---- pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix | 4 ---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkgs/test/texlive/default.nix b/pkgs/test/texlive/default.nix index e2c714100f9..12c42444f18 100644 --- a/pkgs/test/texlive/default.nix +++ b/pkgs/test/texlive/default.nix @@ -373,7 +373,7 @@ rec { # (1) binaries requiring --help or -h help = [ "arlatex" "bundledoc" "cachepic" "checklistings" "dvipos" "extractres" "fig4latex" "fragmaster" "kpsewhere" "latex-git-log" "ltxfileinfo" "mendex" "perltex" "pn2pdf" "psbook" "psnup" "psresize" "purifyeps" - "simpdftex" "tex2xindy" "texluac" "texluajitc" "urlbst" "yplan" ]; + "simpdftex" "tex2xindy" "texluac" "texluajitc" "upmendex" "urlbst" "yplan" ]; shortHelp = [ "adhocfilelist" "authorindex" "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "ctanupload" "disdvi" "dvibook" "dviconcat" "getmapdl" "latex2man" "listings-ext.sh" "pygmentex" ]; # (2) binaries that return non-zero exit code even if correctly asked for help diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 31720ae8022..2780c1d63f4 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -206,7 +206,7 @@ core = stdenv.mkDerivation rec { inherit (core-big) metafont mflua metapost luatex luahbtex xetex; luajittex = core.big.luajittex or null; -core-big = stdenv.mkDerivation { #TODO: upmendex +core-big = stdenv.mkDerivation { pname = "texlive-core-big.bin"; inherit version; @@ -260,7 +260,7 @@ core-big = stdenv.mkDerivation { #TODO: upmendex texk/{afm2pl,bibtex-x,chktex,cjkutils,detex,dtl,dvi2tty,dvidvi,dviljk,dviout-util} \ texk/{dvipdfm-x,dvipng,dvipos,dvipsk,dvisvgm,gregorio,gsftopk,kpathsea} \ texk/{lcdf-typetools,makeindexk,makejvf,mendexk,musixtnt,ps2pk,psutils,ptexenc} \ - texk/{seetexk,tex4htk,texlive,ttf2pk2,ttfdump,upmendex,xdvik} \ + texk/{seetexk,tex4htk,texlive,ttf2pk2,ttfdump,xdvik} \ utils/{asymptote,autosp,axodraw2,devnag,lacheck,m-tx,pmx,ps2eps,t1utils,texdoctk} \ utils/{tpic2pdftex,vlna,xindy,xml2pmx,xpdfopen} mkdir WorkDir @@ -274,8 +274,8 @@ core-big = stdenv.mkDerivation { #TODO: upmendex ([ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "hitex" "pdftex" "web-progs" "synctex" ] ++ lib.optionals (!withLuaJIT) [ "luajittex" "luajithbtex" "mfluajit" ]) - /* disable all packages, re-enable web2c package */ - ++ [ "--disable-all-pkgs" "--enable-web2c" ] + /* disable all packages, re-enable upmendex, web2c packages */ + ++ [ "--disable-all-pkgs" "--enable-upmendex" "--enable-web2c" ] /* kpathsea requires specifying the kpathsea location manually */ ++ [ "--with-kpathsea-includes=${core.dev}/include" ]; diff --git a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix index cbf5bb89ffa..6b974c72143 100644 --- a/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix +++ b/pkgs/tools/typesetting/tex/texlive/tlpdb-overrides.nix @@ -106,10 +106,6 @@ in lib.recursiveUpdate orig rec { # remove man texlive-scripts.binfiles = lib.remove "man" orig.texlive-scripts.binfiles; - - # upmendex is "TODO" in bin.nix - uptex.binfiles = lib.remove "upmendex" orig.uptex.binfiles; - # xindy is broken on some platforms unfortunately xindy.binfiles = if bin ? xindy then lib.subtractLists [ "xindy.mem" "xindy.run" ] orig.xindy.binfiles From dee562a0cb88b94e0de07e69feba1a53909903be Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Sun, 27 Aug 2023 19:48:35 +0100 Subject: [PATCH 10/12] texlive.bin.core: remove obsolete setup hook --- pkgs/tools/typesetting/tex/texlive/bin.nix | 1 - pkgs/tools/typesetting/tex/texlive/setup-hook.sh | 7 ------- 2 files changed, 8 deletions(-) delete mode 100644 pkgs/tools/typesetting/tex/texlive/setup-hook.sh diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 2780c1d63f4..7bed0175b5d 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -191,7 +191,6 @@ core = stdenv.mkDerivation rec { cp texk/web2c/.libs/himktables $dev/bin/himktables '' + common.moveBins; - setupHook = ./setup-hook.sh; # TODO: maybe texmf-nix -> texmf (and all references) passthru = { inherit version buildInputs; }; meta = with lib; { diff --git a/pkgs/tools/typesetting/tex/texlive/setup-hook.sh b/pkgs/tools/typesetting/tex/texlive/setup-hook.sh deleted file mode 100644 index 5faef7fe5df..00000000000 --- a/pkgs/tools/typesetting/tex/texlive/setup-hook.sh +++ /dev/null @@ -1,7 +0,0 @@ -addTeXMFPath () { - if test -d "$1/share/texmf-nix"; then - export TEXINPUTS="${TEXINPUTS}${TEXINPUTS:+:}$1/share/texmf-nix//:" - fi -} - -addEnvHooks "$targetOffset" addTeXMFPath From 4fcb32e474d82de3df6362cdcd41a8c1c4f84529 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:56:02 +0100 Subject: [PATCH 11/12] texlive.bin.core: remove disabled libraries and packages from the source tree --- pkgs/tools/typesetting/tex/texlive/bin.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 7bed0175b5d..20b50590378 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -151,7 +151,9 @@ core = stdenv.mkDerivation rec { preConfigure = '' rm -r libs/{cairo,freetype2,gd,gmp,graphite2,harfbuzz,icu,libpaper,libpng} \ - libs/{lua53,luajit,mpfr,pixman,zlib,zziplib} + libs/{lua53,luajit,mpfr,pixman,pplib,teckit,zlib,zziplib} \ + texk/{bibtex-x,chktex,dvipng,dvisvgm,upmendex,xdvik} \ + utils/{asymptote,texdoctk,xindy,xpdfopen} mkdir WorkDir cd WorkDir ''; @@ -185,7 +187,6 @@ core = stdenv.mkDerivation rec { '' + /* remove redundant texmf-dist (content provided by TeX Live packages) */ '' rm -fr "$out"/share/texmf-dist - rm "$out"/bin/texdoctk # installed even with --disable-linked-scripts '' + /* install himktables in separate output for use in cross compilation */ '' mkdir -p $dev/bin cp texk/web2c/.libs/himktables $dev/bin/himktables From e27c4d3aa29a5ded1ed397cebf2950d376d96b22 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:57:02 +0100 Subject: [PATCH 12/12] texlive.bin.core: do not run and then undo texlinks --- pkgs/tools/typesetting/tex/texlive/bin.nix | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 20b50590378..ff4adcb78d2 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -176,15 +176,11 @@ core = stdenv.mkDerivation rec { doCheck = false; # triptest fails, likely due to missing TEXMF tree preCheck = "patchShebangs ../texk/web2c"; - installTargets = [ "install" "texlinks" ]; + installTargets = [ "install" ]; # TODO: perhaps improve texmf.cnf search locations postInstall = - /* links format -> engine will be regenerated in texlive.combine - note: for unlinking, the texlinks patch is irrelevant, so we use - the included texlinks.sh to avoid the dependency on bin.texlinks */ '' - PATH="$out/bin:$PATH" sh ../texk/texlive/linked_scripts/texlive-extra/texlinks.sh --cnffile "../texk/texlive/tl_support/fmtutil.cnf" --unlink "$out/bin" - '' + /* remove redundant texmf-dist (content provided by TeX Live packages) */ + /* remove redundant texmf-dist (content provided by TeX Live packages) */ '' rm -fr "$out"/share/texmf-dist '' + /* install himktables in separate output for use in cross compilation */ ''