Merge pull request #206763 from xworld21/texlive-centralize-version

texlive: use version info from tlpdb instead of hardcoding

pkgs/test/texlive/default.nix

@@ -2,9 +2,9 @@
 
 {
 
-  tlpdb-nix = runCommand "texlive-test-tlpdb-nix" {
+  tlpdbNix = runCommand "texlive-test-tlpdb-nix" {
     nixpkgsTlpdbNix = ../../tools/typesetting/tex/texlive/tlpdb.nix;
-    tlpdbNix = texlive.tlpdb-nix;
+    tlpdbNix = texlive.tlpdb.nix;
   }
   ''
     mkdir -p "$out"

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -1,62 +1,46 @@
 # Notes on maintaining/upgrading
 
-## Upgrading texlive.bin
+## Upgrading `texlive.bin`
 
-texlive contains a few binaries, defined in bin.nix and released once a year.
+`texlive` contains a few binaries, defined in `bin.nix` and released once a year.
 
 In order to reduce closure size for users who just need a few of them, we split it into
-packages such as core, core-big, xvdi, etc. This requires making assumptions
+packages such as `core`, `core-big`, `xdvi`, etc. This requires making assumptions
 about dependencies between the projects that may change between releases; if
 you upgrade you may have to do some work here.
 
-
 ## Updating the package set
 
-texlive contains several thousand packages from CTAN, defined in pkgs.nix.
+`texlive` contains several thousand packages from CTAN, defined in `tlpdb.nix`.
 
 The CTAN mirrors are not version-controlled and continuously moving,
 with more than 100 updates per month.
 
-To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN
+To create a consistent and reproducible package set in nixpkgs, we generate nix
-and generate nix expressions for all packages in texlive at that point.
+expressions for all packages in TeX Live at a certain day.
-
-We mirror CTAN sources of this snapshot on community-operated servers and on IPFS.
-
-To upgrade the package snapshot, follow this process:
-
-
-### Snapshot sources and texlive package database
-
-Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`).
-See https://tug.org/texlive/acquire-mirror.html for instructions.
 
+To upgrade the package snapshot, follow this process.
 
 ### Upgrade package information from texlive package database
 
-First, edit `default.nix` as follows.
+Update `version` in `default.nix` with the day of the new snapshot, the new TeX
-
+Live year, and the final status of the snapshot. Then update
-If upgrading to a daily snapshot:
+`texlive.tlpdbxz.hash` to match the new hash of `texlive.tlpdb.xz` and run
-- change `snapshot.year`, `snapshot.month`, `snapshot.day`;
-- ensure `urlPrefixes` uses the https://texlive.info/tlnet-archive mirror;
-- ensure `texlive.extraVersion` uses the `snapshot` info.
-
-If upgrading to a final release:
-- upgrade `texlive.bin` first;
-- ensure `urlPrefixes` uses the historic mirrors;
-- ensure `texlive.extraVersion` is `"-final"`.
-
-Then upgrade `tlpdb.hash` to match the new hash of `texlive.tlpdb.xz` and run
 
 ```bash
-nix-build ../../../../.. -A texlive.tlpdb-nix --no-out-link
+nix-build ../../../../.. -A texlive.tlpdb.nix --no-out-link
 ```
 
-This will download the daily snapshot of the CTAN package database
+This will download either the daily or the final snapshot of the TeX Live
-`texlive.tlpdb.xz` and extract the relevant package info (including version
+package database `texlive.tlpdb.xz` and extract the relevant package info
-numbers and sha512 hashes) for the selected upstream distribution. Then replace
+(including version numbers and sha512 hashes) for the selected upstream
-`tlpdb.nix` with the generated file.
+distribution.
 
-The test `pkgs.tests.texlive.tlpdb-nix` verifies that the file `tlpdb.nix`
+Finally, replace `tlpdb.nix` with the generated file. Note that if the
+`version` info does not match the metadata of `tlpdb.nix` (as found in the
+`00texlive.config` package), TeX Live packages will not evaluate.
+
+The test `pkgs.tests.texlive.tlpdbNix` verifies that the file `tlpdb.nix`
 in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.
 
 ### Build packages locally and generate fix hashes
@@ -81,4 +65,4 @@ mv fixedHashes-new.nix fixedHashes.nix
 
 ### Commit changes
 
-Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository.
+Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository.

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -14,7 +14,7 @@
 let
   withSystemLibs = map (libname: "--with-system-${libname}");
 
-  year = "2022";
+  year = toString ((import ./tlpdb.nix)."00texlive.config").year;
   version = year; # keep names simple for now
 
   common = {

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -10,8 +10,8 @@ let
   pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {
     # include a fake "core" package
     core.pkgs = [
-      (bin.core.out // { pname = "core"; version = "0"; tlType = "bin"; })
+      (bin.core.out // { pname = "core"; tlType = "bin"; })
-      (bin.core.doc // { pname = "core"; version = "0"; tlType = "doc"; })
+      (bin.core.doc // { pname = "core"; tlType = "doc"; })
     ];
   };
   pkgList = rec {

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -28,9 +28,13 @@ let
     ghostscript = ghostscript_headless;
   };
 
+  tlpdb = import ./tlpdb.nix;
+
+  tlpdbVersion = tlpdb."00texlive.config";
+
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    orig = import ./tlpdb.nix;
+    orig = removeAttrs tlpdb [ "00texlive.config" ];
 
     overridden = orig // {
       # overrides of texlive.tlpdb
@@ -56,10 +60,12 @@ let
       };
 
       texdoc = orig.texdoc // {
+        version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision);
+
         # build Data.tlpdb.lua (part of the 'tlType == "run"' package)
         postUnpack = ''
           if [[ -f "$out"/scripts/texdoc/texdoc.tlu ]]; then
-            unxz --stdout "${tlpdb}" > texlive.tlpdb
+            unxz --stdout "${tlpdbxz}" > texlive.tlpdb
 
             # create dummy doc file to ensure that texdoc does not return an error
             mkdir -p support/texdoc
@@ -109,11 +115,15 @@ let
             ( bin.${pname} // { tlType = "bin"; } );
     };
 
-  # for daily snapshots
+  version = {
-  snapshot = {
+    # day of the snapshot being taken
-    year = "2022";
+    year = "2023";
-    month = "12";
+    month = "03";
-    day = "27";
+    day = "19";
+    # TeX Live version
+    texliveYear = 2022;
+    # final (historic) release or snapshot
+    final = true;
   };
 
   # The tarballs on CTAN mirrors for the current release are constantly
@@ -121,27 +131,30 @@ let
   # need to be used instead. Ideally, for the release branches of NixOS we
   # should be switching to the tlnet-final versions
   # (https://tug.org/historic/).
-  urlPrefixes = [
+  urlPrefixes = with version; lib.optionals final  [
-    # tlnet-final snapshot
+    # tlnet-final snapshot; used when texlive.tlpdb is frozen
-    "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive"
+    # the TeX Live yearly freeze typically happens in mid-March
-    "ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive"
+    "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final"
-
+    "ftp://tug.org/texlive/historic/${toString texliveYear}/tlnet-final"
-    # Daily snapshots hosted by one of the texlive release managers
+  ] ++ [
-    #"https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive"
+    # daily snapshots hosted by one of the texlive release managers;
+    # used for non-final snapshots and as fallback for final snapshots that have not reached yet the historic mirrors
+    # please note that this server is not meant for large scale deployment and should be avoided on release branches
+    # https://tug.org/pipermail/tex-live/2019-November/044456.html
+    "https://texlive.info/tlnet-archive/${year}/${month}/${day}/tlnet"
   ];
 
-  tlpdb = fetchurl {
+  tlpdbxz = fetchurl {
-    # use the same mirror(s) as urlPrefixes above
+    urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes;
-    urls = map (up: "${up}/../tlpkg/texlive.tlpdb.xz") urlPrefixes;
     hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw=";
   };
 
-  tlpdb-nix = runCommand "tlpdb.nix" {
+  tlpdbNix = runCommand "tlpdb.nix" {
-    inherit tlpdb;
+    inherit tlpdbxz;
     tl2nix = ./tl2nix.sed;
   }
   ''
-    xzcat "$tlpdb" | sed -rn -f "$tl2nix" | uniq > "$out"
+    xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out"
   '';
 
   # create a derivation that contains an unpacked upstream TL package
@@ -153,7 +166,7 @@ let
       fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
-        map (up: "${up}/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
+        map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
 
     in runCommand "texlive-${tlName}"
       ( {
@@ -198,9 +211,21 @@ let
       operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
     });
 
+  assertions =
+    lib.assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
+    lib.assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";
+
 in
   tl // {
-    inherit bin combine tlpdb-nix;
+
+    tlpdb = {
+      # nested in an attribute set to prevent them from appearing in search
+      nix = tlpdbNix;
+      xz = tlpdbxz;
+    };
+
+    bin = assert assertions; bin;
+    combine = assert assertions; combine;
 
     # Pre-defined combined packages for TeX Live schemes,
     # to make nix-env usage more comfortable and build selected on Hydra.
@@ -215,8 +240,7 @@ in
           (combine {
             ${pname} = attrs;
             extraName = "combined" + lib.removePrefix "scheme" pname;
-            #extraVersion = "-final";
+            extraVersion = with version; if final then "-final" else ".${year}${month}${day}";
-            extraVersion = ".${snapshot.year}${snapshot.month}${snapshot.day}";
           })
         )
         { inherit (tl)

pkgs/tools/typesetting/tex/texlive/fixedHashes.nix

@@ -797,8 +797,8 @@
 "texdiff.doc-0.4"="0f7q7cszy6ca6fmwzi7fn6p72szgzyj9fldvzz1rk3ywkhsnpakr";
 "texdirflatten-1.3"="0q2m138gpk9cvfp19jvsw746mkxa613c2204qfxw1bxw8b454d7m";
 "texdirflatten.doc-1.3"="19iiznql82a34d2yhzqcp339zszbhfz9fh0h4mb11696mj8rx3v7";
-"texdoc-4.0.1"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130";
+"texdoc-4.0.1-tlpdb-66590"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130";
-"texdoc.doc-4.0.1"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg";
+"texdoc.doc-4.0.1-tlpdb-66590"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg";
 "texdoctk-0.6.0"="13j0c5m12if8jx8sp7hlirz33y6l2jk8lvnms7dalz52mxz8kq4h";
 "texdoctk.doc-0.6.0"="1k41z2lbfjsar5rl47xnk1adbcnnb4w97bzbgf2rnjhimd1gn9hq";
 "texfot-1.48"="19aswiyr722n9misvg9ajdsgpn1ghmic2xfmr4w0w00sai5sh5rx";

pkgs/tools/typesetting/tex/texlive/tl2nix.sed

@@ -2,6 +2,17 @@
 1i{ # no indentation
 $a}
 
+# extract repository metadata
+/^name 00texlive\.config$/,/^$/{
+  s/^name (.*)$/"\1" = {/p
+  /^$/,1i};
+
+  s!^depend frozen/0$!  frozen = false;!p
+  s!^depend frozen/1$!  frozen = true;!p
+  s!^depend release/(.*)$!  year = \1;!p
+  s!^depend revision/(.*)$!  revision = \1;!p
+}
+
 # form an attrmap per package
 # ignore packages whose name contains "." (such as binaries)
 /^name ([^.]+)$/,/^$/{

pkgs/tools/typesetting/tex/texlive/tlpdb.nix

@@ -1,4 +1,9 @@
 { # no indentation
+"00texlive.config" = {
+  frozen = true;
+  year = 2022;
+  revision = 66590;
+};
 "12many" = {
   revision = 15878;
   stripPrefix = 0;