nuget-to-nix: find sources deterministically
The source used to download a particular package still isn't deterministic in nuget. Even worse, the hash of the package can vary between sources. This makes nuget use the first enabled source containing the package. The order of the dependencies may be slightly different because it now uses glob order of the lower-case package names and versions, instead of sorting the output. If the package actually downloaded was the first source that contains the package, then it will be hashed from disk to avoid downloading it again.
This commit is contained in:
parent
f8763b87e0
commit
b60c9fd2fe
|
@ -172,7 +172,7 @@ stdenvNoCC.mkDerivation (args // {
|
|||
writeShellScript "fetch-${pname}-deps" ''
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}"
|
||||
export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}"
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
|
@ -218,6 +218,8 @@ stdenvNoCC.mkDerivation (args // {
|
|||
-p:Deterministic=true \
|
||||
--packages "$tmp/nuget_pkgs" \
|
||||
--runtime "$rid" \
|
||||
--no-cache \
|
||||
--force \
|
||||
${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \
|
||||
${lib.optionalString (flags != []) (toString flags)}
|
||||
}
|
||||
|
|
|
@ -4,11 +4,12 @@
|
|||
, substituteAll
|
||||
, nix
|
||||
, coreutils
|
||||
, findutils
|
||||
, gnused
|
||||
, jq
|
||||
, yq
|
||||
, curl
|
||||
, gnugrep
|
||||
, gawk
|
||||
, dotnet-sdk
|
||||
}:
|
||||
|
||||
runCommandLocal "nuget-to-nix" {
|
||||
|
@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" {
|
|||
binPath = lib.makeBinPath [
|
||||
nix
|
||||
coreutils
|
||||
findutils
|
||||
gnused
|
||||
jq
|
||||
yq
|
||||
curl
|
||||
gnugrep
|
||||
gawk
|
||||
dotnet-sdk
|
||||
];
|
||||
};
|
||||
|
||||
|
|
|
@ -3,6 +3,8 @@
|
|||
set -euo pipefail
|
||||
|
||||
export PATH="@binPath@"
|
||||
# used for glob ordering of package names
|
||||
export LC_ALL=C
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
>&2 echo "Usage: $0 <packages directory> [path to excluded package source] > deps.nix"
|
||||
|
@ -10,37 +12,73 @@ if [ $# -eq 0 ]; then
|
|||
fi
|
||||
|
||||
pkgs=$1
|
||||
tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX)
|
||||
trap "rm -f ${tmpfile}" EXIT
|
||||
tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)")
|
||||
trap 'rm -r "$tmp"' EXIT
|
||||
excluded_source=$(realpath "${2:-$tmp/empty}")
|
||||
|
||||
declare -A nuget_sources_cache
|
||||
export DOTNET_NOLOGO=1
|
||||
export DOTNET_CLI_TELEMETRY_OPTOUT=1
|
||||
|
||||
mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }')
|
||||
|
||||
declare -A base_addresses
|
||||
|
||||
for index in "${sources[@]}"; do
|
||||
base_addresses[$index]=$(
|
||||
curl --compressed --netrc -fsL "$index" | \
|
||||
jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')
|
||||
done
|
||||
|
||||
echo "{ fetchNuGet }: ["
|
||||
|
||||
while read pkg_spec; do
|
||||
{ read pkg_name; read pkg_version; } < <(
|
||||
# Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3`
|
||||
sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkg_spec")
|
||||
cd "$pkgs"
|
||||
for package in *; do
|
||||
cd "$package"
|
||||
for version in *; do
|
||||
id=$(xq -r .package.metadata.id "$version/$package".nuspec)
|
||||
|
||||
if [[ -e "$excluded_source/${pkg_name}.$pkg_version".nupkg ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)"
|
||||
|
||||
pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")"
|
||||
if [[ -d $pkg_src ]]; then
|
||||
if [[ -e "$excluded_source/$id.$version".nupkg ]]; then
|
||||
continue
|
||||
elif [[ $pkg_src != https://api.nuget.org/* ]]; then
|
||||
pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}"
|
||||
pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg"
|
||||
echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile}
|
||||
else
|
||||
echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile}
|
||||
fi
|
||||
done < <(find $1 -name '*.nuspec')
|
||||
fi
|
||||
|
||||
LC_ALL=C sort --ignore-case ${tmpfile}
|
||||
used_source="$(jq -r '.source' "$version"/.nupkg.metadata)"
|
||||
for source in "${sources[@]}"; do
|
||||
url="${base_addresses[$source]}$package/$version/$package.$version.nupkg"
|
||||
if [[ "$source" == "$used_source" ]]; then
|
||||
sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)"
|
||||
found=true
|
||||
break
|
||||
else
|
||||
if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then
|
||||
# If multiple remote sources are enabled, nuget will try them all
|
||||
# concurrently and use the one that responds first. We always use the
|
||||
# first source that has the package.
|
||||
echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2
|
||||
found=true
|
||||
break
|
||||
else
|
||||
if ! grep -q 'HTTP error 404' "$tmp/error"; then
|
||||
cat "$tmp/error" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "]"
|
||||
if ! ${found-false}; then
|
||||
echo "couldn't find $package $version" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then
|
||||
echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })"
|
||||
else
|
||||
echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })"
|
||||
fi
|
||||
done
|
||||
cd ..
|
||||
done
|
||||
|
||||
cat << EOL
|
||||
]
|
||||
EOL
|
||||
|
|
|
@ -302,12 +302,12 @@ stdenv.mkDerivation rec {
|
|||
# Inspired by passthru.fetch-deps in pkgs/build-support/build-dotnet-module/default.nix
|
||||
passthru.createDepsFile = writeShellApplication {
|
||||
name = "create-deps-file";
|
||||
runtimeInputs = [ dotnetSdk nuget-to-nix ];
|
||||
runtimeInputs = [ dotnetSdk (nuget-to-nix.override { dotnet-sdk = dotnetSdk; }) ];
|
||||
text = ''
|
||||
# Disable telemetry data
|
||||
export DOTNET_CLI_TELEMETRY_OPTOUT=1
|
||||
|
||||
rundir=$(pwd)
|
||||
deps_file="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")"
|
||||
|
||||
printf "\n* Setup workdir\n"
|
||||
workdir="$(mktemp -d /tmp/${pname}.XXX)"
|
||||
|
@ -324,8 +324,6 @@ stdenv.mkDerivation rec {
|
|||
dotnet restore src/ActionsRunner.sln --packages nuget_pkgs --no-cache --force --runtime "${rid}"
|
||||
'') (lib.attrValues runtimeIds)}
|
||||
|
||||
cd "$rundir"
|
||||
deps_file=''${1-"/tmp/${pname}-deps.nix"}
|
||||
printf "\n* Make %s file\n" "$(basename "$deps_file")"
|
||||
nuget-to-nix "$workdir/nuget_pkgs" > "$deps_file"
|
||||
printf "\n* Dependency file writen to %s" "$deps_file"
|
||||
|
|
Loading…
Reference in a new issue