b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

copy-tarballs: use all the urls of each file

Browse source 
If a file specifies multiple urls, try fetching all of them until
nix-prefetch-url is successful.
This commit is contained in:
Vincenzo Mantova 2023-03-26 15:49:49 +01:00
parent a2bd679bd7
commit 8fd7b69da1
2 changed files with 60 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
maintainers/scripts/copy-tarballs.pl
View file

@ -159,13 +159,18 @@ elsif (defined $expr) {
# Check every fetchurl call discovered by find-tarballs.nix.
my $mirrored = 0;
my $have = 0;
foreach my $fetch (sort { $a->{url} cmp $b->{url} } @{$fetches}) {
my $url = $fetch->{url};
foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
my $urls = $fetch->{urls};
my $algo = $fetch->{type};
my $hash = $fetch->{hash};
my $name = $fetch->{name};
my $isPatch = $fetch->{isPatch};
if ($isPatch) {
print STDERR "skipping $urls->[0] (support for patches is missing)\n";
next;
}
if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
$algo = $1;
$hash = `nix hash to-base16 $hash` or die;
@ -180,62 +185,60 @@ elsif (defined $expr) {
chomp $hash;
}
if (defined $ENV{DEBUG}) {
print "$url $algo $hash\n";
next;
}
if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
print STDERR "skipping $url (unsupported scheme)\n";
next;
}
if ($isPatch) {
print STDERR "skipping $url (support for patches is missing)\n";
next;
}
next if defined $exclude && $url =~ /$exclude/;
if (alreadyMirrored($algo, $hash)) {
$have++;
next;
}
my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
for my $url (@$urls) {
if (defined $ENV{DEBUG}) {
print "$url $algo $hash\n";
next;
}
if ($dryRun) {
if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
print STDERR "skipping $url (unsupported scheme)\n";
next;
}
next if defined $exclude && $url =~ /$exclude/;
if (alreadyMirrored($algo, $hash)) {
$have++;
last;
}
print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
if ($dryRun) {
$mirrored++;
last;
}
# Substitute the output.
if (!isValidPath($storePath)) {
system("nix-store", "-r", $storePath);
}
# Otherwise download the file using nix-prefetch-url.
if (!isValidPath($storePath)) {
$ENV{QUIET} = 1;
$ENV{PRINT_PATH} = 1;
my $fh;
my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
waitpid($pid, 0) or die;
if ($? != 0) {
print STDERR "failed to fetch $url: $?\n";
next;
}
<$fh>; my $storePath2 = <$fh>; chomp $storePath2;
if ($storePath ne $storePath2) {
warn "strange: $storePath != $storePath2\n";
next;
}
}
uploadFile($storePath, $url);
$mirrored++;
next;
last;
}
# Substitute the output.
if (!isValidPath($storePath)) {
system("nix-store", "-r", $storePath);
}
# Otherwise download the file using nix-prefetch-url.
if (!isValidPath($storePath)) {
$ENV{QUIET} = 1;
$ENV{PRINT_PATH} = 1;
my $fh;
my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
waitpid($pid, 0) or die;
if ($? != 0) {
print STDERR "failed to fetch $url: $?\n";
next;
}
<$fh>; my $storePath2 = <$fh>; chomp $storePath2;
if ($storePath ne $storePath2) {
warn "strange: $storePath != $storePath2\n";
next;
}
}
uploadFile($storePath, $url);
$mirrored++;
}
print STDERR "mirrored $mirrored files, already have $have files\n";

8
maintainers/scripts/find-tarballs.nix
View file

@ -9,12 +9,12 @@ let
root = expr;
uniqueUrls = map (x: x.file) (genericClosure {
startSet = map (file: { key = file.url; inherit file; }) urls;
uniqueFiles = map (x: x.file) (genericClosure {
startSet = map (file: { key = with file; (if type == null then "" else type + "+") + hash; inherit file; }) files;
operator = const [ ];
});
urls = map (drv: { url = head (drv.urls or [ drv.url ]); hash = drv.outputHash; isPatch = (drv?postFetch && drv.postFetch != ""); type = drv.outputHashAlgo; name = drv.name; }) fetchurlDependencies;
files = map (drv: { urls = drv.urls or [ drv.url ]; hash = drv.outputHash; isPatch = (drv?postFetch && drv.postFetch != ""); type = drv.outputHashAlgo; name = drv.name; }) fetchurlDependencies;
fetchurlDependencies =
filter
@ -47,4 +47,4 @@ let
canEval = val: (builtins.tryEval val).success;
in uniqueUrls
in uniqueFiles