Add a function "fetchzip"
This function downloads and unpacks a file in one fixed-output derivation. This is primarily useful for dynamically generated zip files, such as GitHub's /archive URLs, where the unpacked content of the zip file doesn't change, but the zip file itself may (e.g. due to minor changes in the compression algorithm, or changes in timestamps). Fetchzip is implemented by extending fetchurl with a "postFetch" hook that is executed after the file has been downloaded. This hook can thus perform arbitrary checks or transformations on the downloaded file.
This commit is contained in:
parent
2a43a4163a
commit
c8df888858
|
@ -17,12 +17,16 @@ curl="curl \
|
||||||
$NIX_CURL_FLAGS"
|
$NIX_CURL_FLAGS"
|
||||||
|
|
||||||
|
|
||||||
|
downloadedFile="$out"
|
||||||
|
if [ -n "$downloadToTemp" ]; then downloadedFile="$TMPDIR/file"; fi
|
||||||
|
|
||||||
|
|
||||||
tryDownload() {
|
tryDownload() {
|
||||||
local url="$1"
|
local url="$1"
|
||||||
echo
|
echo
|
||||||
header "trying $url"
|
header "trying $url"
|
||||||
success=
|
success=
|
||||||
if $curl --fail "$url" --output "$out"; then
|
if $curl --fail "$url" --output "$downloadedFile"; then
|
||||||
success=1
|
success=1
|
||||||
fi
|
fi
|
||||||
stopNest
|
stopNest
|
||||||
|
@ -30,6 +34,8 @@ tryDownload() {
|
||||||
|
|
||||||
|
|
||||||
finish() {
|
finish() {
|
||||||
|
set +o noglob
|
||||||
|
runHook postFetch
|
||||||
stopNest
|
stopNest
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
|
@ -54,6 +54,9 @@ in
|
||||||
# first element of `urls').
|
# first element of `urls').
|
||||||
name ? ""
|
name ? ""
|
||||||
|
|
||||||
|
, # A string to be appended to the name, if the name is derived from `url'.
|
||||||
|
nameSuffix ? ""
|
||||||
|
|
||||||
# Different ways of specifying the hash.
|
# Different ways of specifying the hash.
|
||||||
, outputHash ? ""
|
, outputHash ? ""
|
||||||
, outputHashAlgo ? ""
|
, outputHashAlgo ? ""
|
||||||
|
@ -61,6 +64,17 @@ in
|
||||||
, sha1 ? ""
|
, sha1 ? ""
|
||||||
, sha256 ? ""
|
, sha256 ? ""
|
||||||
|
|
||||||
|
, recursiveHash ? false
|
||||||
|
|
||||||
|
, # Shell code executed after the file has been fetched
|
||||||
|
# succesfully. This can do things like check or transform the file.
|
||||||
|
postFetch ? ""
|
||||||
|
|
||||||
|
, # Whether to download to a temporary path rather than $out. Useful
|
||||||
|
# in conjunction with postFetch. The location of the temporary file
|
||||||
|
# is communicated to postFetch via $downloadedFile.
|
||||||
|
downloadToTemp ? false
|
||||||
|
|
||||||
, # If set, don't download the file, but write a list of all possible
|
, # If set, don't download the file, but write a list of all possible
|
||||||
# URLs (resulting from resolving mirror:// URLs) to $out.
|
# URLs (resulting from resolving mirror:// URLs) to $out.
|
||||||
showURLs ? false
|
showURLs ? false
|
||||||
|
@ -83,11 +97,11 @@ stdenv.mkDerivation {
|
||||||
name =
|
name =
|
||||||
if showURLs then "urls"
|
if showURLs then "urls"
|
||||||
else if name != "" then name
|
else if name != "" then name
|
||||||
else baseNameOf (toString (builtins.head urls_));
|
else baseNameOf (toString (builtins.head urls_)) + nameSuffix;
|
||||||
|
|
||||||
builder = ./builder.sh;
|
builder = ./builder.sh;
|
||||||
|
|
||||||
buildInputs = [curl];
|
buildInputs = [ curl ];
|
||||||
|
|
||||||
urls = urls_;
|
urls = urls_;
|
||||||
|
|
||||||
|
@ -101,7 +115,9 @@ stdenv.mkDerivation {
|
||||||
outputHash = if outputHash != "" then outputHash else
|
outputHash = if outputHash != "" then outputHash else
|
||||||
if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
|
if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
|
||||||
|
|
||||||
inherit curlOpts showURLs mirrorsFile impureEnvVars;
|
outputHashMode = if recursiveHash then "recursive" else "flat";
|
||||||
|
|
||||||
|
inherit curlOpts showURLs mirrorsFile impureEnvVars postFetch downloadToTemp;
|
||||||
|
|
||||||
# Doing the download on a remote machine just duplicates network
|
# Doing the download on a remote machine just duplicates network
|
||||||
# traffic, so don't do that.
|
# traffic, so don't do that.
|
||||||
|
|
42
pkgs/build-support/fetchzip/default.nix
Normal file
42
pkgs/build-support/fetchzip/default.nix
Normal file
|
@ -0,0 +1,42 @@
|
||||||
|
# This function downloads and unpacks a zip file. This is primarily
|
||||||
|
# useful for dynamically generated zip files, such as GitHub's
|
||||||
|
# /archive URLs, where the unpacked content of the zip file doesn't
|
||||||
|
# change, but the zip file itself may (e.g. due to minor changes in
|
||||||
|
# the compression algorithm, or changes in timestamps).
|
||||||
|
|
||||||
|
{ lib, fetchurl, unzip }:
|
||||||
|
|
||||||
|
{ # Optionally move the contents of the unpacked tree up one level.
|
||||||
|
stripRoot ? true
|
||||||
|
, ... } @ args:
|
||||||
|
|
||||||
|
fetchurl (args // {
|
||||||
|
# Apply a suffix to the name. Otherwise, unpackPhase will get
|
||||||
|
# confused by the .zip extension.
|
||||||
|
nameSuffix = "-unpacked";
|
||||||
|
|
||||||
|
recursiveHash = true;
|
||||||
|
|
||||||
|
downloadToTemp = true;
|
||||||
|
|
||||||
|
postFetch =
|
||||||
|
''
|
||||||
|
export PATH=${unzip}/bin:$PATH
|
||||||
|
mkdir $out
|
||||||
|
cd $out
|
||||||
|
renamed="$TMPDIR/''${name%-unpacked}"
|
||||||
|
mv "$downloadedFile" "$renamed"
|
||||||
|
unpackFile "$renamed"
|
||||||
|
''
|
||||||
|
# FIXME: handle zip files that contain a single regular file.
|
||||||
|
+ lib.optionalString stripRoot ''
|
||||||
|
shopt -s dotglob
|
||||||
|
if [ "$(ls -d $out/* | wc -l)" != 1 ]; then
|
||||||
|
echo "error: zip file must contain a single directory."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fn=$(cd "$out" && echo *)
|
||||||
|
mv $out/$fn/* "$out/"
|
||||||
|
rmdir "$out/$fn"
|
||||||
|
'';
|
||||||
|
})
|
|
@ -1,13 +1,13 @@
|
||||||
{ stdenv, fetchurl, python, pythonPackages, sysstat, unzip, tornado
|
{ stdenv, fetchzip, python, pythonPackages, sysstat, unzip, tornado
|
||||||
, makeWrapper }:
|
, makeWrapper }:
|
||||||
|
|
||||||
stdenv.mkDerivation rec {
|
stdenv.mkDerivation rec {
|
||||||
version = "4.2.1";
|
version = "4.2.1";
|
||||||
name = "dd-agent-${version}";
|
name = "dd-agent-${version}";
|
||||||
|
|
||||||
src = fetchurl {
|
src = fetchzip {
|
||||||
url = "https://github.com/DataDog/dd-agent/archive/${version}.zip";
|
url = "https://github.com/DataDog/dd-agent/archive/${version}.zip";
|
||||||
sha256 = "0s1lg7rqx86z0y111105gwkknzplq149cxd7v3yg30l22wn68dmv";
|
sha256 = "06f9nkvnpfzs2nw75cac2y9wnp2bay4sg94zz0wjm8886rigjjjm";
|
||||||
};
|
};
|
||||||
|
|
||||||
buildInputs = [ python unzip makeWrapper pythonPackages.psycopg2 ];
|
buildInputs = [ python unzip makeWrapper pythonPackages.psycopg2 ];
|
||||||
|
|
|
@ -338,6 +338,8 @@ let
|
||||||
# linked curl in the case of stdenv-linux).
|
# linked curl in the case of stdenv-linux).
|
||||||
fetchurlBoot = stdenv.fetchurlBoot;
|
fetchurlBoot = stdenv.fetchurlBoot;
|
||||||
|
|
||||||
|
fetchzip = import ../build-support/fetchzip { inherit lib fetchurl unzip; };
|
||||||
|
|
||||||
resolveMirrorURLs = {url}: fetchurl {
|
resolveMirrorURLs = {url}: fetchurl {
|
||||||
showURLs = true;
|
showURLs = true;
|
||||||
inherit url;
|
inherit url;
|
||||||
|
|
Loading…
Reference in a new issue