google-cloud-sdk: add support for components

This commit adds a set of Google Cloud SDK components in
`pkgs.google-cloud-sdk.components` which can be installed along with
the SDK through `pkgs.google-cloud-sdk.withExtraComponents [ ... ]`.

pkgs/tools/admin/google-cloud-sdk/components.nix

@@ -0,0 +1,175 @@
+{ stdenv
+, lib
+, google-cloud-sdk
+, system
+, snapshotPath
+, ...
+}:
+
+let
+  # Mapping from GCS component architecture names to Nix archictecture names
+  arches = {
+    x86 = "i686";
+    x86_64 = "x86_64";
+    # TODO arm
+  };
+
+  # Mapping from GCS component operating systems to Nix operating systems
+  oses = {
+    LINUX = "linux";
+    MACOSX = "darwin";
+    WINDOWS = "windows";
+    CYGWIN = "cygwin";
+  };
+
+  # Convert an archicecture + OS to a Nix platform
+  toNixPlatform = arch: os:
+    let
+      arch' = builtins.getAttr arch arches;
+      os' = builtins.getAttr os oses;
+    in
+    if (builtins.hasAttr arch arches && builtins.hasAttr os oses)
+    then "${arch'}-${os'}"
+    else throw "unsupported architecture '${arch}' and/or os '${os}'";
+
+  # All architectures that are supported
+  allArches = builtins.attrValues arches;
+
+  # A description of all available google-cloud-sdk components.
+  # It's a JSON file with a list of components, along with some metadata
+  snapshot = builtins.fromJSON (builtins.readFile snapshotPath);
+
+  # Generate a snapshot file for a single component.  It has the same format as
+  # `snapshot`, but only contains a single component.  These files are
+  # installed with google-cloud-sdk to let it know which components are
+  # available.
+  snapshotFromComponent =
+    { component
+    , revision
+    , schema_version
+    , version
+    }:
+    builtins.toJSON {
+      components = [ component ];
+      inherit revision schema_version version;
+    };
+
+  # Generate a set of components from a JSON file describing these components
+  componentsFromSnapshot =
+    { components
+    , revision
+    , schema_version
+    , version
+    , ...
+    }:
+    lib.fix (
+      self:
+      builtins.listToAttrs (
+        builtins.map
+          (component: {
+            name = component.id;
+            value = componentFromSnapshot self { inherit component revision schema_version version; };
+          })
+          components
+      )
+    );
+
+  # Generate a single component from its snapshot, along with a set of
+  # available dependencies to choose from.
+  componentFromSnapshot =
+    # Component derivations that can be used as dependencies
+    components:
+    # This component's snapshot
+    { component
+    , revision
+    , schema_version
+    , version
+    } @ attrs:
+    let
+      baseUrl = builtins.dirOf schema_version.url;
+      # Architectures supported by this component.  Defaults to all available
+      # architectures.
+      architectures = builtins.filter
+        (arch: builtins.elem arch (builtins.attrNames arches))
+        (lib.attrByPath [ "platform" "architectures" ] allArches component);
+      # Operating systems supported by this component
+      operating_systems = builtins.filter
+        (os: builtins.elem os (builtins.attrNames oses))
+        component.platform.operating_systems;
+    in
+    mkComponent
+      {
+        name = component.id;
+        version = component.version.version_string;
+        src =
+          if lib.hasAttrByPath [ "data" "source" ] component
+          then "${baseUrl}/${component.data.source}"
+          else "";
+        sha256 = lib.attrByPath [ "data" "checksum" ] "" component;
+        dependencies = builtins.map (dep: builtins.getAttr dep components) component.dependencies;
+        platforms =
+          if component.platform == { }
+          then lib.platforms.all
+          else
+            builtins.concatMap
+              (arch: builtins.map (os: toNixPlatform arch os) operating_systems)
+              architectures;
+        snapshot = snapshotFromComponent attrs;
+      };
+
+  # Filter out dependencies not supported by current system
+  filterForSystem = builtins.filter (drv: builtins.elem system drv.meta.platforms);
+
+  # Make a google-cloud-sdk component
+  mkComponent =
+    { name
+    , version
+      # Source tarball, if any
+    , src ? ""
+      # Checksum for the source tarball, if there is a source
+    , sha256 ? ""
+      # Other components this one depends on
+    , dependencies ? [ ]
+      # Short text describing the component
+    , description ? ""
+      # Platforms supported
+    , platforms ? lib.platforms.all
+      # The snapshot corresponding to this component
+    , snapshot
+    }: stdenv.mkDerivation {
+      inherit name version snapshot;
+      src =
+        if src != "" then
+          builtins.fetchurl
+            {
+              url = src;
+              inherit sha256;
+            } else "";
+      phases = [ "installPhase" "fixupPhase" ];
+      installPhase = ''
+        mkdir -p $out/google-cloud-sdk/.install
+
+        # If there is a source, unpack it
+        if [ ! -z "$src" ]; then
+          tar -xf $src -C $out/google-cloud-sdk/
+
+          # If the source has binaries, link them to `$out/bin`
+          if [ -d "$out/google-cloud-sdk/bin" ]; then
+            mkdir $out/bin
+            find $out/google-cloud-sdk/bin/ -type f -exec ln -s {} $out/bin/ \;
+          fi
+        fi
+
+        # Write the snapshot file to the `.install` folder
+        cp $snapshotPath $out/google-cloud-sdk/.install/${name}.snapshot.json
+      '';
+      passthru = {
+        dependencies = filterForSystem dependencies;
+      };
+      passAsFile = [ "snapshot" ];
+      meta = {
+        inherit description platforms;
+      };
+    };
+in
+componentsFromSnapshot snapshot

pkgs/tools/admin/google-cloud-sdk/data.nix

@@ -28,5 +28,10 @@
         url = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-397.0.0-linux-x86.tar.gz";
         sha256 = "1xw4jzl85didin26q9sg7j1pip4bmap5d0ac9ywbj0vni71mqx26";
       };
+    components =
+      {
+        url = "https://dl.google.com/dl/cloudsdk/channels/rapid/components-v397.0.0.json";
+        sha256 = "1bm3z63lh0z8scfsvjxwvprbrbl5iciz6pbl1hkklxhbc0k9axbc";
+      };
   };
 }

pkgs/tools/admin/google-cloud-sdk/default.nix

@@ -7,7 +7,7 @@
 #   3) used by `google-cloud-sdk` only on GCE guests
 #
 
-{ stdenv, lib, fetchurl, makeWrapper, nixosTests, python, openssl, jq, with-gce ? false }:
+{ stdenv, lib, fetchurl, makeWrapper, nixosTests, python, openssl, jq, callPackage, with-gce ? false }:
 
 let
   pythonEnv = python.withPackages (p: with p; [
@@ -21,6 +21,12 @@ let
   sources = system:
     data.googleCloudSdkPkgs.${system} or (throw "Unsupported system: ${system}");
 
+  components = callPackage ./components.nix {
+    snapshotPath = builtins.fetchurl data.googleCloudSdkPkgs.components;
+  };
+
+  withExtraComponents = callPackage ./withExtraComponents.nix { inherit components; };
+
 in stdenv.mkDerivation rec {
   pname = "google-cloud-sdk";
   inherit (data) version;
@@ -105,6 +111,10 @@ in stdenv.mkDerivation rec {
     $out/bin/gcloud version --format json | jq '."Google Cloud SDK"' | grep "${version}"
   '';
 
+  passthru = {
+    inherit components withExtraComponents;
+  };
+
   meta = with lib; {
     description = "Tools for the google cloud platform";
     longDescription = "The Google Cloud SDK. This package has the programs: gcloud, gsutil, and bq";

pkgs/tools/admin/google-cloud-sdk/update.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env nix-shell
 #! nix-shell -i bash -p nix
 
-BASE_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk"
+CHANNEL_URL="https://dl.google.com/dl/cloudsdk/channels/rapid"
+BASE_URL="$CHANNEL_URL/downloads/google-cloud-sdk"
 
 # Version of Google Cloud SDK from
 # https://cloud.google.com/sdk/docs/release-notes
@@ -17,6 +18,15 @@ function genMainSrc() {
     echo "      };"
 }
 
+function genComponentsSrc() {
+    local url="${CHANNEL_URL}/components-v${VERSION}.json"
+    local sha256
+    sha256=$(nix-prefetch-url "$url")
+    echo "      {"
+    echo "        url = \"${url}\";"
+    echo "        sha256 = \"${sha256}\";"
+    echo "      };"
+}
 {
     cat <<EOF
 # DO NOT EDIT! This file is generated automatically by update.sh
@@ -41,6 +51,9 @@ EOF
     echo "    i686-linux ="
     genMainSrc "linux" "x86"
 
+    echo "    components ="
+    genComponentsSrc
+
     echo "  };"
     echo "}"
 

pkgs/tools/admin/google-cloud-sdk/withExtraComponents.nix

@@ -0,0 +1,61 @@
+{ lib, google-cloud-sdk, callPackage, runCommand, components }:
+
+comps_:
+
+let
+  # Remove components which are already installed by default
+  filterPreInstalled =
+    let
+      preInstalledComponents = with components; [ bq bq-nix core core-nix gcloud-deps gcloud gsutil gsutil-nix ];
+    in
+    builtins.filter (drv: !(builtins.elem drv preInstalledComponents));
+
+  # Recursively build a list of components with their dependencies
+  # TODO this could be made faster, it checks the dependencies too many times
+  findDepsRecursive = lib.converge
+    (drvs: lib.unique (drvs ++ (builtins.concatMap (drv: drv.dependencies) drvs)));
+
+  # Components to install by default
+  defaultComponents = with components; [ alpha beta ];
+
+  comps = [ google-cloud-sdk ] ++ filterPreInstalled (findDepsRecursive (defaultComponents ++ comps_));
+in
+# Components are installed by copying the `google-cloud-sdk` package, along
+# with each component, over to a new location, and then patching that location
+# with `sed` to ensure the proper paths are used.
+# For some reason, this does not work properly with a `symlinkJoin`: the
+# `gcloud` binary doesn't seem able to find the installed components.
+runCommand "google-cloud-sdk-${google-cloud-sdk.version}"
+{
+  inherit (google-cloud-sdk) meta;
+  inherit comps;
+  passAsFile = [ "comps" ];
+
+  doInstallCheck = true;
+  installCheckPhase =
+    let
+      compNames = builtins.map (drv: drv.name) comps_;
+    in
+    ''
+      $out/bin/gcloud components list > component_list.txt
+      for comp in ${builtins.toString compNames}; do
+        if [ ! grep ... component_list.txt | grep "Not Installed" ]; then
+          echo "Failed to install component '$comp'"
+          exit 1
+        fi
+      done
+    '';
+}
+  ''
+    mkdir -p $out
+
+    # Install each component
+    for comp in $(cat $compsPath); do
+      echo "installing component $comp"
+      cp -dRf $comp/. $out
+      find $out -type d -exec chmod 744 {} +
+    done
+
+    # Replace references to the original google-cloud-sdk with this one
+    find $out/google-cloud-sdk/bin/ -type f -exec sed -i -e "s#${google-cloud-sdk}#$out#" {} \;
+  ''