From 4b5864666ed9155f3c24223525ffc0fda8e811d8 Mon Sep 17 00:00:00 2001
From: Marc Weber <marco-oweber@gmx.de>
Date: Tue, 15 Dec 2009 23:26:52 +0000
Subject: [PATCH] * refactoring kvm test and bootstrapping image.   expose
 makeInfo (used by test now)   expose config hack * Adding tests to
 release.nix * fixes * removing dependency on perl

refactoring details:
Move all configuration modules used by the NixOS installation test script
into one directory.

svn path=/nixos/trunk/; revision=18982
---
 .../tools/installer2/nixos-bootstrap.sh       | 20 +++---
 .../tools/installer2/nixos-prepare-install.sh | 22 ++++--
 .../tools/installer2/run-in-chroot.sh         | 14 ----
 .../README-BOOTSTRAP-NIXOS                    |  4 --
 modules/installer/tools/tools.nix             |  9 +--
 release.nix                                   | 31 +++++---
 .../configuration-iso.nix                     | 26 +++++++
 .../configuration.nix                         | 10 +--
 .../module-insecure.nix                       | 23 +++---
 .../test.nix}                                 | 72 ++++++++++++-------
 tests/test-nixos-installer2.nix               |  4 +-
 11 files changed, 147 insertions(+), 88 deletions(-)
 create mode 100644 tests/test-nixos-install-from-cd/configuration-iso.nix
 rename modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix => tests/test-nixos-install-from-cd/configuration.nix (60%)
 rename modules/installer/cd-dvd/installation-cd-minimal-test-insecure.nix => tests/test-nixos-install-from-cd/module-insecure.nix (71%)
 rename tests/{test-nixos-install-from-cd.nix => test-nixos-install-from-cd/test.nix} (71%)

diff --git a/modules/installer/tools/installer2/nixos-bootstrap.sh b/modules/installer/tools/installer2/nixos-bootstrap.sh
index 97e66bc88ed..ad9baad4dd7 100644
--- a/modules/installer/tools/installer2/nixos-bootstrap.sh
+++ b/modules/installer/tools/installer2/nixos-bootstrap.sh
@@ -15,7 +15,10 @@ usage(){
 
   --debug:    set -x
 
-  -j n   : will be passed to nix-env = number of build tasks done simultaniously
+  options which will be passed to nix-env:
+
+  -j n        : number of build tasks done simultaniously
+  --keep-going: Build as much as possible.
 
   Description:
   This scripts installs NixOS and should be run within the target chroot.
@@ -46,7 +49,7 @@ NIXOS_CONFIG=${NIXOS_CONFIG:-/etc/nixos/configuration.nix}
 NIXOS_PULL=${NIXOS_PULL:-1}
 NIXOS_INSTALL_GRUB=${NIXOS_INSTALL_GRUB:-1} 
 
-ps="with-prepared-chroot.sh"
+ps="run-in-chroot"
 
 check "$NIXOS_CONFIG"
 check "$NIXOS/modules"  "nixos repo not found"
@@ -70,12 +73,13 @@ INSTALL=
 
 for arg in $@; do
   case $arg in
-    --no-pull) NIXOS_PULL=0;;
-    --install) INSTALL=1;;
-    --no-grub) NIXOS_INSTALL_GRUB=;;
-    --debug)   set -x;;
-    -j*)       NIX_ENV_ARGS="$NIX_ENV_ARGS $arg";;
-    *)         usage;
+    --no-pull)     NIXOS_PULL=0;;
+    --install)     INSTALL=1;;
+    --no-grub)     NIXOS_INSTALL_GRUB=;;
+    --debug)       set -x;;
+    -j*)           NIX_ENV_ARGS="$NIX_ENV_ARGS $arg";;
+    --keep-going)  NIX_ENV_ARGS="$NIX_ENV_ARGS $arg";;
+    *)             usage;
   esac
 done
 
diff --git a/modules/installer/tools/installer2/nixos-prepare-install.sh b/modules/installer/tools/installer2/nixos-prepare-install.sh
index b85ef2b9eaa..548581f0c21 100644
--- a/modules/installer/tools/installer2/nixos-prepare-install.sh
+++ b/modules/installer/tools/installer2/nixos-prepare-install.sh
@@ -100,8 +100,6 @@ if ! grep -F -q " $mountPoint " /proc/mounts && [ "$MUST_BE_MOUNTPOINT" = 1 ]; t
     die "$mountPoint doesn't appear to be a mount point"
 fi
 
-# Get the store paths to copy from the references graph.
-storePaths=$(@perl@/bin/perl @pathsFromGraph@ @nixClosure@)
 # = utils =
 
 backup(){
@@ -151,6 +149,19 @@ realise_repo(){
 
 }
 
+# only keep /nix/store/* lines
+# print them only once
+pathsFromGraph(){
+  declare -A a
+  local prefix=/nix/store/
+  while read l; do
+    if [ "${l/#$prefix/}" != "$l" ] && [ -z "${a["$l"]}" ]; then
+      echo "$l";
+      a["$l"]=1;
+    fi
+  done
+}
+
 # = run actions: =
 for a in $ACTIONS; do
   case "$a" in
@@ -173,13 +184,14 @@ for a in $ACTIONS; do
     ;;
 
     copy-nix)
-      if [ -n "$FROM_ARCHIVE" ]; then
+      if [ "$FROM_ARCHIVE" = 1 ]; then
         NIX_CLOSURE=${mountPoint}@nixClosure@
       else
         INFO "Copy Nix to the Nix store on the target device."
         createDirs
         echo "copying Nix to $mountPoint...."
-        for i in $storePaths; do
+
+        for i in `cat $NIX_CLOSURE | pathsFromGraph`; do
             echo "  $i"
             rsync -a $i $mountPoint/nix/store/
         done
@@ -218,7 +230,7 @@ if [ -e "$T/nixos" ] && [ -e "$T/nixpkgs" ] && [ -e "$T/configuration.nix" ]; th
   cat << EOF
     To realise your NixOS installtion execute:
 
-    run-in-chroot "/nix/store/nixos-bootstrap --install"
+    run-in-chroot "/nix/store/nixos-bootstrap --install -j2 --keep-going"
 EOF
 else
   for t in "$T/configuration.nix" "$T/nixpkgs" "$T/configuration.nix"; do
diff --git a/modules/installer/tools/installer2/run-in-chroot.sh b/modules/installer/tools/installer2/run-in-chroot.sh
index 1d4deb921ee..f4cc4cb9523 100644
--- a/modules/installer/tools/installer2/run-in-chroot.sh
+++ b/modules/installer/tools/installer2/run-in-chroot.sh
@@ -11,8 +11,6 @@ usage(){
   --unprepare  : only unmount
   --debug      : set -x
 
-  requisites: perl
-
   $SCRIPT --prepare:   prepare chroot only
   $SCRIPT --unprepare: unprepare chroot only
   $SCRIPT command:     run /bin/sh command in chroot
@@ -27,18 +25,6 @@ EOF
 die(){ echo "!>> " $@; exit 1; }
 INFO(){ echo "INFO: " $@; }
 
-# this is no longer needed because umount -l unmounts rescursive --rbound mounts as well
-#umountUnder() {
-#    local dir="$1"
-#    for i in $(grep -F " $dir" /proc/mounts \
-#        | perl -e 'while (<>) { /^\S+\s+(\S+)\s+/; print "$1\n"; }' \
-#        | sort -r);
-#    do
-#        echo umount $i || true
-#    done
-#}
-
-
 prepare(){
   INFO "Enable networking: copying /etc/resolv.conf"
   mkdir -m 0755 -p $mountPoint/etc
diff --git a/modules/installer/tools/nixos-bootstrap-archive/README-BOOTSTRAP-NIXOS b/modules/installer/tools/nixos-bootstrap-archive/README-BOOTSTRAP-NIXOS
index 041588bb7e4..b70dc010cc3 100644
--- a/modules/installer/tools/nixos-bootstrap-archive/README-BOOTSTRAP-NIXOS
+++ b/modules/installer/tools/nixos-bootstrap-archive/README-BOOTSTRAP-NIXOS
@@ -10,7 +10,3 @@ $EDITOR $mountPoint/etc/nixos/configuration.nix
 
 # you can replace --install by --help to get more info
 bash $mountPoint/nix/store/run-in-chroot "/nix/store/nixos-bootstrap --install"
-
-
-This archive is create by
-nix-build $NIXOS -A config.system.build.minimalInstaller
diff --git a/modules/installer/tools/tools.nix b/modules/installer/tools/tools.nix
index f9027c3448f..7a0e8dbf936 100644
--- a/modules/installer/tools/tools.nix
+++ b/modules/installer/tools/tools.nix
@@ -67,7 +67,6 @@ let
       name = "nixos-prepare-install";
       src = ./installer2/nixos-prepare-install.sh;
 
-      inherit (pkgs) perl pathsFromGraph;
       inherit nix nixClosure nixosBootstrap;
     };
 
@@ -87,8 +86,10 @@ let
       nixpkgsURL = config.installer.nixpkgsURL;
     };
 
-    # see ./nixos-bootstrap-archive/README
-    minimalInstaller = import ./nixos-bootstrap-archive {
+    # see ./nixos-bootstrap-archive/README-BOOTSTRAP-NIXOS
+    # TODO refactor: It should *not* depend on configuration.nix
+    # maybe even move this in nixpkgs?
+    minimalInstallArchive = import ./nixos-bootstrap-archive {
       inherit (pkgs) stdenv runCommand perl pathsFromGraph gnutar coreutils bzip2;
       inherit nixosPrepareInstall runInChroot nixosBootstrap nixClosure;
     };
@@ -129,6 +130,6 @@ in
     inherit nixosInstall;
 
     # expose scripts
-    inherit (installer2) nixosPrepareInstall runInChroot nixosBootstrap minimalInstaller;
+    inherit (installer2) nixosPrepareInstall runInChroot nixosBootstrap minimalInstallArchive;
   };
 }
diff --git a/release.nix b/release.nix
index 8a3b69b7261..412e660857a 100644
--- a/release.nix
+++ b/release.nix
@@ -17,11 +17,13 @@ let
       version = builtins.readFile ./VERSION + (if officialRelease then "" else "pre${toString nixosSrc.rev}");
 
       versionModule = { system.nixosVersion = version; };
-      
-      iso = (import lib/eval-config.nix {
+
+      config = (import lib/eval-config.nix {
         inherit system nixpkgs;
         modules = [ module versionModule ];
-      }).config.system.build.isoImage;
+      }).config;
+      
+      iso = config.system.build.isoImage;
 
     in
       # Declare the ISO as a build product so that it shows up in Hydra.
@@ -31,6 +33,7 @@ let
             maintainers = map (x: lib.getAttr x lib.maintainers) maintainers;
           };
           inherit iso;
+          passthru = { inherit config; };
         }
         ''
           ensureDir $out/nix-support
@@ -40,6 +43,8 @@ let
 
   jobs = rec {
 
+    inherit makeIso; # used by tests/test-nixos-install-from-cd/
+
 
     tarball =
       { nixosSrc ? {outPath = ./.; rev = 1234;}
@@ -90,11 +95,6 @@ let
       description = "minimal";
     };
 
-    iso_minimal_test_insecure = makeIso {
-      module = ./modules/installer/cd-dvd/installation-cd-minimal-test-insecure.nix;
-      description = "minimal-testing-only";
-    };
-
     iso_minimal_fresh_kernel = makeIso {
       module = ./modules/installer/cd-dvd/installation-cd-minimal-fresh-kernel.nix;
       description = "minimal with 2.6.31-zen-branch";
@@ -113,6 +113,9 @@ let
       description = "graphical";
     };
 
+    # Hacky: doesn't depend on configuration. Yet configuration is evaluated (TODO)
+    minimal_install_archive = {system ? "i686-linux"}: (iso_minimal {inherit system;})
+      .config.system.build.minimalInstallArchive;
 
     tests.subversion =
       { services ? ../services }:
@@ -138,6 +141,18 @@ let
         system = "i686-linux";
       }).test;
 
+    ### tests about installing NixOS
+
+    # installs NixOs in a qemu_kvm instance using a tweaked iso.
+    tests.nixosInstallation = 
+      (import ./tests/test-nixos-install-from-cd/test.nix {
+        inherit nixpkgs;
+      }).test;
+
+    # the archive installer can't be tested without chroot which requires being root
+    # options: run in kvm or uml ?
+    # TODO
+
   };
   
 
diff --git a/tests/test-nixos-install-from-cd/configuration-iso.nix b/tests/test-nixos-install-from-cd/configuration-iso.nix
new file mode 100644
index 00000000000..e203734e74c
--- /dev/null
+++ b/tests/test-nixos-install-from-cd/configuration-iso.nix
@@ -0,0 +1,26 @@
+# configuration used to build ISO used to install NixOS when running NixOS kvm installation test
+
+# The configuration is prebuild before starting the vm because starting the vm
+# causes some overhead.
+{pkgs, config, ...}:
+
+let
+  doOverride = pkgs.lib.mkOverride 0 {};
+in
+
+{
+
+  # make system boot and accessible:
+  require = [ ./module-insecure.nix
+              ../../modules/installer/cd-dvd/installation-cd-minimal.nix
+            ];
+
+  fonts = {
+    enableFontConfig = false; 
+  };
+
+  boot.loader.grub.timeout = doOverride 0; 
+  boot.loader.grub.default = 2;
+  boot.loader.grub.version = doOverride 2;
+
+}
diff --git a/modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix b/tests/test-nixos-install-from-cd/configuration.nix
similarity index 60%
rename from modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix
rename to tests/test-nixos-install-from-cd/configuration.nix
index e0dcea6e061..7ec29b4382f 100644
--- a/modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix
+++ b/tests/test-nixos-install-from-cd/configuration.nix
@@ -1,10 +1,10 @@
-# this is the configuration which will be installed.
-# The configuration is prebuild before starting the vm because starting the vm
-# causes some overhead.
+# configuration being installed by NixOS kvm installation test
 {pkgs, config, ...}: {
 
   # make system boot and accessible:
-  require = [ ./installation-cd-minimal-test-insecure.nix ];
+  require = [
+    ./module-insecure.nix
+  ];
 
   boot.loader.grub = {
     device = "/dev/sda";
@@ -19,6 +19,8 @@
     }
   ];
 
+  swapDevices = [ { device = "/dev/sda2"; } ];
+
   fonts = {
     enableFontConfig = false; 
   };
diff --git a/modules/installer/cd-dvd/installation-cd-minimal-test-insecure.nix b/tests/test-nixos-install-from-cd/module-insecure.nix
similarity index 71%
rename from modules/installer/cd-dvd/installation-cd-minimal-test-insecure.nix
rename to tests/test-nixos-install-from-cd/module-insecure.nix
index 2992724373e..68240204ff2 100644
--- a/modules/installer/cd-dvd/installation-cd-minimal-test-insecure.nix
+++ b/tests/test-nixos-install-from-cd/module-insecure.nix
@@ -1,33 +1,29 @@
-# See installation-cd-minimal.nix
-# it's called insecure because it allows logging in as root without password
-# So don't boot this cdrom to install your system :-)
+# this allows logging in as root without password.
 
-{config, pkgs, ...}:
+# This module is shared by the iso configuration and the system configuration
+# which is build by the test
+
+{pkgs, config, ...}:
 
 let
   doOverride = pkgs.lib.mkOverride 0 {};
 in
 
 {
-  require = [ ./installation-cd-minimal.nix ];
 
-  installer.configModule = "./nixos/modules/installer/cd-dvd/installation-cd-minimal-test-insecure";
-
-  services.sshd.permitRootLogin = "yes";
+  services.sshd = {
+    enable = true;
+    permitRootLogin = "yes";
+  };
   jobs.sshd = {
     startOn = doOverride "started network-interfaces";
   };
 
-
   boot.initrd.kernelModules =
     ["cifs" "virtio_net" "virtio_pci" "virtio_blk" "virtio_balloon" "nls_utf8"];
 
   environment.systemPackages = [ pkgs.vim_configurable ];
 
-  boot.loader.grub.timeout = doOverride 0; 
-  boot.loader.grub.default = 2;
-  boot.loader.grub.version = doOverride 2;
-
   # FIXME: rewrite pam.services the to be an attr list
   # I only want to override sshd
   security.pam.services = doOverride
@@ -52,4 +48,3 @@ in
       ];
 
 }
-
diff --git a/tests/test-nixos-install-from-cd.nix b/tests/test-nixos-install-from-cd/test.nix
similarity index 71%
rename from tests/test-nixos-install-from-cd.nix
rename to tests/test-nixos-install-from-cd/test.nix
index f832d6d464f..18b46b82c1b 100644
--- a/tests/test-nixos-install-from-cd.nix
+++ b/tests/test-nixos-install-from-cd/test.nix
@@ -1,8 +1,7 @@
-{ nixos ? ./..
-, nixpkgs ? ../../nixpkgs
-, services ? ../../nixos/services
+{ nixos ? ../..
+, nixpkgs ? ../../../nixpkgs
+, services ? ../../../nixos/services
 , system ? builtins.currentSystem
-, configPath ? ./test-nixos-install-from-cd.nix
 }:
 
 let
@@ -21,22 +20,30 @@ let
    nixosTarball = makeTarball "nixos.tar.bz2" (cleanSource ../../..);
    nixpkgsTarball = makeTarball "nixpkgs.tar.bz2" (cleanSource pkgs.path);
 
+   If test fails at "waiting for socket" rerun the test
+
 */
 
-  isos = (import ../release.nix) { inherit nixpkgs; };
+  configuration_iso =  ./configuration-iso.nix;
+  configuration_install = ./configuration.nix;
 
-  isoFile = 
-    # passed system = systom of iso 
-    (isos.iso_minimal_test_insecure { inherit system; }).iso;
+  release = (import ../../release.nix) { inherit nixpkgs; };
 
-  configuration = ../modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix;
+  isoFile = (
+    release.makeIso 
+      {
+        module = configuration_iso;
+        description = "minimal-testing-only";
+        maintainers = ["MarcWeber"];
+      }
+      { inherit system; }
+      ).iso;
 
-  eval = import ../lib/eval-config.nix {
+  eval = import ../../lib/eval-config.nix {
     inherit system nixpkgs;
-    modules = [ configuration ];
+    modules = [ configuration_install ];
   };
 
-
   inherit (eval) pkgs config;
 
   inherit (pkgs) qemu_kvm;
@@ -57,9 +64,13 @@ rec {
     # FIXME: X shouldn't be required
     # Is there a way to use kvm when not running as root?
     # Would using uml provide any advantages?
+
+    # TODO add nix-env -i command and verify that root can install additional
+    # tools such as git or sshfs-fuse!
+    # run this test when booting both: the iso and the installed system
     pkgs.runCommand "nixos-installation-test" { inherit systemDerivation; } ''
 
-    INFO(){ echo $@; }
+    INFO(){ echo "INFO: " $@; }
 
     die(){ echo $@; exit 1; }
 
@@ -86,7 +97,7 @@ rec {
     cat >> run-kvm.sh << EOF
     #!/bin/sh -e
     # maybe swap should be used ?
-    exec qemu-system-x86_64 -m 2048 \
+    exec qemu-system-x86_64 -m 512 \
         -no-kvm-irqchip \
         -net nic,model=virtio -net user -smb /nix \
         -hda image \
@@ -135,7 +146,10 @@ rec {
 
 
     INFO "creating image file"
-    qemu-img create -f qcow2 image 2G
+    # 2GB = data; 1GB=swap
+    # Maybe using 1GB swap is much. But qcow2 doesn't fill holes so it only
+    # used when required (?)
+    qemu-img create -f qcow2 image 3G
 
     RUN_KVM -boot d -cdrom $(echo ${isoFile}/iso/*.iso)
     
@@ -144,13 +158,22 @@ rec {
     # INSTALLATION
     INFO "creating filesystem .."
     SSH_STDIN_E << EOF
+      set -x
       parted /dev/sda mklabel msdos
       parted /dev/sda mkpart primary 0 2G
-      while [ ! -e /dev/sda1 ]; do
-        echo "waiting for /dev/sda1 to appear"
-        sleep 1;
-      done
+      parted /dev/sda mkpart primary 1G 3G
+      waitFor(){
+        while [ ! -e "\$1" ]; do
+          echo "waiting for \$1 to appear"; sleep 1;
+        done
+      }
+      waitFor /dev/sda2
+      mkswap /dev/sda2
+      swapon /dev/sda2
+
+      waitFor /dev/sda1
       mkfs.ext3 /dev/sda1
+
       mount /dev/sda1 /mnt
       mkdir -p /mnt/nix-on-host
       mount //10.0.2.4/qemu -oguest,username=nobody,noperm -tcifs /mnt/nix-on-host
@@ -162,7 +185,7 @@ rec {
       nixos-hardware-scan > /tmp/test.nix
     EOF
 
-    INFO "copying sources and Nix, preparing configuration, starting installation"
+    INFO "copying sources and Nix, starting installation"
     SSH_STDIN_E << EOF
 
       nixos-prepare-install
@@ -170,12 +193,11 @@ rec {
       # has the generated configuration.nix file syntax errors?
       nix-instantiate --eval-only /tmp/test.nix
 
-      # NixOS sources are in /etc/nixos, copy those configuration files.
-      cp /etc/nixos/nixos/modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix /mnt/etc/nixos/configuration.nix
-      # the configuration file is referencing additional files:
-      cp -r /etc/nixos/nixos/modules/installer/cd-dvd/*.nix /mnt/etc/nixos/
+      echo 'export NIXOS_CONFIG=/etc/nixos/nixos/tests/test-nixos-install-from-cd/configuration.nix' >> /root/.bashrc
+      . ~/.bashrc
       export NIX_OTHER_STORES=/nix-on-host
-      run-in-chroot "/nix/store/nixos-bootstrap --install"
+
+      run-in-chroot "/nix/store/nixos-bootstrap --install --no-pull"
       #nixos-install
     EOF
 
diff --git a/tests/test-nixos-installer2.nix b/tests/test-nixos-installer2.nix
index 7afb9cd892e..ec7ce0965c0 100644
--- a/tests/test-nixos-installer2.nix
+++ b/tests/test-nixos-installer2.nix
@@ -18,8 +18,8 @@ let
 
   isos = (import ../release.nix) { inherit nixpkgs; };
 
-  # using same configuration as ased by kvm test.
-  configuration = ../modules/installer/cd-dvd/test-nixos-install-from-cd-config.nix;
+  # using same configuration as used by kvm test.
+  configuration = ./test-nixos-install-from-cd/configuration.nix;
 
   eval = import ../lib/eval-config.nix {
     inherit system nixpkgs;