Merge remote-tracking branch 'gum/master' into new-populate

19 files changed, 366 insertions, 275 deletions

diff --git a/Makefile b/Makefile
index aa5d5d8ca..3857a2390 100644
--- a/Makefile
+++ b/Makefile
@@ -41,23 +41,25 @@ target_path ?= $(_target_path)
 endif
 endif
 
-export target_host ?= $(system)
-export target_user ?= root
-export target_port ?= 22
-export target_path ?= /var/src
+target_host ?= $(system)
+target_user ?= root
+target_port ?= 22
+target_path ?= /var/src
 
 $(if $(target_host),,$(error unbound variable: target_host))
 $(if $(target_user),,$(error unbound variable: target_user))
 $(if $(target_port),,$(error unbound variable: target_port))
 $(if $(target_path),,$(error unbound variable: target_path))
 
+target ?= $(target_user)@$(target_host):$(target_port)$(target_path)
+
 build = \
 	nix-build \
 		--no-out-link \
 		--show-trace \
 		-I nixos-config=$(nixos-config) \
 		-I stockholm=$(stockholm) \
-		-E "let build = import <stockholm>; in $(1)"
+		-E "with import <stockholm>; $(1)"
 
 evaluate = \
 	nix-instantiate \
@@ -68,26 +70,37 @@ evaluate = \
 		-I stockholm=$(stockholm) \
 		-E "let eval = import <stockholm>; in with eval; $(1)"
 
-execute = \
-	result=$$($(call evaluate,config.krebs.build.$(1))) && \
-	script=$$(echo "$$result" | jq -r .) && \
-	echo "$$script" | PS5=% sh
-
 ifeq ($(MAKECMDGOALS),)
 $(error No goals specified)
 endif
 
 # usage: make deploy system=foo [target_host=bar]
+ifeq ($(debug),true)
+deploy: rebuild-command = dry-activate
+else
+deploy: rebuild-command = switch
+endif
 deploy: ssh ?= ssh
 deploy:
-	$(call execute,populate)
+	$(MAKE) populate debug=false
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
-			nixos-rebuild switch --show-trace -I $(target_path)
+			nixos-rebuild $(rebuild-command) --show-trace -I $(target_path)
+
+# usage: make populate system=foo
+ifeq ($(debug),true)
+populate: populate-flags += --debug
+endif
+ifneq ($(ssh),)
+populate: populate-flags += --ssh=$(ssh)
+endif
+populate:
+	$(call evaluate,config.krebs.build.source) --json --strict | \
+	populate $(target) $(populate-flags)
 
-# usage: make build.pkgs.get
-build build.:;@$(call build,$${expr-eval})
-build.%:;@$(call build,$@)
+# usage: make pkgs.populate
+pkgs:;@$(error no package selected)
+pkgs.%:;@$(call build,$@)
 
 # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
 eval eval.:;@$(call evaluate,$${expr-eval})
@@ -99,7 +112,7 @@ install:
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env target_path=$(target_path) \
 			sh -s prepare < krebs/4lib/infest/prepare.sh
-	target_path=/mnt$(target_path) $(call execute,populate)
+	$(MAKE) populate target_path=/mnt$(target_path)
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env NIXOS_CONFIG=$(target_path)/nixos-config \
 				STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
@@ -117,8 +130,7 @@ $(error bad method: $(method))
 endif
 endif
 test: ssh ?= ssh
-test:
-	$(call execute,populate)
+test: populate
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		$(command) --show-trace -I $(target_path) \
 			-A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 9cd095622..4848748cd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -1,165 +1,27 @@
-{ config, lib, ... }:
+{ config, ... }:
 
 with config.krebs.lib;
 
-let
-  out = {
+{
+  options.krebs.build = {
     # TODO deprecate krebs.build.host
-    options.krebs.build.host = mkOption {
+    host = mkOption {
       type = types.host;
     };
 
-    # TODO make krebs.build.profile shell safe
-    options.krebs.build.profile = mkOption {
-      type = types.str;
+    profile = mkOption {
+      type = types.absolute-path;
       default = "/nix/var/nix/profiles/system";
     };
 
-    # TODO deprecate krebs.build.user
-    options.krebs.build.user = mkOption {
-      type = types.user;
-    };
-
-    options.krebs.build.source = mkOption {
-      type = with types; attrsOf (either str (submodule {
-        options = {
-          url = str;
-          rev = str;
-        };
-      }));
+    source = mkOption {
+      type = types.attrsOf types.source;
       default = {};
     };
 
-    options.krebs.build.populate = mkOption {
-      type = types.str;
-      default = let
-        target-user = maybeEnv "target_user" "root";
-        target-host = maybeEnv "target_host" config.krebs.build.host.name;
-        target-port = maybeEnv "target_port" "22";
-        target-path = maybeEnv "target_path" "/var/src";
-        out = ''
-          #! /bin/sh
-          set -eu
-
-          ssh=''${ssh-ssh}
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          { printf 'PS5=%q%q\n' @ "$PS5"
-            echo ${shell.escape git-script}
-          } | verbose $ssh -p ${shell.escape target-port} \
-                  ${shell.escape "${target-user}@${target-host}"} -T
-
-          unset tmpdir
-          trap '
-            rm -f "$tmpdir"/*
-            rmdir "$tmpdir"
-            trap - EXIT INT QUIT
-          '        EXIT INT QUIT
-          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
-          chmod 0755 "$tmpdir"
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
-            verbose ln -s ${shell.escape symlink.target} \
-                          "$tmpdir"/${shell.escape name}
-          '') source-by-method.symlink)}
-
-          verbose proot \
-              -b "$tmpdir":${shell.escape target-path} \
-              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
-                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
-              ) source-by-method.file)} \
-              rsync \
-                  -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        " (name:
-                    "-f ${shell.escape "R /${name}"}"
-                  ) (attrNames source-by-method.file)} \
-                  --delete \
-                  -vFrlptD \
-                  -e "$ssh -p ${shell.escape target-port}" \
-                  ${shell.escape target-path}/ \
-                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
-        '';
-
-        git-script = ''
-          #! /bin/sh
-          set -efu
-
-          export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          fetch_git() {(
-            dst_dir=$1
-            src_url=$2
-            src_ref=$3
-
-            if ! test -e "$dst_dir"; then
-              git clone "$src_url" "$dst_dir"
-            fi
-
-            cd "$dst_dir"
-
-            if ! url=$(git config remote.origin.url); then
-              git remote add origin "$src_url"
-            elif test "$url" != "$src_url"; then
-              git remote set-url origin "$src_url"
-            fi
-
-            # TODO resolve src_ref to commit hash
-            hash=$src_ref
-
-            if ! test "$(git log --format=%H -1)" = "$hash"; then
-              git fetch origin
-              git checkout "$hash" -- "$dst_dir"
-              git checkout -f "$hash"
-            fi
-
-            git clean -dxf
-          )}
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
-            verbose fetch_git ${concatMapStringsSep " " shell.escape [
-              "${target-path}/${name}"
-              git.url
-              git.rev
-            ]}
-          '') source-by-method.git)}
-        '';
-      in out;
+    # TODO deprecate krebs.build.user
+    user = mkOption {
+      type = types.user;
     };
-
-  };
-
-  source-by-method = let
-    known-methods = ["git" "file" "symlink"];
-  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
-    git = source-by-scheme.http or {} //
-          source-by-scheme.https or {};
   };
-
-  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
-    ${v.scheme}.${k} = v;
-  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
-
-  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
-    path = f (toString x);
-    string = f {
-      url = if substring 0 1 x == "/" then "file://${x}" else x;
-    };
-    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
-      inherit scheme;
-    } // {
-      symlink.target = removePrefix "symlink:" x.url;
-      file.path = # TODO file://host/...
-                  assert hasPrefix "file:///" x.url;
-                  removePrefix "file://" x.url;
-    }.${scheme} or {};
-  }; in f) config.krebs.build.source;
-in out
+}
diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix
index 075066961..d04f1cab2 100644
--- a/krebs/3modules/tv/default.nix
+++ b/krebs/3modules/tv/default.nix
@@ -7,6 +7,30 @@ with config.krebs.lib;
     "viljetic.de" = "regfish";
   };
   hosts = mapAttrs (_: setAttr "owner" config.krebs.users.tv) {
+    alnus = {
+      cores = 2;
+      nets = {
+        retiolum = {
+          ip4.addr = "10.243.21.1";
+          ip6.addr = "42:0:0:0:0:0:0:2101";
+          aliases = [
+            "alnus.r"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIIBCgKCAQEAyDGucukxY1xFSkqDaicpiCXZe3NX1Max7N+E9PKXO2yE0EFoGdUP
+            /4hZFO9IbteDwlsTd/RQIhhUWF818TLWzwasUxgmqBFN4d23IIDLHJxgRZ8cPzAs
+            gmBWwnVWRetDETc6HZK6m2rLU6PG53rRLvheZHW/B9nSfUp7n+puehJdGLnBQ8W+
+            q5d/yUmN8hqS6h62yfAZEJSr7Gh/AW6Irmf3gjKRJlRmD2z28hR5tFH+Q/ulxJXQ
+            rNVzusASjRBO9VYOSWnNWI3Zl9vaUtbtEnvyl3PaV9N3gcHzB2HHlyDIotjqXvxU
+            cPLMN0lWOZeDae/9SDT62l/YuETYQo6TxwIDAQAB
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+      ssh.privkey.path = <secrets/ssh.id_rsa>;
+      ssh.pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDP9JS2Nyjx4Pn+/4MrFi1EvBBYVKkGm2Q4lhgaAiSuiGLol53OSsL2KIo01mbcSSBWow9QpQpn8KDoRnT2aMLDrdTFqL20ztDLOXmtrSsz3flgCjmW4f6uOaoZF0RNjAybd1coqwSJ7EINugwoqOsg1zzN2qeIGKYFvqFIKibYFAnQ8hcksmkvPdIO5O8CbdIiP9sZSrSDp0ZyLK2T0PML2jensVZOeqSPulQDFqLsbmavpVLkpDjdzzPRwbZWNB4++YeipbYNOkX4GR1EB4wMZ93IbBV7kpJtib2Zb2AnUf7UW37hxWBjILdstj9ClwNOQggn8kD9ub7YxBzH1dz0Xd8a0mPOAWIDJz9MypXgFRc3vdvPB/W1I4Se0CLbgOkORun9CkgijKr9oEY8JNt8HFd6viZcAaQxOyIm6PNHZTnHfdSc7bIBS2n3e3IZBv0fTd77knGLXg402aTuu2bm/kxsKivxsILXIaGbeXe4ceN3Fynr3FzSM2bUkzHb0mAHu1BQ9YaX0xzCwjVueA5nzGls7ODSFkXsiBfg2FvMN/sTLFca6tnwyqcnD6nujoiS5+BxjDWPgnZYqCaW3B/IkpTsRMsX6QrfhOFcsP8qlJ2Cp82orWoDK/D0vZ9pdzAc6PFGga0RofuJKY2yiq+SRZ7/e9E6VncIVCYZ1OfN0Q==";
+    };
     caxi = {
       cores = 2;
       extraZones = {
@@ -391,6 +415,9 @@ with config.krebs.lib;
     };
   };
   users = {
+    dv = {
+      mail = "dv@alnus.r";
+    };
     mv = {
       mail = "mv@cd.r";
       pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGer9e2+Lew7vnisgBbsFNECEIkpNJgEaqQqgb9inWkQ mv@vod";
diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix
index aa7b7a9f5..8906eff4a 100644
--- a/krebs/4lib/types.nix
+++ b/krebs/4lib/types.nix
@@ -188,6 +188,75 @@ types // rec {
     };
   });
 
+
+  source = submodule ({ config, ... }: {
+    options = {
+      type = let
+        types = ["file" "git" "symlink"];
+      in mkOption {
+        type = enum types;
+        default = let
+          cands = filter (k: config.${k} != null) types;
+        in
+          if length cands == 1
+            then head cands
+            else throw "cannot determine type";
+      };
+      file = let
+        file-path = (file-source.getSubOptions "FIXME").path.type;
+      in mkOption {
+        type = nullOr (either file-source file-path);
+        default = null;
+        apply = x:
+          if file-path.check x
+            then { path = x; }
+            else x;
+      };
+      git = mkOption {
+        type = nullOr git-source;
+        default = null;
+      };
+      symlink = let
+        symlink-target = (symlink-source.getSubOptions "FIXME").target.type;
+      in mkOption {
+        type = nullOr (either symlink-source symlink-target);
+        default = null;
+        apply = x:
+          if symlink-target.check x
+            then { target = x; }
+            else x;
+      };
+    };
+  });
+
+  file-source = submodule {
+    options = {
+      path = mkOption {
+        type = absolute-pathname;
+      };
+    };
+  };
+
+  git-source = submodule {
+    options = {
+      ref = mkOption {
+        type = str; # TODO types.git.ref
+      };
+      url = mkOption {
+        type = str; # TODO types.git.url
+      };
+    };
+  };
+
+  symlink-source = submodule {
+    options = {
+      target = mkOption {
+        type = pathname; # TODO relative-pathname
+      };
+    };
+  };
+
+
   suffixed-str = suffs:
     mkOptionType {
       name = "string suffixed by ${concatStringsSep ", " suffs}";
diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix
new file mode 100644
index 000000000..09b29c6c3
--- /dev/null
+++ b/krebs/5pkgs/populate/default.nix
@@ -0,0 +1,36 @@
+{ coreutils, fetchgit, git, jq, openssh, proot, rsync, stdenv, ... }:
+
+let
+  PATH = stdenv.lib.makeBinPath [
+    coreutils
+    git
+    jq
+    openssh
+    proot
+    rsync
+  ];
+in
+
+stdenv.mkDerivation rec {
+  name = "populate";
+  version = "1.1.1";
+
+  src = fetchgit {
+    url = http://cgit.cd.krebsco.de/populate;
+    rev = "refs/tags/v${version}";
+    sha256 = "139f4lzn56lca3qgqy9g33r94m3xi1mqns9340lkb4qm6626yvqd";
+  };
+
+  phases = [
+    "unpackPhase"
+    "installPhase"
+  ];
+
+  installPhase = ''
+    sed \
+      '1s,.*,&\nPATH=${PATH},' \
+      -i bin/populate
+
+    cp -r . $out
+  '';
+}
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index ab6bc557c..2a3ebd6fc 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip)
 cat > shared/2configs/temp/dirs.nix <<EOF
 _: {
   krebs.build.source = {
-    secrets = "$krebs_secrets";
-    stockholm = "$(pwd)";
+    secrets.file = "$krebs_secrets";
+    stockholm.file = "$(pwd)";
   };
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     "$(cat ${krebs_ssh}.pub)"
diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix
index ae1416204..7a4df5c7c 100644
--- a/krebs/5pkgs/urlwatch/default.nix
+++ b/krebs/5pkgs/urlwatch/default.nix
@@ -1,11 +1,11 @@
 { stdenv, fetchurl, python3Packages }:
 
 python3Packages.buildPythonPackage rec {
-  name = "urlwatch-2.2";
+  name = "urlwatch-2.5";
 
   src = fetchurl {
     url = "https://thp.io/2008/urlwatch/${name}.tar.gz";
-    sha256 = "0s9056mm1hkj5gpzsb5bz6fwxk0nm73i0dhnqwa1bfddjnvpl9d3";
+    sha256 = "0qirpymdmpsx0klmhbx3icmiwpm6fx4wjma646gl9m90pifs8430";
   };
 
   propagatedBuildInputs = with python3Packages; [
@@ -15,10 +15,6 @@ python3Packages.buildPythonPackage rec {
     requests2
   ];
 
-  patches = [
-    ./setup.patch
-  ];
-
   postFixup = ''
     wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH"
   '';
diff --git a/krebs/5pkgs/urlwatch/setup.patch b/krebs/5pkgs/urlwatch/setup.patch
deleted file mode 100644
index 66626dbf0..000000000
--- a/krebs/5pkgs/urlwatch/setup.patch
+++ /dev/null
@@ -1,42 +0,0 @@
-From ebe7b90100a3d960f53fdc9409d2d89eaa61bf11 Mon Sep 17 00:00:00 2001
-From: Thomas Perl <m@thp.io>
-Date: Tue, 28 Jun 2016 18:15:51 +0200
-Subject: [PATCH] Check current directory and use os.path.relpath (Fixes #73)
-
----
- setup.py | 11 ++++++++---
- 1 file changed, 8 insertions(+), 3 deletions(-)
-
-diff --git a/setup.py b/setup.py
-index 947a7c8..45405cd 100644
---- a/setup.py
-+++ b/setup.py
-@@ -7,10 +7,15 @@
- 
- import os
- import re
-+import sys
- 
- PACKAGE_NAME = 'urlwatch'
- DEPENDENCIES = ['minidb', 'PyYAML', 'requests']
--HERE = os.path.dirname(__file__)
-+HERE = os.path.abspath(os.path.dirname(__file__))
-+
-+if os.path.normpath(os.getcwd()) != os.path.normpath(HERE):
-+    print('You must run {} inside {} (cwd={})'.format(os.path.basename(__file__), HERE, os.getcwd()))
-+    sys.exit(1)
- 
- # Assumptions:
- #  1. Package name equals main script file name (and only one script)
-@@ -29,9 +34,9 @@
- 
- m['scripts'] = [os.path.join(HERE, PACKAGE_NAME)]
- m['package_dir'] = {'': os.path.join(HERE, 'lib')}
--m['packages'] = ['.'.join(dirname[len(HERE)+1:].split(os.sep)[1:])
-+m['packages'] = ['.'.join(os.path.relpath(dirname, HERE).split(os.sep)[1:])
-                  for dirname, _, files in os.walk(os.path.join(HERE, 'lib')) if '__init__.py' in files]
--m['data_files'] = [(dirname[len(HERE)+1:], [os.path.join(dirname[len(HERE)+1:], fn) for fn in files])
-+m['data_files'] = [(os.path.relpath(dirname, HERE), [os.path.join(os.path.relpath(dirname, HERE), fn) for fn in files])
-                    for dirname, _, files in os.walk(os.path.join(HERE, 'share')) if files]
- m['install_requires'] = DEPENDENCIES
- 
diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix
index 12e378667..46a4157ee 100644
--- a/lass/2configs/buildbot-standalone.nix
+++ b/lass/2configs/buildbot-standalone.nix
@@ -64,7 +64,7 @@ in {
 
       # prepare nix-shell
       # the dependencies which are used by the test script
-      deps = [ "gnumake", "jq", "nix", "rsync", "proot" ]
+      deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
       # TODO: --pure , prepare ENV in nix-shell command:
       #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
       nixshell = ["nix-shell",
@@ -112,8 +112,7 @@ in {
         for i in [ "prism", "mors", "echelon" ]:
           addShell(f,name="populate-{}".format(i),env=env_lass,
                   command=nixshell + \
-                            ["{}( make system={} eval.config.krebs.build.populate \
-                               | jq -er .)".format("!" if "failing" in i else "",i)])
+                            ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
 
         addShell(f,name="build-test-minimal",env=env_lass,
                   command=nixshell + \
@@ -146,7 +145,7 @@ in {
     masterhost = "localhost";
     username = "testslave";
     password = "lasspass";
-    packages = with pkgs;[ git nix gnumake jq rsync ];
+    packages = with pkgs; [ gnumake jq nix populate ];
     extraEnviron = {
       NIX_PATH="nixpkgs=/var/src/nixpkgs";
     };
diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix
index 7bf8e82e4..81abff3ed 100644
--- a/lass/2configs/default.nix
+++ b/lass/2configs/default.nix
@@ -53,16 +53,14 @@ with config.krebs.lib;
     search-domain = "retiolum";
     build = {
       user = config.krebs.users.lass;
-      source = mapAttrs (_: mkDefault) ({
-        nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix";
-        secrets = if getEnv "dummy_secrets" == "true"
-          then toString <stockholm/lass/2configs/tests/dummy-secrets>
-          else "/home/lass/secrets/${config.krebs.build.host.name}";
-        #secrets-common = "/home/lass/secrets/common";
-        stockholm = getEnv "PWD";
-      } // optionalAttrs config.krebs.build.host.secure {
-        #secrets-master = "/home/lass/secrets/master";
-      });
+      source = let inherit (config.krebs.build) host; in {
+        nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix";
+        secrets.file =
+          if getEnv "dummy_secrets" == "true"
+            then toString <stockholm/lass/2configs/tests/dummy-secrets>
+            else "/home/lass/secrets/${host.name}";
+        stockholm.file = getEnv "PWD";
+      };
     };
   };
 
diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix
index 8b71b1393..3fcb173ce 100644
--- a/makefu/1systems/vbob.nix
+++ b/makefu/1systems/vbob.nix
@@ -5,23 +5,23 @@
   imports =
     [ # Include the results of the hardware scan.
       ../.
-      <nixpkgs/nixos/modules/virtualisation/virtualbox-image.nix>
+      (toString <nixpkgs/nixos/modules/virtualisation/virtualbox-image.nix>)
+      (toString <nixpkgs/nixos/modules/virtualisation/virtualbox-guest.nix>)
       ../2configs/main-laptop.nix #< base-gui
+      # (toString <secrets>)/extra-hosts.nix
 
       # environment
 
     ];
-  nixpkgs.config.allowUnfree = true;
+  # workaround for https://github.com/NixOS/nixpkgs/issues/16641
+  services.xserver.videoDrivers = lib.mkOverride 45 [ "virtualbox" "modesetting" ];
 
+  nixpkgs.config.allowUnfree = true;
   fileSystems."/nix" = {
     device ="/dev/disk/by-label/nixstore";
     fsType = "ext4";
   };
-  fileSystems."/var/lib/docker" = {
-    device ="/dev/disk/by-label/nix-docker";
-    fsType = "ext4";
-  };
-  #makefu.buildbot.master.enable = true;
+
   # allow vbob to deploy self
   users.extraUsers = {
     root = {
@@ -52,11 +52,7 @@
       "gum"
     ];
   };
-
-  networking.extraHosts = ''
-    172.17.20.190  gitlab
-    172.17.62.27   svbittool01 tool
-  '';
+  virtualisation.docker.enable = false;
 
   fileSystems."/media/share" = {
     fsType = "vboxsf";
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index acd34b0d3..f3bf0c46e 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -18,21 +18,24 @@ with config.krebs.lib;
     enable = true;
 
     dns.providers.siem = "hosts";
+    dns.providers.lan  = "hosts";
     search-domain = "retiolum";
-    build =  {
+    build = {
       user = config.krebs.users.makefu;
-      source =  mapAttrs (_: mkDefault) {
-        nixpkgs = {
+      source = let inherit (config.krebs.build) host user; in {
+        nixpkgs.git = {
           url = https://github.com/nixos/nixpkgs;
           rev = "0546a4a"; # stable @ 2016-06-11
         };
-        secrets = if getEnv "dummy_secrets" == "true"
-                  then toString <stockholm/makefu/6tests/data/secrets>
-                  else "/home/makefu/secrets/${config.krebs.build.host.name}";
-        stockholm = "/home/makefu/stockholm";
+        secrets.file =
+          if getEnv "dummy_secrets" == "true"
+            then toString <stockholm/makefu/6tests/data/secrets>
+            else "/home/makefu/secrets/${host.name}";
+        stockholm.file = "/home/makefu/stockholm";
 
         # Defaults for all stockholm users?
-        nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+        nixos-config.symlink =
+          "stockholm/${user.name}/1systems/${host.name}.nix";
       };
     };
   };
diff --git a/makefu/5pkgs/awesomecfg/kiosk.lua b/makefu/5pkgs/awesomecfg/kiosk.lua
index 81ec99225..ec255a8af 100644
--- a/makefu/5pkgs/awesomecfg/kiosk.lua
+++ b/makefu/5pkgs/awesomecfg/kiosk.lua
@@ -521,13 +521,15 @@ awful.rules.rules = {
 }
 
 -- awful.util.spawn_with_shell("chromium --new-window --kiosk https://www.checkpoint.com/ThreatPortal/livemap.html")
-awful.util.spawn_with_shell("chromium --new-window --kiosk http://wolf:3000/dashboard/db/soc-critical-values")
+--awful.util.spawn_with_shell("chromium --new-window --kiosk http://wolf:3000/dashboard/db/soc-critical-values")
 -- awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://wolf:3000/dashboard/db/aralast")
 --awful.util.spawn_with_shell("chromium --new-window --kiosk http://gast.aramark.de/thales-deutschland/menu/pdf/woche_de.php")
 
-awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://map.norsecorp.com")
+--awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://map.norsecorp.com")
 --awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://threatmap.fortiguard.com")
 
+awful.util.spawn_with_shell("chromium --new-window --kiosk 'https://ossim.siem/ossim/#dashboard/overview/overview'")
+awful.util.spawn_with_shell("chromium --new-window --kiosk 'https://ossim.siem/ossim/#analysis/alarms/alarms'")
 
 -- }}}
 
diff --git a/makefu/5pkgs/git-xlsx-textconv/default.nix b/makefu/5pkgs/git-xlsx-textconv/default.nix
new file mode 100644
index 000000000..1f631f020
--- /dev/null
+++ b/makefu/5pkgs/git-xlsx-textconv/default.nix
@@ -0,0 +1,30 @@
+{ stdenv, lib, goPackages, fetchFromGitHub }:
+let
+  go-xlsx = goPackages.buildGoPackage rec {
+  name = "go-xlsx-${version}";
+  version = "46e6e472d";
+
+  goPackagePath = "github.com/tealeg/xlsx";
+  src = fetchFromGitHub {
+    rev = version;
+    owner = "tealeg";
+    repo = "xlsx";
+    sha256 = "1vls05asms7azhyszbqpgdby9l45jpgisbzzmbrzi30n6cvs89zg";
+  };
+};
+in
+(goPackages.buildGoPackage rec {
+  name = "git-xlsx-textconv-${version}";
+  version = "70685e7f8";
+
+
+  goPackagePath = "github.com/tokuhirom/git-xlsx-textconv";
+
+  src = fetchFromGitHub {
+    rev = version;
+    owner = "tokuhirom";
+    repo = "git-xlsx-textconv";
+    sha256 = "055f3caj1y8v7sc2pz9q0dfyi2ij77d499pby4sjfvm5kjy9msdi";
+  };
+  propagatedBuildInputs = [ go-xlsx ];
+}).bin
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index bbb089c2c..a92a0df35 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -7,15 +7,14 @@ with config.krebs.lib;
 
   # TODO rename shared user to "krebs"
   krebs.build.user = mkDefault config.krebs.users.shared;
-  krebs.build.source = {
-    nixpkgs = mkDefault {
+  krebs.build.source = let inherit (config.krebs.build) host user; in {
+    nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
+    nixpkgs.git = {
       url = https://github.com/NixOS/nixpkgs;
-      rev = "63b9785"; # stable @ 2016-06-01
+      ref = "63b9785"; # stable @ 2016-06-01
     };
-    secrets =  mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
-    stockholm = mkDefault "${getEnv "HOME"}/stockholm";
-
-    nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+    secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
+    stockholm.file = "${getEnv "HOME"}/stockholm";
   };
 
   networking.hostName = config.krebs.build.host.name;
diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix
index 6c40d9966..688f8f9aa 100644
--- a/shared/2configs/shared-buildbot.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -75,7 +75,8 @@
 
   # prepare nix-shell
   # the dependencies which are used by the test script
-  deps = [ "gnumake", "jq","nix","rsync",
+  deps = [ "gnumake", "jq", "nix",
+            "(import <stockholm>).pkgs.populate",
             "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
   # TODO: --pure , prepare ENV in nix-shell command:
   #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
@@ -95,8 +96,7 @@
   for i in [ "test-centos7", "wolf", "test-failing" ]:
     addShell(f,name="populate-{}".format(i),env=env,
             command=nixshell + \
-                      ["{}( make system={} eval.config.krebs.build.populate \
-                         | jq -er .)".format("!" if "failing" in i else "",i)])
+                      ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
 
   # XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
   addShell(f,name="instantiate-test-all-modules",env=env,
@@ -179,7 +179,7 @@
     masterhost = "localhost";
     username = "testslave";
     password = "krebspass";
-    packages = with pkgs;[ git nix gnumake jq rsync ];
+    packages = with pkgs; [ gnumake jq nix populate ];
     # all nix commands will need a working nixpkgs installation