From bf1b6482ce3535ef7e7b3f77879def12ff454c0c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 19:36:19 +0100
Subject: mv makefu->krebs 3 buildbot

---
 krebs/3modules/buildbot/master.nix | 263 +++++++++++++++++++++++++++++++++++++
 krebs/3modules/buildbot/slave.nix  | 185 ++++++++++++++++++++++++++
 krebs/3modules/default.nix         |   2 +
 3 files changed, 450 insertions(+)
 create mode 100644 krebs/3modules/buildbot/master.nix
 create mode 100644 krebs/3modules/buildbot/slave.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
new file mode 100644
index 000000000..2f73e44bc
--- /dev/null
+++ b/krebs/3modules/buildbot/master.nix
@@ -0,0 +1,263 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+    import re
+
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    # TODO: template potential buildslaves
+    # TODO: set password?
+    slavenames= [ 'testslave' ]
+    for i in slavenames:
+      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    stockholm_repo = 'http://cgit.gum/stockholm'
+    c['change_source'] = []
+    c['change_source'].append(changes.GitPoller(
+            stockholm_repo,
+            workdir='stockholm-poller', branch='master',
+            project='stockholm',
+            pollinterval=120))
+
+    ####### Build Scheduler
+    # TODO: configure scheduler
+    c['schedulers'] = []
+
+    # test the master real quick
+    fast = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                name="fast-master-test",
+                                builderNames=["fast-tests"])
+
+    force = schedulers.ForceScheduler(
+                                name="force",
+                                builderNames=["full-tests"])
+
+    # files everyone depends on or are part of the share branch
+    def shared_files(change):
+      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+      for file in change.files:
+        if r.match(file):
+          return True
+      return False
+
+    full = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                fileIsImportant=shared_files,
+                                name="full-master-test",
+                                builderNames=["full-tests"])
+    c['schedulers'] = [ fast, force, full ]
+    ###### The actual build
+    # couple of fast steps:
+    f = util.BuildFactory()
+    ## fetch repo
+    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+    f.addStep(grab_repo)
+
+    # the dependencies which are used by the test script
+    deps = [ "gnumake", "jq" ]
+    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    def addShell(f,**kwargs):
+      f.addStep(steps.ShellCommand(**kwargs))
+
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=wolf"])
+
+    c['builders'] = []
+    c['builders'].append(
+        util.BuilderConfig(name="fast-tests",
+          slavenames=slavenames,
+          factory=f))
+
+    # TODO slow build
+    c['builders'].append(
+        util.BuilderConfig(name="full-tests",
+          slavenames=slavenames,
+          factory=f))
+
+    ####### Status of Builds
+    c['status'] = []
+
+    from buildbot.status import html
+    from buildbot.status.web import authz, auth
+    # TODO: configure if http is wanted
+    authz_cfg=authz.Authz(
+        # TODO: configure user/pw
+        auth=auth.BasicAuth([("krebs","bob")]),
+        gracefulShutdown = False,
+        forceBuild = 'auth',
+        forceAllBuilds = 'auth',
+        pingBuilder = False,
+        stopBuild = False,
+        stopAllBuilds = False,
+        cancelPendingBuild = False,
+    )
+    # TODO: configure nginx
+    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+    from buildbot.status import words
+    ${optionalString (cfg.irc.enable) ''
+      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
+                      # TODO: multiple channels
+                      channels=["${cfg.irc.channel}"],
+                      notify_events={
+                        #'success': 1,
+                        #'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
+
+    ####### PROJECT IDENTITY
+    c['title'] = "Stockholm"
+    c['titleURL'] = "http://krebsco.de"
+
+    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
+    # TODO: configure url
+    c['buildbotURL'] = "http://vbob:8010/"
+
+    ####### DB URL
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.krebs.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channel = mkOption {
+            default = "nix-buildbot-meetup";
+            type = types.str;
+            description = ''
+              irc channel the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = 672626386; #genid buildbotMaster
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.git ];
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+        # TODO: check if git is the only dep
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+          fi
+          # always override the master.cfg
+          cp ${buildbot-master-config} ${workdir}/master.cfg
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chmod 700 -R ${workdir}
+          chown buildbotMaster:buildbotMaster -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
new file mode 100644
index 000000000..65291f63e
--- /dev/null
+++ b/krebs/3modules/buildbot/slave.nix
@@ -0,0 +1,185 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
+  default-packages = [ pkgs.git pkgs.bash ];
+  cfg = config.krebs.buildbot.slave;
+
+  api = {
+    enable = mkEnableOption "Buildbot Slave";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/slave";
+      type = types.str;
+      description = ''
+        Path to build bot slave directory.
+        Will be created on startup.
+      '';
+    };
+
+    masterhost = mkOption {
+      default = "localhost";
+      type = types.str;
+      description = ''
+        Hostname/IP of the buildbot master
+      '';
+    };
+
+    username = mkOption {
+      type = types.str;
+      description = ''
+        slavename used to authenticate with master
+      '';
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        slave password used to authenticate with master
+      '';
+    };
+
+    contact = mkOption {
+      default = "nix slave <buildslave@${config.networking.hostName}>";
+      type = types.str;
+      description = ''
+        contact to be announced by buildslave
+      '';
+    };
+
+    description = mkOption {
+      default = "Nix Generated BuildSlave";
+      type = types.str;
+      description = ''
+        description for hostto be announced by buildslave
+      '';
+    };
+
+    packages = mkOption {
+      default = [ pkgs.git ];
+      type = with types; listOf package;
+      description = ''
+        packages which should be in path for buildslave
+      '';
+    };
+
+    extraEnviron = mkOption {
+      default = {};
+      example = {
+        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
+      };
+      type = types.attrsOf types.str;
+      description = ''
+        extra environment variables to be provided to the buildslave service
+        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        port = 443
+        keepalive = 600
+      '';
+      description = ''
+        extra config evaluated before calling BuildSlave init in .tac file
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotSlave = {
+      uid = 1408105834; #genid buildbotMaster
+      description = "Buildbot Slave";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
+    };
+
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = default-packages ++ cfg.packages;
+
+      environment = {
+          NIX_REMOTE="daemon";
+      } // cfg.extraEnviron;
+
+      serviceConfig = let
+        workdir = "${lib.shell.escape cfg.workDir}";
+        contact = "${lib.shell.escape cfg.contact}";
+        description = "${lib.shell.escape cfg.description}";
+        buildbot = pkgs.buildbot-slave;
+        # TODO:make this
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          mkdir -p ${workdir}/info
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          echo ${contact} > ${workdir}/info/admin
+          echo ${description} > ${workdir}/info/host
+
+          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chmod 700 -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotSlave";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.slave = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..cbc1291fa 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -9,6 +9,8 @@ let
       ./apt-cacher-ng.nix
       ./bepasty-server.nix
       ./build.nix
+      ./buildbot/master.nix
+      ./buildbot/slave.nix
       ./current.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
-- 
cgit v1.2.3


From 56e8346faa75fc42f65d11ea3569a3e5bdd252ec Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 20:53:11 +0100
Subject: k 5 krebs-ci: remove obsolete trap rm

---
 krebs/3modules/buildbot/master.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 2f73e44bc..e66e0d6b2 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -44,7 +44,7 @@ let
 
     # files everyone depends on or are part of the share branch
     def shared_files(change):
-      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+      r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
       for file in change.files:
         if r.match(file):
           return True
-- 
cgit v1.2.3


From f59080e76f950a5a8e33d1edd4314ffaa14187fc Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: m 3 buildbot: add new slow factory to complete integration test

---
 krebs/3modules/buildbot/master.nix | 47 +++++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 16 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..0d9c53977 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
     ###### The actual build
     # couple of fast steps:
     f = util.BuildFactory()
+    # some slow steps
+    s = util.BuildFactory()
     ## fetch repo
     grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
     f.addStep(grab_repo)
+    s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq" ]
-    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    nixshell = ["nix-shell", "-I", ".", "-p" ] + deps + [ "--run" ]
+
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=test-centos7"])
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=wolf"])
+    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
     c['builders'].append(
@@ -87,11 +88,20 @@ let
           slavenames=slavenames,
           factory=f))
 
-    # TODO slow build
+    # slave needs 2 files:
+    # * cac.json
+    # * retiolum
+    for file in ["cac.json", "retiolum.rsa_key.priv"]:
+      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+                              slavedest=file))
+
+    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+             command=nix-shell + ["krebs-ci"])
+
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
           slavenames=slavenames,
-          factory=f))
+          factory=s))
 
     ####### Status of Builds
     c['status'] = []
@@ -119,8 +129,8 @@ let
                       # TODO: multiple channels
                       channels=["${cfg.irc.channel}"],
                       notify_events={
-                        #'success': 1,
-                        #'failure': 1,
+                        'success': 1,
+                        'failure': 1,
                         'exception': 1,
                         'successToFailure': 1,
                         'failureToSuccess': 1,
@@ -221,6 +231,7 @@ let
       path = [ pkgs.git ];
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
         # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
@@ -236,6 +247,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+          # copy secrets
+          cp ${secretsdir}/cac.json ${workdir}
+          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+             ${workdir}/retiolum.rsa_key.priv
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
-- 
cgit v1.2.3


From dc8e270d2a5346e4316b7c2050b26fd428ec3fc3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: m 3 buildbot: add new slow factory to complete integration test

---
 krebs/3modules/buildbot/master.nix | 52 +++++++++++++++++++++++++-------------
 krebs/3modules/buildbot/slave.nix  |  1 +
 2 files changed, 36 insertions(+), 17 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..b4fd6bb2f 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
     ###### The actual build
     # couple of fast steps:
     f = util.BuildFactory()
+    # some slow steps
+    s = util.BuildFactory()
     ## fetch repo
     grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
     f.addStep(grab_repo)
+    s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq" ]
-    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=test-centos7"])
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=wolf"])
+    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
     c['builders'].append(
@@ -87,11 +88,20 @@ let
           slavenames=slavenames,
           factory=f))
 
-    # TODO slow build
+    # slave needs 2 files:
+    # * cac.json
+    # * retiolum
+    for file in ["cac.json", "retiolum.rsa_key.priv"]:
+      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+                              slavedest=file))
+
+    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+             command=nixshell + ["krebs-ci"])
+
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
           slavenames=slavenames,
-          factory=f))
+          factory=s))
 
     ####### Status of Builds
     c['status'] = []
@@ -106,7 +116,7 @@ let
         forceBuild = 'auth',
         forceAllBuilds = 'auth',
         pingBuilder = False,
-        stopBuild = False,
+        stopBuild = 'auth',
         stopAllBuilds = False,
         cancelPendingBuild = False,
     )
@@ -119,8 +129,8 @@ let
                       # TODO: multiple channels
                       channels=["${cfg.irc.channel}"],
                       notify_events={
-                        #'success': 1,
-                        #'failure': 1,
+                        'success': 1,
+                        'failure': 1,
                         'exception': 1,
                         'successToFailure': 1,
                         'failureToSuccess': 1,
@@ -219,8 +229,12 @@ let
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
       path = [ pkgs.git ];
+      environment = {
+        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
         # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
@@ -236,6 +250,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+          # copy secrets
+          cp ${secretsdir}/cac.json ${workdir}
+          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+             ${workdir}/retiolum.rsa_key.priv
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 65291f63e..8711a287a 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -144,6 +144,7 @@ let
       path = default-packages ++ cfg.packages;
 
       environment = {
+          SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
           NIX_REMOTE="daemon";
       } // cfg.extraEnviron;
 
-- 
cgit v1.2.3


From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 11:18:00 +0100
Subject: k 5 mv krebs-ci test/infest-cac-centos7

---
 krebs/3modules/buildbot/master.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index b4fd6bb2f..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,7 +67,7 @@ let
     s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
     nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
 
     def addShell(f,**kwargs):
@@ -95,8 +95,8 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
-             command=nixshell + ["krebs-ci"])
+    addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
+             command=nixshell + ["infest-cac-centos7"])
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
-- 
cgit v1.2.3


From 9a386718714d70f7100b5de297dfd0869d98e47b Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:06:41 +0100
Subject: k 3 buildbot: fix merge fuckup

---
 krebs/3modules/buildbot/master.nix | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 19aecead1..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,13 +95,8 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-<<<<<<< HEAD
     addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
              command=nixshell + ["infest-cac-centos7"])
-=======
-    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
-             command=nix-shell + ["krebs-ci"])
->>>>>>> f59080e76f950a5a8e33d1edd4314ffaa14187fc
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
-- 
cgit v1.2.3


From 1ef9af2c9a49490a2dda21884ad761675c520d1a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:20:27 +0100
Subject: k 3 buildbot/master: send sigterm before sigkill for cleanup

---
 krebs/3modules/buildbot/master.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 483ba18e7..6ce708769 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,8 +95,10 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
-             command=nixshell + ["infest-cac-centos7"])
+    addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+                sigtermTime=60, # SIGTERM 1 minute before SIGKILL
+                timeout=5400,   # 1.5h timeout
+                command=nixshell + ["infest-cac-centos7"])
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
-- 
cgit v1.2.3


From 6b7506dc672b4bd658088bf37fad06fd64c777fe Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:37:02 +0100
Subject: k 3 buildbot: add rsync as explicit dep

do not be pure yet
---
 krebs/3modules/buildbot/master.nix | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6ce708769..6bf3fda2c 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,19 +67,24 @@ let
     s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+    deps = [ "gnumake", "jq","nix",
+             "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
+             "rsync" ]
+    # TODO: --pure , prepare ENV in nix-shell command:
+    #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
     nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-
+    env = {"LOGNAME": "shared",
+           "NIX_REMOTE": "daemon"}
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+    addShell(f,name="centos7-eval",env=env,
              command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+    addShell(f,name="wolf-eval",env=env,
              command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+    addShell(f,name="eval-cross-check",env=env,
              command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
@@ -95,7 +100,7 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+    addShell(s, name="infest-cac-centos7",env=env,
                 sigtermTime=60, # SIGTERM 1 minute before SIGKILL
                 timeout=5400,   # 1.5h timeout
                 command=nixshell + ["infest-cac-centos7"])
-- 
cgit v1.2.3


From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 00:02:59 +0100
Subject: k 3 buildbot.master: refactor

see buildbot-standalone.nix in shared/2configs for the current buildbot config
---
 krebs/3modules/buildbot/master.nix | 341 +++++++++++++++++++++++--------------
 1 file changed, 214 insertions(+), 127 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6bf3fda2c..7078000fe 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -7,134 +7,81 @@ let
     # -*- python -*-
     from buildbot.plugins import *
     import re
-
+    import json
     c = BuildmasterConfig = {}
 
     c['slaves'] = []
-    # TODO: template potential buildslaves
-    # TODO: set password?
-    slavenames= [ 'testslave' ]
-    for i in slavenames:
-      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
 
+    # TODO: configure protocols?
     c['protocols'] = {'pb': {'port': 9989}}
 
     ####### Build Inputs
-    stockholm_repo = 'http://cgit.gum/stockholm'
-    c['change_source'] = []
-    c['change_source'].append(changes.GitPoller(
-            stockholm_repo,
-            workdir='stockholm-poller', branch='master',
-            project='stockholm',
-            pollinterval=120))
+    c['change_source'] = cs = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Change_Source: Begin of ${n}
+        ${v}
+        #### Change_Source: End of ${n}
+      '') cfg.change_source )}
 
     ####### Build Scheduler
-    # TODO: configure scheduler
-    c['schedulers'] = []
-
-    # test the master real quick
-    fast = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                name="fast-master-test",
-                                builderNames=["fast-tests"])
-
-    force = schedulers.ForceScheduler(
-                                name="force",
-                                builderNames=["full-tests"])
-
-    # files everyone depends on or are part of the share branch
-    def shared_files(change):
-      r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
-      for file in change.files:
-        if r.match(file):
-          return True
-      return False
-
-    full = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                fileIsImportant=shared_files,
-                                name="full-master-test",
-                                builderNames=["full-tests"])
-    c['schedulers'] = [ fast, force, full ]
-    ###### The actual build
-    # couple of fast steps:
-    f = util.BuildFactory()
-    # some slow steps
-    s = util.BuildFactory()
-    ## fetch repo
-    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-    f.addStep(grab_repo)
-    s.addStep(grab_repo)
-
-    # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq","nix",
-             "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
-             "rsync" ]
-    # TODO: --pure , prepare ENV in nix-shell command:
-    #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-    nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-    env = {"LOGNAME": "shared",
-           "NIX_REMOTE": "daemon"}
-    def addShell(f,**kwargs):
-      f.addStep(steps.ShellCommand(**kwargs))
-
-    addShell(f,name="centos7-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
-
-    addShell(f,name="wolf-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
-
-    addShell(f,name="eval-cross-check",env=env,
-             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
-
-    c['builders'] = []
-    c['builders'].append(
-        util.BuilderConfig(name="fast-tests",
-          slavenames=slavenames,
-          factory=f))
-
-    # slave needs 2 files:
-    # * cac.json
-    # * retiolum
-    for file in ["cac.json", "retiolum.rsa_key.priv"]:
-      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
-                              slavedest=file))
-
-    addShell(s, name="infest-cac-centos7",env=env,
-                sigtermTime=60, # SIGTERM 1 minute before SIGKILL
-                timeout=5400,   # 1.5h timeout
-                command=nixshell + ["infest-cac-centos7"])
-
-    c['builders'].append(
-        util.BuilderConfig(name="full-tests",
-          slavenames=slavenames,
-          factory=s))
-
-    ####### Status of Builds
-    c['status'] = []
-
-    from buildbot.status import html
-    from buildbot.status.web import authz, auth
-    # TODO: configure if http is wanted
-    authz_cfg=authz.Authz(
-        # TODO: configure user/pw
-        auth=auth.BasicAuth([("krebs","bob")]),
-        gracefulShutdown = False,
-        forceBuild = 'auth',
-        forceAllBuilds = 'auth',
-        pingBuilder = False,
-        stopBuild = 'auth',
-        stopAllBuilds = False,
-        cancelPendingBuild = False,
-    )
-    # TODO: configure nginx
-    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
-    from buildbot.status import words
+    c['schedulers'] = sched = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Schedulers: Begin of ${n}
+        ${v}
+        #### Schedulers: End of ${n}
+      '') cfg.scheduler )}
+
+    ###### Builder
+    c['builders'] = bu = []
+    
+    # Builder Pre: Begin
+    ${cfg.builder_pre}
+    # Builder Pre: End
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Builder: Begin of ${n}
+        ${v}
+        #### Builder: End of ${n}
+      '') cfg.builder )}
+
+
+    ####### Status
+    c['status'] = st = []
+
+    # If you want to configure this url, override with extraConfig
+    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+    ${optionalString (cfg.web.enable) ''
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
+      )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      ''}
+
     ${optionalString (cfg.irc.enable) ''
-      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
-                      # TODO: multiple channels
-                      channels=["${cfg.irc.channel}"],
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+                      channels=${builtins.toJSON cfg.irc.channels},
                       notify_events={
                         'success': 1,
                         'failure': 1,
@@ -145,15 +92,20 @@ let
       c['status'].append(irc)
       ''}
 
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Status: Begin of ${n}
+        ${v}
+        #### Status: End of ${n}
+      '') cfg.status )}
+
     ####### PROJECT IDENTITY
-    c['title'] = "Stockholm"
+    c['title'] = "${cfg.title}"
     c['titleURL'] = "http://krebsco.de"
 
-    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
-    # TODO: configure url
-    c['buildbotURL'] = "http://vbob:8010/"
 
     ####### DB URL
+    # TODO: configure
     c['db'] = {
         'db_url' : "sqlite:///state.sqlite",
     }
@@ -164,6 +116,13 @@ let
 
   api = {
     enable = mkEnableOption "Buildbot Master";
+    title = mkOption {
+      default = "Buildbot CI";
+      type = types.str;
+      description = ''
+        Title of the Buildbot Installation
+      '';
+    };
     workDir = mkOption {
       default = "/var/lib/buildbot/master";
       type = types.str;
@@ -172,16 +131,144 @@ let
         Will be created on startup.
       '';
     };
+
+    slaves = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
+      '';
+    };
+
+    change_source = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        stockholm = ''
+          cs.append(changes.GitPoller(
+                  'http://cgit.gum/stockholm',
+                  workdir='stockholm-poller', branch='master',
+                  project='stockholm',
+                  pollinterval=120))
+        '';
+      };
+      description = ''
+        Attrset of all the change_sources which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>cs</literal>
+      '';
+    };
+
+    scheduler = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        force-scheduler = ''
+          sched.append(schedulers.ForceScheduler(
+                                      name="force",
+                                      builderNames=["full-tests"]))
+        '';
+      };
+      description = ''
+        Attrset of all the schedulers which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>sched</literal>
+      '';
+    };
+
+    builder_pre = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+      '';
+      description = ''
+        some code before the builders are being assembled.
+        can be used to define functions used by multiple builders
+      '';
+    };
+
+    builder = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        fast-test = ''
+        '';
+      };
+      description = ''
+        Attrset of all the builder which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>bu</literal>
+      '';
+    };
+
+    status = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of all the extra status which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>st</literal>
+
+        Right now IRC and Web status can be configured by setting
+        <literal>buildbot.master.irc.enable</literal> and
+        <literal>buildbot.master.web.enable</literal>
+      '';
+    };
+
+    # Configurable Stati
+    web = mkOption {
+      default = {};
+      type = types.submodule ({ config2, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master Web Status";
+          username = mkOption {
+            default = "krebs";
+            type = types.str;
+            description = ''
+              username for web authentication
+            '';
+          };
+          hostname = mkOption {
+            default = config.networking.hostName;
+            type = types.str;
+            description = ''
+              web interface Hostname
+            '';
+          };
+          password = mkOption {
+            default = "bob";
+            type = types.str;
+            description = ''
+              password for web authentication
+            '';
+          };
+          port = mkOption {
+            default = 8010;
+            type = types.int;
+            description = ''
+              port for buildbot web status
+            '';
+          };
+        };
+      });
+    };
+
     irc = mkOption {
       default = {};
       type = types.submodule ({ config, ... }: {
         options = {
           enable = mkEnableOption "Buildbot Master IRC Status";
-          channel = mkOption {
-            default = "nix-buildbot-meetup";
-            type = types.str;
+          channels = mkOption {
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
             description = ''
-              irc channel the bot should connect to
+              irc channels the bot should connect to
             '';
           };
           allowForce = mkOption {
@@ -235,6 +322,7 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+      # TODO: add extra dependencies to master like svn and cvs
       path = [ pkgs.git ];
       environment = {
         SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
@@ -242,7 +330,6 @@ let
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
         secretsdir="${lib.shell.escape (toString <secrets>)}";
-        # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
         Type = "forking";
-- 
cgit v1.2.3


From cef2be532b0cc76071b0b3515fc71214b37591f0 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:44:56 +0100
Subject: m 3 Reaktor: add workdir/state_dir

---
 krebs/3modules/Reaktor.nix | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 1ec49b81e..d219d1800 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -62,6 +62,14 @@ let
         configuration appended to the default or overridden configuration
       '';
     };
+
+    workdir = mkOption {
+      default = "/var/lib/Reaktor";
+      type = types.str;
+      description = ''
+        Reaktor working directory
+      '';
+    };
     extraEnviron = mkOption {
       default = {};
       type = types.attrsOf types.str;
@@ -91,7 +99,7 @@ let
       # uid = config.ids.uids.Reaktor;
       uid = 2066439104; #genid Reaktor
       description = "Reaktor user";
-      home = "/var/lib/Reaktor";
+      home = cfg.workdir;
       createHome = true;
     };
 
@@ -113,6 +121,7 @@ let
         GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         REAKTOR_NICKNAME = cfg.nickname;
         REAKTOR_DEBUG = (if cfg.debug  then "True" else "False");
+        state_dir = cfg.workdir;
         } // cfg.extraEnviron;
       serviceConfig= {
         ExecStartPre = pkgs.writeScript "Reaktor-init" ''
-- 
cgit v1.2.3


From 8f98ae9842963d801945c850e9da1e450e098ce3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:54:02 +0100
Subject: m 3 buildbot/master: use genid

---
 krebs/3modules/buildbot/master.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 7078000fe..5870c3145 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -308,7 +308,7 @@ let
   imp = {
 
     users.extraUsers.buildbotMaster = {
-      uid = 672626386; #genid buildbotMaster
+      uid = genid "buildbotMaster";
       description = "Buildbot Master";
       home = cfg.workDir;
       createHome = false;
-- 
cgit v1.2.3


From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 11:41:41 +0100
Subject: m 3 Reaktor: now supports plugin infra see m/1/pornocauster

---
 krebs/3modules/Reaktor.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 59058bffc..607eb7cac 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -9,6 +9,7 @@ let
       ${cfg.overrideConfig}
       '' else ""}
       ## Extra Config
+      ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)}
       ${cfg.extraConfig}
     '';
   cfg = config.krebs.Reaktor;
@@ -35,7 +36,6 @@ let
       '';
     };
 
-
     overrideConfig = mkOption {
       default = null;
       type = types.nullOr types.str;
@@ -44,6 +44,9 @@ let
         Reaktor default cfg can be retrieved via `reaktor get-config`
       '';
     };
+    plugins = mkOption {
+      default = [pkgs.ReaktorPlugins.nixos-version];
+    };
     extraConfig = mkOption {
       default = "";
       type = types.string;
-- 
cgit v1.2.3


From 5a9ccbef0abe1f3acb16d716a2e1d7faa9bb0af1 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Mon, 28 Dec 2015 19:43:31 +0100
Subject: {tv 2 => krebs 3} backup

---
 krebs/3modules/backup.nix  | 286 +++++++++++++++++++++++++++++++++++++++++++++
 krebs/3modules/default.nix |   1 +
 2 files changed, 287 insertions(+)
 create mode 100644 krebs/3modules/backup.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
new file mode 100644
index 000000000..01bb16a2b
--- /dev/null
+++ b/krebs/3modules/backup.nix
@@ -0,0 +1,286 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  out = {
+    options.krebs.backup = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  cfg = config.krebs.backup;
+
+  api = {
+    enable = mkEnableOption "krebs.backup" // { default = true; };
+    plans = mkOption {
+      default = {};
+      type = types.attrsOf (types.submodule ({
+        # TODO enable = mkEnableOption "TODO" // { default = true; };
+        options = {
+          method = mkOption {
+            type = types.enum ["pull" "push"];
+          };
+          name = mkOption {
+            type = types.str;
+          };
+          src = mkOption {
+            type = types.krebs.file-location;
+          };
+          dst = mkOption {
+            type = types.krebs.file-location;
+          };
+          startAt = mkOption {
+            type = types.str;
+          };
+          snapshots = mkOption {
+            type = types.attrsOf (types.submodule {
+              options = {
+                format = mkOption {
+                  type = types.str; # TODO date's +FORMAT
+                };
+                retain = mkOption {
+                  type = types.nullOr types.int;
+                  default = null; # null = retain all snapshots
+                };
+              };
+            });
+          };
+        };
+      }));
+    };
+  };
+
+  imp = {
+    users.groups.backup.gid = genid "backup";
+    users.users = {}
+      // {
+        root.openssh.authorizedKeys.keys =
+          map (plan: plan.dst.host.ssh.pubkey)
+              (filter isPullSrc (attrValues cfg.plans))
+          ++
+          map (plan: plan.src.host.ssh.pubkey)
+              (filter isPushDst (attrValues cfg.plans))
+          ;
+      }
+      ;
+    systemd.services =
+      flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: {
+        name = "backup.${name}.pull";
+        value = makePullService plan;
+      })
+      //
+      flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: {
+        name = "backup.${name}.push";
+        value = makePushService plan;
+      })
+      ;
+  };
+
+  isPushSrc = plan:
+    plan.method == "push" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPullSrc = plan:
+    plan.method == "pull" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPushDst = plan:
+    plan.method == "push" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  isPullDst = plan:
+    plan.method == "pull" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  # TODO push destination needs this in the dst.user's PATH
+  service-path = [
+    pkgs.coreutils
+    pkgs.gnused
+    pkgs.openssh
+    pkgs.rsync
+    pkgs.utillinux
+  ];
+
+  # TODO if there is plan.user, then use its privkey
+  makePushService = plan: assert isPushSrc plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = push plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  makePullService = plan: assert isPullDst plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = pull plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  push = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.push" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.push.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.src.host.ssh.privkey.path}
+      src=${shell.escape plan.src.path}
+      dst_target=${shell.escape "root@${getFQDN plan.dst.host}"}
+      dst_path=${shell.escape plan.dst.path}
+      dst=$dst_target:$dst_path
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $src -> $dst"
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --rsync-path ${shell.escape
+            "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \
+          --link-dest="$dst_path/current" \
+          "$src/" \
+          "$dst/.partial"
+
+      exec ssh -F /dev/null \
+          -i "$identity" \
+          "$dst_target" \
+          -T \
+          env NOW="$NOW" /bin/sh < ${remote-snapshot}
+      EOF
+    '';
+
+    remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      if test -e "$dst/current"; then
+        mv "$dst/current" "$dst/.previous"
+      fi
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      (${(take-snapshots plan).text})
+    '';
+
+  in out;
+
+  # TODO admit plan.dst.user and its ssh identity
+  pull = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.pull" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.pull.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.dst.host.ssh.privkey.path}
+      src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"}
+      dst=${shell.escape plan.dst.path}
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $dst <- $src"
+      mkdir -m 0700 -p ${shell.escape plan.dst.path}
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --link-dest="$dst/current" \
+          "$src/" \
+          "$dst/.partial"
+      mv "$dst/current" "$dst/.previous"
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      exec ${take-snapshots plan}
+    '';
+  in out;
+
+  take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" ''
+    set -efu
+    NOW=''${NOW-$(date +%s)}
+    dst=${shell.escape plan.dst.path}
+
+    snapshot() {(
+      : $ns $format $retain
+      name=$(date --date="@$NOW" +"$format")
+      if ! test -e "$dst/$ns/$name"; then
+        echo >&2 "create snapshot: $ns/$name"
+        mkdir -m 0700 -p "$dst/$ns"
+        rsync >&2 \
+            -aAXF --delete \
+            --link-dest="$dst/current" \
+            "$dst/current/" \
+            "$dst/$ns/.partial.$name"
+        mv "$dst/$ns/.partial.$name" "$dst/$ns/$name"
+        echo >&2
+      fi
+      case $retain in
+        ([0-9]*)
+          delete_from=$(($retain + 1))
+          ls -r "$dst/$ns" \
+            | sed -n "$delete_from,\$p" \
+            | while read old_name; do
+                echo >&2 "delete snapshot: $ns/$old_name"
+                rm -fR "$dst/$ns/$old_name"
+              done
+          ;;
+        (ALL)
+          :
+          ;;
+      esac
+    )}
+
+    ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }:
+      toString (map shell.escape [
+        "ns=${ns}"
+        "format=${format}"
+        "retain=${if retain == null then "ALL" else toString retain}"
+        "snapshot"
+      ]))
+      plan.snapshots)}
+  '';
+
+  # TODO getFQDN: admit hosts in other domains
+  getFQDN = host: "${host.name}.${config.krebs.search-domain}";
+
+  writeDash = name: text: pkgs.writeScript name ''
+    #! ${pkgs.dash}/bin/dash
+    ${text}
+  '';
+
+  writeDashBin = name: text: pkgs.writeTextFile {
+    executable = true;
+    destination = "/bin/${name}";
+    name = name;
+    text = ''
+      #! ${pkgs.dash}/bin/dash
+      ${text}
+    '';
+  };
+
+in out
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..deccecc48 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -7,6 +7,7 @@ let
   out = {
     imports = [
       ./apt-cacher-ng.nix
+      ./backup.nix
       ./bepasty-server.nix
       ./build.nix
       ./current.nix
-- 
cgit v1.2.3


From 9abd00f6af48676f08b6afdddd03d12410a1d1cd Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 21:20:11 +0100
Subject: k 3 makefu: add ssh pubkeys to hosts

---
 krebs/3modules/makefu/default.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 1970a0777..31516d591 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -83,6 +83,9 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster";
+
     };
 
     vbob = {
@@ -108,6 +111,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos";
     };
     flap = rec {
       cores = 1;
@@ -238,6 +243,8 @@ with lib;
           '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry";
     };
     filepimp = rec {
       cores = 1;
@@ -287,6 +294,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch";
     };
     gum = rec {
       cores = 1;
@@ -327,6 +336,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum";
     };
   };
   users = addNames rec {
-- 
cgit v1.2.3


From d574c0ef78f7572aec88e484d3ff6256247e878c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 01:38:33 +0100
Subject: m 3 buildbot/master: add secrets

---
 krebs/3modules/buildbot/master.nix | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 5870c3145..74385a433 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -132,6 +132,16 @@ let
       '';
     };
 
+    secrets = mkOption {
+      default = [];
+      type = types.listOf types.str;
+      example = [ "cac.json" ];
+      description = ''
+        List of all the secrets in <secrets> which should be copied into the
+        buildbot master directory.
+      '';
+    };
+
     slaves = mkOption {
       default = {};
       type = types.attrsOf types.str;
@@ -344,10 +354,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+
           # copy secrets
-          cp ${secretsdir}/cac.json ${workdir}
-          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
-             ${workdir}/retiolum.rsa_key.priv
+          ${ concatMapStringsSep "\n"
+            (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
-- 
cgit v1.2.3


From a8cb9d41bdefe8a5dc72ca76b3ebc5b4047ddf65 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 02:45:47 +0100
Subject: s 1 test-all-krebs-modules: init

---
 krebs/3modules/shared/default.nix | 1 +
 1 file changed, 1 insertion(+)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix
index b332676c6..518e46587 100644
--- a/krebs/3modules/shared/default.nix
+++ b/krebs/3modules/shared/default.nix
@@ -7,6 +7,7 @@ let
     "test-arch"
     "test-centos6"
     "test-centos7"
+    "test-all-krebs-modules"
   ] (name: {
     inherit name;
     cores = 1;
-- 
cgit v1.2.3


From c962e8549e968fd15d4f15b4d184e86e1cd7ed04 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 11:29:28 +0100
Subject: k 3 Reaktor: add channels Option

---
 krebs/3modules/Reaktor.nix | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 607eb7cac..92400139c 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -70,12 +70,17 @@ let
           REAKTOR_HOST
           REAKTOR_PORT
           REAKTOR_STATEDIR
-          REAKTOR_CHANNELS
 
           debug and nickname can be set separately via the Reaktor api
       '';
     };
-
+    channels = mkOption {
+      default = [ "#krebs" ];
+      type = types.listOf types.str;
+      description = ''
+        Channels the Reaktor should connect to at startup.
+      '';
+    };
     debug = mkOption {
       default = false;
       description = ''
@@ -112,7 +117,9 @@ let
         GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         REAKTOR_NICKNAME = cfg.nickname;
         REAKTOR_DEBUG = (if cfg.debug  then "True" else "False");
+        REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels;
         state_dir = cfg.workdir;
+
         } // cfg.extraEnviron;
       serviceConfig= {
         ExecStartPre = pkgs.writeScript "Reaktor-init" ''
-- 
cgit v1.2.3


From b9c4a5e4a0800dbfb6f4cb1d20bbfd2c1d228d11 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Wed, 30 Dec 2015 14:54:04 +0100
Subject: exim-retiolum: move assert to proper location

---
 krebs/3modules/exim-retiolum.nix | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix
index e1315d8c8..ea012c38c 100644
--- a/krebs/3modules/exim-retiolum.nix
+++ b/krebs/3modules/exim-retiolum.nix
@@ -1,14 +1,12 @@
 { config, pkgs, lib, ... }:
 
-with builtins;
 with lib;
 let
   cfg = config.krebs.exim-retiolum;
 
   out = {
     options.krebs.exim-retiolum = api;
-    config =
-      mkIf cfg.enable imp;
+    config = mkIf cfg.enable imp;
   };
 
   api = {
@@ -16,13 +14,13 @@ let
   };
 
   imp = {
-    services.exim =
-      # This configuration makes only sense for retiolum-enabled hosts.
-      # TODO modular configuration
-      assert config.krebs.retiolum.enable;
-      {
-        enable = true;
-        config = ''
+    services.exim = {
+      enable = true;
+      config =
+        # This configuration makes only sense for retiolum-enabled hosts.
+        # TODO modular configuration
+        assert config.krebs.retiolum.enable;
+        ''
           primary_hostname = ${retiolumHostname}
           domainlist local_domains    = @ : localhost
           domainlist relay_to_domains = *.retiolum
@@ -134,7 +132,7 @@ let
 
           begin authenticators
         '';
-      };
+    };
   };
 
   # TODO get the hostname from somewhere else.
-- 
cgit v1.2.3


From f16742895c26b0f3df71ca8503afc5f4cb97a9ae Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Wed, 30 Dec 2015 17:14:31 +0100
Subject: l: add new host dishfire

---
 krebs/3modules/lass/default.nix | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix
index 26b0947bb..592ed475d 100644
--- a/krebs/3modules/lass/default.nix
+++ b/krebs/3modules/lass/default.nix
@@ -4,6 +4,38 @@ with lib;
 
 {
   hosts = addNames {
+    dishfire = {
+      cores = 4;
+      dc = "lass"; #dc = "cac";
+      nets = rec {
+        internet = {
+          addrs4 = ["144.76.172.188"];
+          aliases = [
+            "dishfire.internet"
+          ];
+        };
+        retiolum = {
+          via = internet;
+          addrs4 = ["10.243.133.9