From 3371d54618aa017be77e2494c1cf82331152f3b7 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Tue, 15 Dec 2015 18:43:40 +0100
Subject: m 3 buildbot: master init

---
 makefu/3modules/buildbot/master.nix | 179 ++++++++++++++++++++++++++++++++++++
 1 file changed, 179 insertions(+)
 create mode 100644 makefu/3modules/buildbot/master.nix

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
new file mode 100644
index 000000000..310b8460d
--- /dev/null
+++ b/makefu/3modules/buildbot/master.nix
@@ -0,0 +1,179 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    # TODO: template potential buildslaves
+    # TODO: set password?
+    for i in [ 'testslave' ]:
+      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    stockholm_repo = 'http://cgit.gum/stockholm'
+    c['change_source'] = []
+    c['change_source'].append(changes.GitPoller(
+            stockholm_repo,
+            workdir='stockholm-poller', branch='master',
+            project='stockholm',
+            pollinterval=300))
+
+    ####### Build Scheduler
+    # TODO: configure scheduler
+    important_files = util.ChangeFilter(
+                  project_re="^((krebs|share)/.*|Makefile|default.nix)",
+                  branch='master')
+    c['schedulers'] = []
+    c['schedulers'].append(schedulers.SingleBranchScheduler(
+                                name="all-important-files",
+                                change_filter=important_files,
+                                # 3 minutes stable tree
+                                treeStableTimer=3*60,
+                                builderNames=["runtests"]))
+    c['schedulers'].append(schedulers.ForceScheduler(
+                                name="force",
+                                builderNames=["runtests"]))
+    ###### The actual build
+    factory = util.BuildFactory()
+    factory.addStep(steps.Git(repourl=stockholm_repo, mode='incremental'))
+
+    deps = [ "gnumake", "jq" ]
+    factory.addStep(steps.ShellCommand(command=["nix-shell", "-p" ] + deps ))
+    factory.addStep(steps.ShellCommand(env={"LOGNAME": "shared"},
+                                       command=["make", "get=krebs.deploy",
+                                                        "system=test-centos7"]))
+
+    # TODO: different Builders?
+    c['builders'] = []
+    c['builders'].append(
+        util.BuilderConfig(name="runtests",
+          # TODO: only some slaves being used in builder?
+          slavenames=c['slaves'],
+          factory=factory))
+
+    ####### Status of Builds
+    c['status'] = []
+
+    from buildbot.status import html
+    from buildbot.status.web import authz, auth
+    # TODO: configure if http is wanted
+    authz_cfg=authz.Authz(
+        # TODO: configure user/pw
+        auth=auth.BasicAuth([("krebs","bob")]),
+        gracefulShutdown = False,
+        forceBuild = 'auth',
+        forceAllBuilds = 'auth',
+        pingBuilder = False,
+        stopBuild = False,
+        stopAllBuilds = False,
+        cancelPendingBuild = False,
+    )
+    # TODO: configure nginx
+    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+    from buildbot.status import words
+    # TODO: configure IRC Bot
+    irc = words.IRC("irc.freenode.net", "krebsbuild",
+                    channels=["krebs"],
+                    notify_events={
+                      'sucess': 1,
+                      'failure': 1,
+                      'exception': 1,
+                      'successToFailure': 1,
+                      'failureToSuccess': 1,
+                    },allowForce=True)
+    c['status'].append(irc)
+
+    ####### PROJECT IDENTITY
+    c['title'] = "Stockholm"
+    c['titleURL'] = "http://krebsco.de"
+
+    c['buildbotURL'] = "http://buildbot.krebsco.de/"
+
+    ####### DB URL
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.makefu.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = 672626386; #genid buildbotMaster
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        PermissionsStartOnly = true;
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          workdir=${lib.shell.escape cfg.workDir}
+          if [ ! -e $workdir ];then
+            mkdir -p $workdir
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f $workdir
+            chown buildbotMaster:buildbotMaster  $workdir
+          fi
+          # always override the master.cfg
+          cp ${toString buildbot-master-config} "$workdir/master.cfg"
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig $workdir
+          # upgrade
+          ${buildbot}/bin/buildbot upgrade-master $workdir
+        '';
+        ExecStart = "${buildbot}/bin/buildbot ${lib.shell.escape cfg.workDir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.makefu.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
-- 
cgit v1.2.3


From a907f926c120f10945c47cdaba7405fe08cfd9ee Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 15 Dec 2015 22:25:46 +0100
Subject: m 3 buildbot: first working commit for buildbot master

---
 makefu/3modules/buildbot/master.nix | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index 310b8460d..d8e917a21 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -12,7 +12,8 @@ let
     c['slaves'] = []
     # TODO: template potential buildslaves
     # TODO: set password?
-    for i in [ 'testslave' ]:
+    slavenames= [ 'testslave' ]
+    for i in slavenames:
       c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
 
     c['protocols'] = {'pb': {'port': 9989}}
@@ -56,7 +57,7 @@ let
     c['builders'].append(
         util.BuilderConfig(name="runtests",
           # TODO: only some slaves being used in builder?
-          slavenames=c['slaves'],
+          slavenames=slavenames,
           factory=factory))
 
     ####### Status of Builds
@@ -84,7 +85,7 @@ let
     irc = words.IRC("irc.freenode.net", "krebsbuild",
                     channels=["krebs"],
                     notify_events={
-                      'sucess': 1,
+                      'success': 1,
                       'failure': 1,
                       'exception': 1,
                       'successToFailure': 1,
@@ -145,26 +146,34 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+      in {
+        pidfile="${workdir}/twistd.pid";
         PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
         # TODO: maybe also prepare buildbot.tac?
         ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
           #!/bin/sh
           set -efux
-          workdir=${lib.shell.escape cfg.workDir}
-          if [ ! -e $workdir ];then
-            mkdir -p $workdir
-            ${buildbot}/bin/buildbot create-master -r -l 10 -f $workdir
-            chown buildbotMaster:buildbotMaster  $workdir
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
           fi
           # always override the master.cfg
-          cp ${toString buildbot-master-config} "$workdir/master.cfg"
+          cp ${buildbot-master-config} ${workdir}/master.cfg
           # sanity
-          ${buildbot}/bin/buildbot checkconfig $workdir
-          # upgrade
-          ${buildbot}/bin/buildbot upgrade-master $workdir
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade?
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot ${lib.shell.escape cfg.workDir}";
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
         PrivateTmp = "true";
         User = "buildbotMaster";
         Restart = "always";
-- 
cgit v1.2.3


From 20a52c8dee414e89dba49f4a4a12e20d6555c55e Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 14:29:46 +0100
Subject: m 3 buildbot/master: make irc configurable

---
 makefu/3modules/buildbot/master.nix | 66 +++++++++++++++++++++++++++++--------
 1 file changed, 52 insertions(+), 14 deletions(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index d8e917a21..5d340f899 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -81,17 +81,19 @@ let
     c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
 
     from buildbot.status import words
-    # TODO: configure IRC Bot
-    irc = words.IRC("irc.freenode.net", "krebsbuild",
-                    channels=["krebs"],
-                    notify_events={
-                      'success': 1,
-                      'failure': 1,
-                      'exception': 1,
-                      'successToFailure': 1,
-                      'failureToSuccess': 1,
-                    },allowForce=True)
-    c['status'].append(irc)
+    ${optionalString (cfg.irc.enable) ''
+      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
+                      # TODO: multiple channels
+                      channels=["${cfg.irc.channel}"],
+                      notify_events={
+                        'success': 1,
+                        'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
 
     ####### PROJECT IDENTITY
     c['title'] = "Stockholm"
@@ -119,7 +121,42 @@ let
         Will be created on startup.
       '';
     };
-
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channel = mkOption {
+            default = "nix-buildbot-meetup";
+            type = types.str;
+            description = ''
+              irc channel the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
     extraConfig = mkOption {
       default = "";
       type = types.lines;
@@ -149,7 +186,6 @@ let
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
       in {
-        pidfile="${workdir}/twistd.pid";
         PermissionsStartOnly = true;
         Type = "forking";
         PIDFile = "${workdir}/twistd.pid";
@@ -166,9 +202,11 @@ let
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
-          # TODO: maybe upgrade?
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
           # ${buildbot}/bin/buildbot upgrade-master ${workdir}
 
+          chmod 700 -R ${workdir}
           chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
         ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
-- 
cgit v1.2.3


From 2156aa4d37071408b8e5385a4f639ed029f70620 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 14:30:01 +0100
Subject: m 3 buildbot/slave: init

---
 makefu/3modules/buildbot/slave.nix | 159 +++++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 makefu/3modules/buildbot/slave.nix

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix
new file mode 100644
index 000000000..188a9283c
--- /dev/null
+++ b/makefu/3modules/buildbot/slave.nix
@@ -0,0 +1,159 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
+
+  cfg = config.makefu.buildbot.slave;
+
+  api = {
+    enable = mkEnableOption "Buildbot Slave";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/slave";
+      type = types.str;
+      description = ''
+        Path to build bot slave directory.
+        Will be created on startup.
+      '';
+    };
+
+    masterhost = mkOption {
+      default = "localhost";
+      type = types.str;
+      description = ''
+        Hostname/IP of the buildbot master
+      '';
+    };
+
+    username = mkOption {
+      type = types.str;
+      description = ''
+        slavename used to authenticate with master
+      '';
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        slave password used to authenticate with master
+      '';
+    };
+
+    contact = mkOption {
+      default = "nix slave <buildslave@${config.networking.hostName}>";
+      type = types.str;
+      description = ''
+        contact to be announced by buildslave
+      '';
+    };
+
+    description = mkOption {
+      default = "Nix Generated BuildSlave";
+      type = types.str;
+      description = ''
+        description for hostto be announced by buildslave
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        port = 443
+        keepalive = 600
+      '';
+      description = ''
+        extra config evaluated before calling BuildSlave init in .tac file
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotSlave = {
+      uid = 1408105834; #genid buildbotMaster
+      description = "Buildbot Slave";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
+    };
+
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = let
+        workdir = "${lib.shell.escape cfg.workDir}";
+        contact = "${lib.shell.escape cfg.contact}";
+        description = "${lib.shell.escape cfg.description}";
+        buildbot = pkgs.buildbot-slave;
+        # TODO:make this
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          mkdir -p ${workdir}/info
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          echo ${contact} > ${workdir}/info/admin
+          echo ${description} > ${workdir}/info/host
+
+          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chmod 700 -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotSlave";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.makefu.buildbot.slave = api;
+  config = mkIf cfg.enable imp;
+}
-- 
cgit v1.2.3


From bdc9f6f18813e5840c6a20a0f507d72da49cd759 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 17:10:44 +0100
Subject: m 3 buildbot.slave: add extra packages and environ to configuration

---
 makefu/3modules/buildbot/slave.nix | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix
index 188a9283c..69d0361bf 100644
--- a/makefu/3modules/buildbot/slave.nix
+++ b/makefu/3modules/buildbot/slave.nix
@@ -38,7 +38,7 @@ let
                   allow_shutdown=allow_shutdown)
     s.setServiceParent(application)
     '';
-
+  default-packages = [ pkgs.git pkgs.bash ];
   cfg = config.makefu.buildbot.slave;
 
   api = {
@@ -91,6 +91,26 @@ let
       '';
     };
 
+    packages = mkOption {
+      default = [ pkgs.git ];
+      type = with types; listOf package;
+      description = ''
+        packages which should be in path for buildslave
+      '';
+    };
+
+    extraEnviron = mkOption {
+      default = {};
+      example = {
+        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
+      };
+      type = types.attrsOf types.str;
+      description = ''
+        extra environment variables to be provided to the buildslave service
+        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
+      '';
+    };
+
     extraConfig = mkOption {
       default = "";
       type = types.lines;
@@ -121,6 +141,12 @@ let
       description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+      path = default-packages ++ cfg.packages;
+
+      environment = {
+          NIX_REMOTE="daemon";
+      } // cfg.extraEnviron;
+
       serviceConfig = let
         workdir = "${lib.shell.escape cfg.workDir}";
         contact = "${lib.shell.escape cfg.contact}";
-- 
cgit v1.2.3


From 87694e24df0ebbaaa08d4f632fea72f48bc430f5 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 17:11:42 +0100
Subject: m 3 buildbot.master: add deps, refactor

---
 makefu/3modules/buildbot/master.nix | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index 5d340f899..0073e5aed 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -43,14 +43,26 @@ let
                                 name="force",
                                 builderNames=["runtests"]))
     ###### The actual build
-    factory = util.BuildFactory()
-    factory.addStep(steps.Git(repourl=stockholm_repo, mode='incremental'))
+    f = util.BuildFactory()
+    f.addStep(steps.Git(repourl=stockholm_repo, mode='incremental'))
 
+    # the dependencies which are used by the test script
     deps = [ "gnumake", "jq" ]
-    factory.addStep(steps.ShellCommand(command=["nix-shell", "-p" ] + deps ))
-    factory.addStep(steps.ShellCommand(env={"LOGNAME": "shared"},
-                                       command=["make", "get=krebs.deploy",
-                                                        "system=test-centos7"]))
+    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    def addShell(**kwargs):
+      f.addStep(steps.ShellCommand(**kwargs))
+
+    # TODO: combined strings somewhat defeat the reason why an array was used in first place
+    addShell(name=env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=test-centos7"])
+    addShell(env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=wolf"])
 
     # TODO: different Builders?
     c['builders'] = []
@@ -58,7 +70,7 @@ let
         util.BuilderConfig(name="runtests",
           # TODO: only some slaves being used in builder?
           slavenames=slavenames,
-          factory=factory))
+          factory=f))
 
     ####### Status of Builds
     c['status'] = []
@@ -183,8 +195,10 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.git ];
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
+        # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
         Type = "forking";
-- 
cgit v1.2.3


From ad625d6d6830c7d7f7a61cc5ee1e2ad398ab70a6 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 17:48:49 +0100
Subject: m 3 buildbot.master: add fast and full tests

---
 makefu/3modules/buildbot/master.nix | 67 +++++++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 22 deletions(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index 0073e5aed..1a9ef4db6 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -25,50 +25,71 @@ let
             stockholm_repo,
             workdir='stockholm-poller', branch='master',
             project='stockholm',
-            pollinterval=300))
+            pollinterval=120))
 
     ####### Build Scheduler
     # TODO: configure scheduler
-    important_files = util.ChangeFilter(
-                  project_re="^((krebs|share)/.*|Makefile|default.nix)",
-                  branch='master')
     c['schedulers'] = []
-    c['schedulers'].append(schedulers.SingleBranchScheduler(
-                                name="all-important-files",
-                                change_filter=important_files,
-                                # 3 minutes stable tree
-                                treeStableTimer=3*60,
-                                builderNames=["runtests"]))
-    c['schedulers'].append(schedulers.ForceScheduler(
+
+    # test the master real quick
+    fast = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                name="fast-master-test",
+                                builderNames=["fast-tests"])
+
+    force = schedulers.ForceScheduler(
                                 name="force",
-                                builderNames=["runtests"]))
+                                builderNames=["full-tests"])
+
+    # files everyone depends on or are part of the share branch
+    def shared_files(change):
+      import re
+      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+      for file in change.files:
+        if r.match(file):
+          return True
+      return False
+
+    full = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                fileIsImportant=shared_files,
+                                name="full-master-test",
+                                builderNames=["full-tests"])
+    c['schedulers'] = [ fast, force, full ]
     ###### The actual build
+    # couple of fast steps:
     f = util.BuildFactory()
-    f.addStep(steps.Git(repourl=stockholm_repo, mode='incremental'))
+    ## fetch repo
+    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+    f.addStep(grab_repo)
 
     # the dependencies which are used by the test script
     deps = [ "gnumake", "jq" ]
     nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
-    def addShell(**kwargs):
+    def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    # TODO: combined strings somewhat defeat the reason why an array was used in first place
-    addShell(name=env={"LOGNAME": "shared",
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
                   "get" : "krebs.deploy",
                   "filter" : "json"
                  },
              command=nixshell + ["make -s eval system=test-centos7"])
-    addShell(env={"LOGNAME": "shared",
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
                   "get" : "krebs.deploy",
                   "filter" : "json"
                  },
              command=nixshell + ["make -s eval system=wolf"])
 
-    # TODO: different Builders?
     c['builders'] = []
     c['builders'].append(
-        util.BuilderConfig(name="runtests",
-          # TODO: only some slaves being used in builder?
+        util.BuilderConfig(name="fast-tests",
+          slavenames=slavenames,
+          factory=f))
+
+    # TODO slow build
+    c['builders'].append(
+        util.BuilderConfig(name="full-tests",
           slavenames=slavenames,
           factory=f))
 
@@ -111,7 +132,9 @@ let
     c['title'] = "Stockholm"
     c['titleURL'] = "http://krebsco.de"
 
-    c['buildbotURL'] = "http://buildbot.krebsco.de/"
+    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
+    # TODO: configure url
+    c['buildbotURL'] = "http://vbob:8010/"
 
     ####### DB URL
     c['db'] = {
@@ -124,7 +147,6 @@ let
 
   api = {
     enable = mkEnableOption "Buildbot Master";
-
     workDir = mkOption {
       default = "/var/lib/buildbot/master";
       type = types.str;
@@ -169,6 +191,7 @@ let
         };
       });
     };
+
     extraConfig = mkOption {
       default = "";
       type = types.lines;
-- 
cgit v1.2.3


From 956d2091ec2ba931080ee8a09f12f5c645fbf672 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 17:58:29 +0100
Subject: m 3 buildbot.master: only alert on state change

---
 makefu/3modules/buildbot/master.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'makefu/3modules/buildbot')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index 1a9ef4db6..58e2f8175 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -6,6 +6,7 @@ let
   buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
     # -*- python -*-
     from buildbot.plugins import *
+    import re
 
     c = BuildmasterConfig = {}
 
@@ -43,7 +44,6 @@ let
 
     # files everyone depends on or are part of the share branch
     def shared_files(change):
-      import re
       r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
       for file in change.files:
         if r.match(file):
@@ -119,8 +119,8 @@ let
                       # TODO: multiple channels
                       channels=["${cfg.irc.channel}"],
                       notify_events={
-                        'success': 1,
-                        'failure': 1,
+                        #'success': 1,
+                        #'failure': 1,
                         'exception': 1,
                         'successToFailure': 1,
                         'failureToSuccess': 1,
-- 
cgit v1.2.3