From c2fd296ad671a73e85f830c84d860e988587d9ac Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:01:22 +0100 Subject: s 1 wolf: provide graphite --- shared/1systems/wolf.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index a3e527a3b..2c51ac8fe 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -12,6 +12,7 @@ in ../2configs/shack-nix-cacher.nix ../2configs/shack-drivedroid.nix ../2configs/cac-ci.nix + ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by # apt-cacher-ng in first place) -- cgit v1.2.3 From bdc9f6f18813e5840c6a20a0f507d72da49cd759 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:10:44 +0100 Subject: m 3 buildbot.slave: add extra packages and environ to configuration --- makefu/3modules/buildbot/slave.nix | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix index 188a9283c..69d0361bf 100644 --- a/makefu/3modules/buildbot/slave.nix +++ b/makefu/3modules/buildbot/slave.nix @@ -38,7 +38,7 @@ let allow_shutdown=allow_shutdown) s.setServiceParent(application) ''; - + default-packages = [ pkgs.git pkgs.bash ]; cfg = config.makefu.buildbot.slave; api = { @@ -91,6 +91,26 @@ let ''; }; + packages = mkOption { + default = [ pkgs.git ]; + type = with types; listOf package; + description = '' + packages which should be in path for buildslave + ''; + }; + + extraEnviron = mkOption { + default = {}; + example = { + NIX_PATH = "nixpkgs=/path/to/my/nixpkgs"; + }; + type = types.attrsOf types.str; + description = '' + extra environment variables to be provided to the buildslave service + if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. + ''; + }; + extraConfig = mkOption { default = ""; type = types.lines; @@ -121,6 +141,12 @@ let description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; + path = default-packages ++ cfg.packages; + + environment = { + NIX_REMOTE="daemon"; + } // cfg.extraEnviron; + serviceConfig = let workdir = "${lib.shell.escape cfg.workDir}"; contact = "${lib.shell.escape cfg.contact}"; -- cgit v1.2.3 From 87694e24df0ebbaaa08d4f632fea72f48bc430f5 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:11:42 +0100 Subject: m 3 buildbot.master: add deps, refactor --- makefu/3modules/buildbot/master.nix | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix index 5d340f899..0073e5aed 100644 --- a/makefu/3modules/buildbot/master.nix +++ b/makefu/3modules/buildbot/master.nix @@ -43,14 +43,26 @@ let name="force", builderNames=["runtests"])) ###### The actual build - factory = util.BuildFactory() - factory.addStep(steps.Git(repourl=stockholm_repo, mode='incremental')) + f = util.BuildFactory() + f.addStep(steps.Git(repourl=stockholm_repo, mode='incremental')) + # the dependencies which are used by the test script deps = [ "gnumake", "jq" ] - factory.addStep(steps.ShellCommand(command=["nix-shell", "-p" ] + deps )) - factory.addStep(steps.ShellCommand(env={"LOGNAME": "shared"}, - command=["make", "get=krebs.deploy", - "system=test-centos7"])) + nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] + def addShell(**kwargs): + f.addStep(steps.ShellCommand(**kwargs)) + + # TODO: combined strings somewhat defeat the reason why an array was used in first place + addShell(name=env={"LOGNAME": "shared", + "get" : "krebs.deploy", + "filter" : "json" + }, + command=nixshell + ["make -s eval system=test-centos7"]) + addShell(env={"LOGNAME": "shared", + "get" : "krebs.deploy", + "filter" : "json" + }, + command=nixshell + ["make -s eval system=wolf"]) # TODO: different Builders? c['builders'] = [] @@ -58,7 +70,7 @@ let util.BuilderConfig(name="runtests", # TODO: only some slaves being used in builder? slavenames=slavenames, - factory=factory)) + factory=f)) ####### Status of Builds c['status'] = [] @@ -183,8 +195,10 @@ let description = "Buildbot Master"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; + path = [ pkgs.git ]; serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; + # TODO: check if git is the only dep in { PermissionsStartOnly = true; Type = "forking"; -- cgit v1.2.3 From ad625d6d6830c7d7f7a61cc5ee1e2ad398ab70a6 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:48:49 +0100 Subject: m 3 buildbot.master: add fast and full tests --- makefu/3modules/buildbot/master.nix | 67 +++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix index 0073e5aed..1a9ef4db6 100644 --- a/makefu/3modules/buildbot/master.nix +++ b/makefu/3modules/buildbot/master.nix @@ -25,50 +25,71 @@ let stockholm_repo, workdir='stockholm-poller', branch='master', project='stockholm', - pollinterval=300)) + pollinterval=120)) ####### Build Scheduler # TODO: configure scheduler - important_files = util.ChangeFilter( - project_re="^((krebs|share)/.*|Makefile|default.nix)", - branch='master') c['schedulers'] = [] - c['schedulers'].append(schedulers.SingleBranchScheduler( - name="all-important-files", - change_filter=important_files, - # 3 minutes stable tree - treeStableTimer=3*60, - builderNames=["runtests"])) - c['schedulers'].append(schedulers.ForceScheduler( + + # test the master real quick + fast = schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"]) + + force = schedulers.ForceScheduler( name="force", - builderNames=["runtests"])) + builderNames=["full-tests"]) + + # files everyone depends on or are part of the share branch + def shared_files(change): + import re + r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + full = schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + name="full-master-test", + builderNames=["full-tests"]) + c['schedulers'] = [ fast, force, full ] ###### The actual build + # couple of fast steps: f = util.BuildFactory() - f.addStep(steps.Git(repourl=stockholm_repo, mode='incremental')) + ## fetch repo + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + f.addStep(grab_repo) # the dependencies which are used by the test script deps = [ "gnumake", "jq" ] nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] - def addShell(**kwargs): + def addShell(f,**kwargs): f.addStep(steps.ShellCommand(**kwargs)) - # TODO: combined strings somewhat defeat the reason why an array was used in first place - addShell(name=env={"LOGNAME": "shared", + addShell(f,name="centos7-eval",env={"LOGNAME": "shared", "get" : "krebs.deploy", "filter" : "json" }, command=nixshell + ["make -s eval system=test-centos7"]) - addShell(env={"LOGNAME": "shared", + + addShell(f,name="wolf-eval",env={"LOGNAME": "shared", "get" : "krebs.deploy", "filter" : "json" }, command=nixshell + ["make -s eval system=wolf"]) - # TODO: different Builders? c['builders'] = [] c['builders'].append( - util.BuilderConfig(name="runtests", - # TODO: only some slaves being used in builder? + util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + + # TODO slow build + c['builders'].append( + util.BuilderConfig(name="full-tests", slavenames=slavenames, factory=f)) @@ -111,7 +132,9 @@ let c['title'] = "Stockholm" c['titleURL'] = "http://krebsco.de" - c['buildbotURL'] = "http://buildbot.krebsco.de/" + #c['buildbotURL'] = "http://buildbot.krebsco.de/" + # TODO: configure url + c['buildbotURL'] = "http://vbob:8010/" ####### DB URL c['db'] = { @@ -124,7 +147,6 @@ let api = { enable = mkEnableOption "Buildbot Master"; - workDir = mkOption { default = "/var/lib/buildbot/master"; type = types.str; @@ -169,6 +191,7 @@ let }; }); }; + extraConfig = mkOption { default = ""; type = types.lines; -- cgit v1.2.3 From 20d9a610189da29cd1f4abf60089d0579a1e291a Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:53:35 +0100 Subject: m 1 vbob: add firewall exception, extraEnviron --- makefu/1systems/vbob.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index 5b03d40a8..a24cefd0d 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -35,6 +35,8 @@ in { masterhost = "localhost"; username = "testslave"; password = "krebspass"; + packages = with pkgs;[ git nix ]; + extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; }; krebs.build.source.git.nixpkgs = { @@ -63,6 +65,7 @@ in { networking.firewall.allowedTCPPorts = [ 25 80 + 8010 ]; krebs.retiolum = { -- cgit v1.2.3 From 956d2091ec2ba931080ee8a09f12f5c645fbf672 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 17:58:29 +0100 Subject: m 3 buildbot.master: only alert on state change --- makefu/3modules/buildbot/master.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix index 1a9ef4db6..58e2f8175 100644 --- a/makefu/3modules/buildbot/master.nix +++ b/makefu/3modules/buildbot/master.nix @@ -6,6 +6,7 @@ let buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * + import re c = BuildmasterConfig = {} @@ -43,7 +44,6 @@ let # files everyone depends on or are part of the share branch def shared_files(change): - import re r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") for file in change.files: if r.match(file): @@ -119,8 +119,8 @@ let # TODO: multiple channels channels=["${cfg.irc.channel}"], notify_events={ - 'success': 1, - 'failure': 1, + #'success': 1, + #'failure': 1, 'exception': 1, 'successToFailure': 1, 'failureToSuccess': 1, -- cgit v1.2.3 From c666325c15726107598dbac3c64de175e6ff13ca Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Dec 2015 18:01:14 +0100 Subject: m 3 buildbot.slave: add to imports --- makefu/3modules/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix index 4b2b36e64..ffbf54cc0 100644 --- a/makefu/3modules/default.nix +++ b/makefu/3modules/default.nix @@ -3,6 +3,7 @@ _: { imports = [ ./buildbot/master.nix + ./buildbot/slave.nix ]; } -- cgit v1.2.3 From 3f4bd94233164a9b12d61c1a460b6eff83c39209 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 17 Dec 2015 17:38:33 +0100 Subject: m 2 git/brain-retiolum: remove obsolete users --- makefu/2configs/git/brain-retiolum.nix | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/makefu/2configs/git/brain-retiolum.nix b/makefu/2configs/git/brain-retiolum.nix index 066d50a28..25ef584bf 100644 --- a/makefu/2configs/git/brain-retiolum.nix +++ b/makefu/2configs/git/brain-retiolum.nix @@ -59,16 +59,7 @@ let set-owners repo all-makefu ++ set-ro-access repo krebsminister; in { - imports = [{ - krebs.users.makefu-omo = { - name = "makefu-omo" ; - pubkey= with builtins; readFile ../../../krebs/Zpubkeys/makefu_omo.ssh.pub; - }; - krebs.users.makefu-tsp = { - name = "makefu-tsp" ; - pubkey= with builtins; readFile ../../../krebs/Zpubkeys/makefu_tsp.ssh.pub; - }; - }]; + imports = [ ]; krebs.git = { enable = true; cgit = false; -- cgit v1.2.3 From cfe266c222123c41fb7645a3739ac2ef448f527c Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 19 Dec 2015 16:02:27 +0100 Subject: k 5 cac: bump version --- krebs/5pkgs/cac/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/krebs/5pkgs/cac/default.nix b/krebs/5pkgs/cac/default.nix index e29f091e4..40dd56412 100644 --- a/krebs/5pkgs/cac/default.nix +++ b/krebs/5pkgs/cac/default.nix @@ -4,9 +4,9 @@ stdenv.mkDerivation { name = "cac-1.0.0"; src = fetchgit { - url = http://cgit.cd.retiolum/cac; - rev = "14de1d3c78385e3f8b6d694f5d799eb1b613159e"; - sha256 = "9b2a3d47345d6f8f27d9764c4f2f2acff17d3dde145dd0e674e4183e9312fec3"; + url = http://cgit.gum/cac; + rev = "fe3b2ecb0aaf7d863842b896e18cd2b829f2297b"; + sha256 = "05bnd7wyjhqy8srmpnc8d234rv3jxdjgb4z0hlfb9kg7mb12w1ya"; }; phases = [ -- cgit v1.2.3 From 5821d8438578db623a3e248c52fefa424fad0b51 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 19 Dec 2015 16:02:52 +0100 Subject: s 1 test-centos7: prepare to use generated networking --- shared/1systems/test-centos7.nix | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix index 51e99600c..077a5d61b 100644 --- a/shared/1systems/test-centos7.nix +++ b/shared/1systems/test-centos7.nix @@ -3,29 +3,13 @@ let inherit (lib) head; - ip = "168.235.145.85"; - gw = "168.235.145.1"; in { imports = [ ../2configs/base.nix ../2configs/os-templates/CAC-CentOS-7-64bit.nix - { - networking.interfaces.enp2s1.ip4 = [ - { - address = ip; - prefixLength = 24; - } - ]; - networking.defaultGateway = gw; - networking.nameservers = [ - "8.8.8.8" - ]; - - } - { - sound.enable = false; - } + ../2configs/os-templates/temp-networking.nix ]; + sound.enable = false; krebs.build.host = config.krebs.hosts.test-centos7; } -- cgit v1.2.3