From 8f98ae9842963d801945c850e9da1e450e098ce3 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 10:54:02 +0100 Subject: m 3 buildbot/master: use genid --- krebs/3modules/buildbot/master.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 7078000fe..5870c3145 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -308,7 +308,7 @@ let imp = { users.extraUsers.buildbotMaster = { - uid = 672626386; #genid buildbotMaster + uid = genid "buildbotMaster"; description = "Buildbot Master"; home = cfg.workDir; createHome = false; -- cgit v1.2.3 From 669e4be273ac2abe9505ca6411d5ee37f1771d4c Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:06:11 +0100 Subject: k 5 Reaktor/plugins: converted plugins from makefu/2/Reaktor --- krebs/5pkgs/Reaktor/plugins.nix | 124 ++++++++++++++++++++++----- krebs/5pkgs/Reaktor/scripts/random-issue.sh | 20 +++++ krebs/5pkgs/Reaktor/scripts/sed-plugin.py | 53 ++++++++++++ krebs/5pkgs/Reaktor/scripts/shack-correct.sh | 6 ++ makefu/2configs/Reaktor/full.nix | 18 ---- makefu/2configs/Reaktor/random-emoji.nix | 26 ------ makefu/2configs/Reaktor/random-emoji.sh | 6 -- makefu/2configs/Reaktor/random-issue.sh | 20 ----- makefu/2configs/Reaktor/sed-plugin.nix | 18 ---- makefu/2configs/Reaktor/sed-plugin.py | 53 ------------ makefu/2configs/Reaktor/shack-correct.nix | 20 ----- makefu/2configs/Reaktor/shack-correct.sh | 6 -- makefu/2configs/Reaktor/simpleExtend.nix | 19 ---- makefu/2configs/Reaktor/stockholmLentil.nix | 27 ------ makefu/2configs/Reaktor/titlebot.nix | 38 -------- 15 files changed, 181 insertions(+), 273 deletions(-) create mode 100644 krebs/5pkgs/Reaktor/scripts/random-issue.sh create mode 100644 krebs/5pkgs/Reaktor/scripts/sed-plugin.py create mode 100644 krebs/5pkgs/Reaktor/scripts/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/full.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.sh delete mode 100644 makefu/2configs/Reaktor/random-issue.sh delete mode 100644 makefu/2configs/Reaktor/sed-plugin.nix delete mode 100644 makefu/2configs/Reaktor/sed-plugin.py delete mode 100644 makefu/2configs/Reaktor/shack-correct.nix delete mode 100644 makefu/2configs/Reaktor/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/simpleExtend.nix delete mode 100644 makefu/2configs/Reaktor/stockholmLentil.nix delete mode 100644 makefu/2configs/Reaktor/titlebot.nix diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 05ede38e1..3b2508862 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -1,38 +1,118 @@ { stdenv, lib, pkgs, makeWrapper }: rec { - buildReaktorPlugin = { name - # TODO: profiles - , extraConfig + # Begin API + buildBaseReaktorPlugin = { name + , config # python extra configuration for plugin , phases ? [] , ... } @ attrs: stdenv.mkDerivation (attrs // { name = "Reaktor-plugin-" + name; - phases = phases ++ [ "installPhase" ]; isReaktorPlugin = true; }); - random-emoji = buildReaktorPlugin rec { - name = "random-emoji"; - src = ./scripts/random-emoji.sh; + buildSimpleReaktorPlugin = name: { script + , path ? [] + , env ? {} + , pattern ? "" + , ... } @ attrs: + let + path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); }; + src_dir = pkgs.substituteAll ( { + inherit name; + dir = "bin"; + isExecutable = true; + src = script; + }); + src_file = "${src_dir}/bin/${name}"; + config = '' + public_commands.insert(0,{ + 'capname' : "${name}", + 'pattern' : ${if pattern == "" then + ''indirect_pattern.format("${name}")'' else + ''"${pattern}"'' }, + 'argv' : ["${src_file}"], + 'env' : ${builtins.toJSON path_env // env})}) + ''; + config_file = pkgs.writeText "plugin.py" config; + in buildBaseReaktorPlugin (attrs // rec { + inherit name config; + + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin $out/etc/Reaktor + ln -s ${src_file} $out/bin + wrapProgram $out/bin/${name} \ + --prefix PATH : ${path_env.PATH} + ln -s ${config_file} $out/etc/Reaktor/plugin.py + ''; + + }); + # End API + + # Begin Plugins + random-emoji = buildSimpleReaktorPlugin "emoji" { + path = with pkgs; [ gnused gnugrep xmlstarlet curl ]; + script = ./scripts/random-emoji.sh; + }; + + sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { + path = [ pkgs.gnused ]; + # only support s///gi the plugin needs to see every msg + # TODO: this will eat up the last regex, fix Reaktor to support fallthru + pattern = "^(?P.*)$$"; + script = ./scripts/sed-plugin.py; + }; + + shack-correct = buildSimpleReaktorPlugin "shack-correct" { + path = [ pkgs.gnused ]; + pattern = "^(?P.*Shack.*)$$"; + script = ./scripts/shack-correct.sh; + }; + + nixos-version = buildSimpleReaktorPlugin "nixos-version" { + script = pkgs.writeScript "nixos-version" '' + #! /bin/sh + . /etc/os-release + echo "$PRETTY_NAME" + ''; + }; + stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { + script = ./scripts/random-issue.sh; + path = with pkgs; [ git gnused lentil ]; + env = { "origin"= "http://cgit.gum/stockholm"; }; + }; + + titlebot = + let + pypkgs = pkgs.python3Packages; + titlebot_cmds = pypkgs.buildPythonPackage { + name = "titlebot_cmds"; + propagatedBuildInputs = with pypkgs; [ setuptools ]; + src = pkgs.fetchurl { + url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; + sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; + }; + }; + in buildBaseReaktorPlugin rec { + name = "titlebot"; phases = [ "installPhase" ]; - buildInputs = [ makeWrapper ]; installPhase = '' - mkdir -p $out/bin - install -vm 755 ${src} $out/bin/random-emoji.sh - wrapProgram $out/bin/random-emoji.sh \ - --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl])}; + mkdir -p $out + ln -s ${titlebot_cmds}/* $out ''; - extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["random-emoji.sh"]) + config = '' + def titlebot_cmd(cmd): + from os import environ + return { 'capname': cmd, + 'env': { 'TITLEDB': + environ['state_dir']+'/suggestions.json' }, + 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', + 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } + for i in ['up','help','list','top','new']: + public_commands.insert(0,titlebot_cmd(i)) + commands.insert(0,titlebot_cmd('clear')) ''; }; } diff --git a/krebs/5pkgs/Reaktor/scripts/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh new file mode 100644 index 000000000..5c47c6156 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/random-issue.sh @@ -0,0 +1,20 @@ +#! /bin/sh +set -eu +# requires env: +# $state_dir +# $origin + +# in PATH: git,lentil,coreutils +subdir=`echo "$1" | tr -dc "[:alnum:]"` +name=`echo "$origin" | tr -dc "[:alnum:]"` +track="$state_dir/$name-checkout" +(if test -e "$track" ;then + cd "$track" + git fetch origin master + git reset --hard origin/master +else + git clone "$origin" "$track" +fi) >&2 + +cd "$track" +lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py new file mode 100644 index 000000000..8103c9585 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Usage: +# _from=krebs state_dir=. python sed-plugin.py 'dick butt' +# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' +## dick bull +import shelve +from os import environ +from os.path import join +from sys import argv +d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) +usr = environ['_from'] +import re + +def is_regex(line): + myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') + return myre.match(line) + +line = argv[1] +m = is_regex(line) + +if m: + f,t,flagstr = m.groups() + fn = f.replace('\/','/') + tn = t.replace('\/','/') + flags = 0 + count = 1 + if flagstr: + if 'i' in flagstr: + flags = re.IGNORECASE + if 'g' in flagstr: + count = 0 + else: + flagstr = '' + last = d.get(usr,None) + if last: + #print(re.sub(fn,tn,last,count=count,flags=flags)) + from subprocess import Popen,PIPE + p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) + so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) + if p.returncode: + print("something went wrong when trying to process your regex: {}".format(se.decode())) + ret = so.decode() + print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) + if ret: + d[usr] = ret + + else: + print("no last message") +else: + d[usr] = line + +d.close() diff --git a/krebs/5pkgs/Reaktor/scripts/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh new file mode 100644 index 000000000..3b4d04f80 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh @@ -0,0 +1,6 @@ +#! /bin/sh +set -eu +printf "Sie meinten wohl \"" +echo -n $@ | sed 's/Shack/shack/g' +echo "\"" +echo "${_from}--" diff --git a/makefu/2configs/Reaktor/full.nix b/makefu/2configs/Reaktor/full.nix deleted file mode 100644 index 50620890f..000000000 --- a/makefu/2configs/Reaktor/full.nix +++ /dev/null @@ -1,18 +0,0 @@ -_: -{ - # implementation of the complete Reaktor bot - imports = [ - #./stockholmLentil.nix - ./simpleExtend.nix - ./random-emoji.nix - ./titlebot.nix - ./shack-correct.nix - ./sed-plugin.nix - ]; - krebs.Reaktor.nickname = "Reaktor|bot"; - krebs.Reaktor.enable = true; - - krebs.Reaktor.extraEnviron = { - REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace"; - }; -} diff --git a/makefu/2configs/Reaktor/random-emoji.nix b/makefu/2configs/Reaktor/random-emoji.nix deleted file mode 100644 index 3113a826b..000000000 --- a/makefu/2configs/Reaktor/random-emoji.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - rpkg = pkgs.substituteAll( { - name="random-emoji"; - dir= "bin"; - isExecutable=true; - src= ./random-emoji.sh; - }); - rpkg-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["${rpkg}/bin/random-emoji"], - 'env' : { 'PATH':'${rpkg-path}' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/makefu/2configs/Reaktor/random-emoji.sh deleted file mode 100644 index 386aa68b9..000000000 --- a/makefu/2configs/Reaktor/random-emoji.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -curl http://emojicons.com/random -s | \ - grep data-text | \ - sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \ - head -n 1 | \ - xmlstarlet unesc diff --git a/makefu/2configs/Reaktor/random-issue.sh b/makefu/2configs/Reaktor/random-issue.sh deleted file mode 100644 index 5c47c6156..000000000 --- a/makefu/2configs/Reaktor/random-issue.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh -set -eu -# requires env: -# $state_dir -# $origin - -# in PATH: git,lentil,coreutils -subdir=`echo "$1" | tr -dc "[:alnum:]"` -name=`echo "$origin" | tr -dc "[:alnum:]"` -track="$state_dir/$name-checkout" -(if test -e "$track" ;then - cd "$track" - git fetch origin master - git reset --hard origin/master -else - git clone "$origin" "$track" -fi) >&2 - -cd "$track" -lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/makefu/2configs/Reaktor/sed-plugin.nix b/makefu/2configs/Reaktor/sed-plugin.nix deleted file mode 100644 index a451e0d3e..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = ./sed-plugin.py; -in { - #TODO: this will eat up the last regex, fix Reaktor - krebs.Reaktor.extraConfig = '' - public_commands.append({ - 'capname' : "sed-plugin", - # only support s///gi - 'pattern' : '^(?P.*)$$', - 'argv' : ["${pkgs.python3}/bin/python3","${script}"], - 'env' : { 'state_dir' : workdir, - 'PATH':'${lib.makeSearchPath "bin" [pkgs.gnused]}' }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/makefu/2configs/Reaktor/sed-plugin.py deleted file mode 100644 index 8103c9585..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -# Usage: -# _from=krebs state_dir=. python sed-plugin.py 'dick butt' -# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' -## dick bull -import shelve -from os import environ -from os.path import join -from sys import argv -d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) -usr = environ['_from'] -import re - -def is_regex(line): - myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') - return myre.match(line) - -line = argv[1] -m = is_regex(line) - -if m: - f,t,flagstr = m.groups() - fn = f.replace('\/','/') - tn = t.replace('\/','/') - flags = 0 - count = 1 - if flagstr: - if 'i' in flagstr: - flags = re.IGNORECASE - if 'g' in flagstr: - count = 0 - else: - flagstr = '' - last = d.get(usr,None) - if last: - #print(re.sub(fn,tn,last,count=count,flags=flags)) - from subprocess import Popen,PIPE - p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) - so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) - if p.returncode: - print("something went wrong when trying to process your regex: {}".format(se.decode())) - ret = so.decode() - print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) - if ret: - d[usr] = ret - - else: - print("no last message") -else: - d[usr] = line - -d.close() diff --git a/makefu/2configs/Reaktor/shack-correct.nix b/makefu/2configs/Reaktor/shack-correct.nix deleted file mode 100644 index 8f30807f1..000000000 --- a/makefu/2configs/Reaktor/shack-correct.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = pkgs.substituteAll ( { - name="shack-correct"; - isExecutable=true; - dir = ""; - src = ./shack-correct.sh; - }); -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "shack-correct", - 'pattern' : '^(?P.*Shack.*)$$', - 'argv' : ["${script}"], - 'env' : { }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/makefu/2configs/Reaktor/shack-correct.sh deleted file mode 100644 index 3b4d04f80..000000000 --- a/makefu/2configs/Reaktor/shack-correct.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh -set -eu -printf "Sie meinten wohl \"" -echo -n $@ | sed 's/Shack/shack/g' -echo "\"" -echo "${_from}--" diff --git a/makefu/2configs/Reaktor/simpleExtend.nix b/makefu/2configs/Reaktor/simpleExtend.nix deleted file mode 100644 index 95175a4e0..000000000 --- a/makefu/2configs/Reaktor/simpleExtend.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - nixos-version-script = pkgs.writeScript "nix-version" '' - #! /bin/sh - . /etc/os-release - echo "$PRETTY_NAME" - ''; -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "nixos-version", - 'pattern' : indirect_pattern.format("nixos-version"), - 'argv' : ["${nixos-version-script}"], - 'env' : { 'state_dir': workdir } }) - ''; -} - diff --git a/makefu/2configs/Reaktor/stockholmLentil.nix b/makefu/2configs/Reaktor/stockholmLentil.nix deleted file mode 100644 index 21f0305fb..000000000 --- a/makefu/2configs/Reaktor/stockholmLentil.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - random-issue = pkgs.substituteAll( { - name="random-issue"; - dir= "bin"; - isExecutable=true; - src= ./random-issue.sh; - }); - random-issue-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - git - gnused - lentil]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "stockholm-issue", - 'pattern' : indirect_pattern.format("stockholm-issue"), - 'argv' : ["${random-issue}/bin/random-issue"], - 'env' : { 'state_dir': workdir, - 'PATH':'${random-issue-path}', - 'origin':'http://cgit.pnp/stockholm' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/titlebot.nix b/makefu/2configs/Reaktor/titlebot.nix deleted file mode 100644 index 9ef02548b..000000000 --- a/makefu/2configs/Reaktor/titlebot.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ stdenv,config, lib, pkgs, ... }: - -with pkgs; -let - pypkgs = pkgs.python3Packages; - titlebot_cmds = pypkgs.buildPythonPackage { - name = "titlebot_cmds"; - propagatedBuildInputs = with pypkgs; [ setuptools ]; - src = fetchurl { - # https://github.com/makefu/reaktor-titlebot tag 2.1.0 - url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; - sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; - }; - }; - pub_cmds = ["up" "help" "list" "top" "highest" "undo" ]; - priv_cmds = [ "clear" ]; -in { - # TODO: write identify file in - # {config.users.extraUsers.Reaktor.home}/state/admin.lst - krebs.Reaktor.extraConfig = '' - def titlebot_cmd(cmd): - return { - 'capname': cmd, - 'env': { - 'TITLEDB': - '${config.users.extraUsers.Reaktor.home}/suggestions.json' - }, - 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', - 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } - # TODO: for each element in ${titlebot_cmds}/bin/* - public_commands.insert(0,titlebot_cmd('up')) - public_commands.insert(0,titlebot_cmd('help')) - public_commands.insert(0,titlebot_cmd('list')) - public_commands.insert(0,titlebot_cmd('top')) - public_commands.insert(0,titlebot_cmd('new')) - commands.insert(0,titlebot_cmd('clear')) - ''; -} -- cgit v1.2.3 From a2f5e7e320bb0fbca0a0694d91e4fb20dc4ef329 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:31:09 +0100 Subject: k 5 ReaktorPlugins: hotfix for env generation --- krebs/5pkgs/Reaktor/plugins.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 3b2508862..b1a61d3fa 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON path_env // env})}) + 'env' : ${builtins.toJSON (path_env // env)})}) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { -- cgit v1.2.3 From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:41:41 +0100 Subject: m 3 Reaktor: now supports plugin infra see m/1/pornocauster --- krebs/3modules/Reaktor.nix | 5 ++++- krebs/5pkgs/Reaktor/plugins.nix | 4 ++-- makefu/1systems/pornocauster.nix | 13 +++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 59058bffc..607eb7cac 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -9,6 +9,7 @@ let ${cfg.overrideConfig} '' else ""} ## Extra Config + ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)} ${cfg.extraConfig} ''; cfg = config.krebs.Reaktor; @@ -35,7 +36,6 @@ let ''; }; - overrideConfig = mkOption { default = null; type = types.nullOr types.str; @@ -44,6 +44,9 @@ let Reaktor default cfg can be retrieved via `reaktor get-config` ''; }; + plugins = mkOption { + default = [pkgs.ReaktorPlugins.nixos-version]; + }; extraConfig = mkOption { default = ""; type = types.string; diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index b1a61d3fa..5c7b89f5c 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON (path_env // env)})}) + 'env' : ${builtins.toJSON (path_env // env)} }) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { @@ -81,7 +81,7 @@ rec { stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { script = ./scripts/random-issue.sh; path = with pkgs; [ git gnused lentil ]; - env = { "origin"= "http://cgit.gum/stockholm"; }; + env = { "origin" = "http://cgit.gum/stockholm"; }; }; titlebot = diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix index 28b77d330..690e26b36 100644 --- a/makefu/1systems/pornocauster.nix +++ b/makefu/1systems/pornocauster.nix @@ -26,6 +26,7 @@ # services ../2configs/git/brain-retiolum.nix ../2configs/tor.nix + # ../2configs/buildbot-standalone.nix # hardware specifics are in here ../2configs/hw/tp-x220.nix @@ -36,14 +37,14 @@ ]; nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; - buildbot = let - pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; - in pkgs1509.buildbot; }; - makefu.buildbot.master.enable = true; - #krebs.Reaktor.enable = true; - #krebs.Reaktor.nickname = "makefu|r"; + krebs.Reaktor = { + enable = true; + nickname = "makefu|r"; + plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ]; + }; + # nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ]; environment.systemPackages = with pkgs;[ -- cgit v1.2.3 From ecab96d64f057d639795578784f2785ee5f486a9 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 27 Dec 2015 19:33:12 +0100 Subject: tv vim: ensure relevant dirs exist --- tv/2configs/vim.nix | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/tv/2configs/vim.nix b/tv/2configs/vim.nix index 0822fb5bf..0537fa7d8 100644 --- a/tv/2configs/vim.nix +++ b/tv/2configs/vim.nix @@ -4,7 +4,7 @@ with lib; let out = { environment.systemPackages = [ - pkgs.vim + vim ]; # Nano really is just a stupid name for Vim. @@ -22,14 +22,38 @@ let "${pkgs.vimPlugins.undotree}/share/vim-plugins/undotree" ]; + dirs = { + backupdir = "$HOME/.cache/vim/backup"; + swapdir = "$HOME/.cache/vim/swap"; + undodir = "$HOME/.cache/vim/undo"; + }; + files = { + viminfo = "$HOME/.cache/vim/info"; + }; + + mkdirs = let + dirOf = s: let out = concatStringsSep "/" (init (splitString "/" s)); + in assert out != ""; out; + alldirs = attrValues dirs ++ map dirOf (attrValues files); + in unique (sort lessThan alldirs); + + vim = pkgs.writeScriptBin "vim" '' + #! ${pkgs.dash}/bin/dash + set -f + umask 0077 + ${concatStringsSep "\n" (map (x: "mkdir -p ${x}") mkdirs)} + umask 0022 + exec ${pkgs.vim}/bin/vim "$@" + ''; + vimrc = pkgs.writeText "vimrc" '' set nocompatible set autoindent set backspace=indent,eol,start set backup - set backupdir=$HOME/.vim/backup/ - set directory=$HOME/.vim/cache// + set backupdir=${dirs.backupdir}/ + set directory=${dirs.swapdir}// set hlsearch set incsearch set mouse=a @@ -40,11 +64,11 @@ let set showcmd set showmatch set ttimeoutlen=0 - set undodir=$HOME/.vim/undo + set undodir=${dirs.undodir} set undofile set undolevels=1000000 set undoreload=1000000 - set viminfo='20,<1000,s100,h,n$HOME/.vim/cache/info + set viminfo='20,<1000,s100,h,n${files.viminfo} set visualbell set wildignore+=*.o,*.class,*.hi,*.dyn_hi,*.dyn_o set wildmenu -- cgit v1.2.3 From 5b11cb09e24b8f6b89f80f6ca092cf4000418741 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 27 Dec 2015 23:12:01 +0100 Subject: tv: disable display-manager --- tv/2configs/xserver/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tv/2configs/xserver/default.nix b/tv/2configs/xserver/default.nix index f56da7dcc..facde4e76 100644 --- a/tv/2configs/xserver/default.nix +++ b/tv/2configs/xserver/default.nix @@ -48,7 +48,7 @@ let "slock" ]; - systemd.services.display-manager = mkForce {}; + systemd.services.display-manager.enable = false; services.xserver.enable = true; -- cgit v1.2.3 From b17e484d614486dc0c1cc45a954ead28f15dc074 Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 28 Dec 2015 00:02:58 +0100 Subject: tv backup: initial commit --- tv/1systems/cd.nix | 1 + tv/1systems/xu.nix | 1 + tv/2configs/backup.nix | 254 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 tv/2configs/backup.nix diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix index 8c2a9ae43..7cb903a44 100644 --- a/tv/1systems/cd.nix +++ b/tv/1systems/cd.nix @@ -7,6 +7,7 @@ with lib; krebs.build.target = "root@cd.internet"; imports = [ + ../2configs/backup.nix ../2configs/hw/CAC-Developer-2.nix ../2configs/fs/CAC-CentOS-7-64bit.nix #../2configs/consul-server.nix diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index 1f3e010a4..e1a9076dc 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -9,6 +9,7 @@ with lib; "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a"; imports = [ + ../2configs/backup.nix # TODO ../2configs/hw/x220.nix #../2configs/consul-client.nix ../2configs/git.nix diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix new file mode 100644 index 000000000..1cef0a6dc --- /dev/null +++ b/tv/2configs/backup.nix @@ -0,0 +1,254 @@ +{ config, lib, pkgs, ... }: +with lib; +let + # Users that are allowed to connect to the backup user. + # Note: the user must own a push plan destination otherwise no rsync. + backup-users = [ + config.krebs.users.tv + ]; + + ## TODO parse.file-location admit user + ## loc has the form : + #parse.file-location = loc: let + # parts = splitString ":" loc; + # host-name = head parts; + # path = concatStringsSep ":" (tail parts); + #in { + # type = "types.krebs.file-location"; + # host = config.krebs.hosts.${host-name}; + # path = path; + #}; + + # TODO assert plan.dst.path & co + plans = with config.krebs.users; with config.krebs.hosts; addNames { + xu-test-cd = { + method = "push"; + #src = parse.file-location xu:/tmp/xu-test; + #dst = parse.file-location cd:/krebs/backup/xu-test; + src = { user = tv; host = xu; path = "/tmp/xu-test"; }; + dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; }; + startAt = "0,6,12,18:00"; + retain = { + hourly = 4; # sneakily depends on startAt + daily = 7; + weekly = 4; + monthly = 3; + }; + }; + #xu-test-wu = { + # method = "push"; + # dst = { user = tv; host = wu; path = "/krebs/backup/xu-test"; }; + #}; + cd-test-xu = { + method = "pull"; + #src = parse.file-location cd:/tmp/cd-test; + #dst = parse.file-location xu:/bku/cd-test; + src = { user = tv; host = cd; path = "/tmp/cd-test"; }; + dst = { user = tv; host = xu; path = "/bku/cd-test"; }; + }; + + }; + + out = { + #options.krebs.backup = api; + config = imp; + }; + + imp = { + users.groups.backup.gid = genid "backup"; + users.users = map makeUser (filter isPushDst (attrValues plans)); + systemd.services = + flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: { + name = "backup.${name}"; + value = makePushService plan; + }); + }; + + + # TODO getFQDN: admit hosts in other domains + getFQDN = host: "${host.name}.${config.krebs.search-domain}"; + + isPushSrc = plan: + plan.method == "push" && + plan.src.host.name == config.krebs.build.host.name; + + makePushService = plan: assert isPushSrc plan; { + startAt = plan.startAt; + serviceConfig.ExecStart = writeSh plan "rsync" '' + exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [ + "-a" + "-e" + "${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}" + "${plan.src.path}" + "${plan.name}@${getFQDN plan.dst.host}::push" + ]} + ''; + }; + + isPushDst = plan: + plan.method == "push" && + plan.dst.host.name == config.krebs.build.host.name; + + makeUser = plan: assert isPushDst plan; rec { + name = plan.name; + uid = genid name; + group = config.users.groups.backup.name; + home = plan.dst.path; + createHome = true; + shell = "${writeSh plan "shell" '' + case $2 in + 'rsync --server --daemon .') + exec ${backup.rsync plan [ "--server" "--daemon" "." ]} + ;; + ''') + echo "ERROR: no command specified" >&2 + exit 23 + ;; + *) + echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2 + exit 23 + ;; + esac + ''}"; + openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ]; + }; + + rsync = plan: args: writeSh plan "rsync" '' + install -v -m 0700 -d ${plan.dst.path}/push >&2 + install -v -m 0700 -d ${plan.dst.path}/list >&2 + + ${pkgs.rsync}/bin/rsync \ + --config=${backup.rsyncd-conf plan { + post-xfer = writeSh plan "rsyncd.post-xfer" '' + case $RSYNC_EXIT_STATUS in 0) + exec ${backup.rsnapshot plan { + preexec = writeSh plan "rsnapshot.preexec" '' + touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL + ''; + postexec = writeSh plan "rsnapshot.postexec" '' + rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL + ''; + }} + esac + ''; + }} \ + ${toString (map shell.escape args)} + + fail=0 + for i in monthly weekly daily hourly; do + if test -e ${plan.dst.path}/rsnapshot.$i; then + rm ${plan.dst.path}/rsnapshot.$i + echo "ERROR: $i snapshot failed" >&2 + fail=1 + fi + done + if test $fail != 0; then + exit -1 + fi + ''; + + rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" '' + fake super = yes + use chroot = no + lock file = ${plan.dst.path}/rsyncd.lock + + [push] + max connections = 1 + path = ${plan.dst.path}/push + write only = yes + read only = no + post-xfer exec = ${conf.post-xfer} + + [list] + path = ${plan.dst.path}/list + read only = yes + write only = no + ''; + + rsnapshot = plan: conf: writeSh plan "rsnapshot" '' + rsnapshot() { + ${pkgs.proot}/bin/proot \ + -b /bin \ + -b /nix \ + -b /run/current-system \ + -b ${plan.dst.path} \ + -r ${plan.dst.path} \ + -w / \ + ${pkgs.rsnapshot}/bin/rsnapshot \ + -c ${pkgs.writeText "${plan.name}.rsnapshot.conf" '' + config_version 1.2 + snapshot_root ${plan.dst.path}/list + cmd_cp ${pkgs.coreutils}/bin/cp + cmd_du ${pkgs.coreutils}/bin/du + #cmd_rm ${pkgs.coreutils}/bin/rm + cmd_rsync ${pkgs.rsync}/bin/rsync + cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff + cmd_preexec ${conf.preexec} + cmd_postexec ${conf.postexec} + retain hourly 4 + retain daily 7 + retain weekly 4 + retain monthly 3 + lockfile ${plan.dst.path}/rsnapshot.pid + link_dest 1 + backup /push ./ + verbose 4 + ''} \ + "$@" + } + + cd ${plan.dst.path}/list/ + + now=$(date +%s) + is_older_than() { + test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \ + -ge $2 + } + + # TODO report stale snapshots + # i.e. there are $interval.$i > $interval.$max + + hour_s=3600 + day_s=86400 + week_s=604800 + month_s=2419200 # 4 weeks + + set -- + + if test -e weekly.3 && is_older_than monthly.0 $month_s; then + set -- "$@" monthly + fi + + if test -e daily.6 && is_older_than weekly.0 $week_s; then + set -- "$@" weekly + fi + + if test -e hourly.3 && is_older_than daily.0 $day_s; then + set -- "$@" daily + fi + + if is_older_than hourly.0 $hour_s; then + set -- "$@" hourly + fi + + + if test $# = 0; then + echo "taking no snapshots" >&2 + else + echo "taking snapshots: $@" >&2 + fi + + export RSNAPSHOT_INTERVAL + for RSNAPSHOT_INTERVAL; do + rsnapshot "$RSNAPSHOT_INTERVAL" + done + ''; + + writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" '' + #! ${pkgs.dash}/bin/dash + set -efu + export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])} + ${text} + ''; + +in out -- cgit v1.2.3 From 743842268327b5fd12ba4d19b6260d47535976a3 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 10:58:13 +0100 Subject: k 5 default: populate supports infesting arg by setting infesting for populate, data will be written to /mnt instead of root. --- krebs/default.nix | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/krebs/default.nix b/krebs/default.nix index ad0205426..81ddd3ea6 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -36,6 +36,7 @@ let out = { { system ? current-host-name , target ? system }@args: let + config = get-config system; in '' #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} @@ -47,6 +48,10 @@ let out = { ${builtins.readFile ./4lib/infest/install-nix.sh} ''} + # Prepare target source via bind-mounting + + (${populate (args // { infesting = true;}) }) + (${nixos-install args}) ${rootssh target '' @@ -98,7 +103,6 @@ let out = { #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} # krebs.nixos-install - (${populate args}) ${rootssh target '' export PATH; PATH=/root/.nix-profile/bin:$PATH @@ -205,6 +209,7 @@ let out = { populate = { system ? current-host-name , target ? system + , infesting ? false }@args: let out = '' #! /bin/sh @@ -217,6 +222,8 @@ let out = { ["dir" "git"])} ''; + + target_prefix=lib.optionalString infesting "/mnt"; config = get-config system; current-host = config.krebs.hosts.${current-host-name}; @@ -225,17 +232,18 @@ let out = { methods.dir = config: let can-push = config.host.name == current-host.name; + target-path = target_prefix + config.target-path; push-method = '' rsync \ --exclude .git \ --exclude .graveyard \ --exclude old \ --exclude tmp \ - --rsync-path='mkdir -p ${config.target-path} && rsync' \ + --rsync-path='mkdir -p ${target-path} && rsync' \ --delete-excluded \ -vrLptgoD \ ${config.path}/ \ - root@${target}:${config.target-path} + root@${target}:${target-path} ''; in if can-push then push-method else @@ -244,9 +252,10 @@ let out = { throw "No way to push ${dir} from ${current-host.name} to ${target}"; methods.git = config: - rootssh target '' - mkdir -p ${config.target-path} - cd ${config.target-path} + let target-path = target_prefix + config.target-path; + in rootssh target '' + mkdir -p ${target-path} + cd ${target-path} if ! test -e .git; then git init fi -- cgit v1.2.3 From 95caa8d7fb6f72cbf5064256f71750096d32a6c0 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 13:23:34 +0100 Subject: k 5 test/infest-cac-centos7: use defer, loop until we get a working cac box --- krebs/5pkgs/test/infest-cac-centos7/notes | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index cfb074423..3f4fcd859 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -8,6 +8,17 @@ set -eufx krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} +clear_defer(){ + echo "${trapstr:-exit}" + trap - INT TERM EXIT KILL +} +defer(){ + if test -z "${debug:-}"; then + trapstr="$1;${trapstr:-exit}" + trap "$trapstr" INT TERM EXIT KILL + fi +} + # Sanity if test ! -r "$krebs_cred";then echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 @@ -24,8 +35,8 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json export cac_tasks_cache=$krebs_secrets/tasks_cache.json export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" -trap "$TRAP" INT TERM EXIT +defer "trap - INT TERM EXIT" +defer "rm -r $krebs_secrets" cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP;exit" INT TERM EXIT -# TODO: timeout? - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 -o ConnectTimeout=10 \ - cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id +# preserve old trap +old_trapstr=$(clear_defer) +while true;do + # Template 26: CentOS7 + # TODO: use cac templates to determine the real Centos7 template in case it changes + name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ + | jq -r .servername) + id=servername:$name + + clear_defer >/dev/null + defer "cac delete $id" + + # TODO: timeout? + + wait_login_cac(){ + # we wait for 15 minutes + for t in `seq 90`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 + } + # die on timeout + if ! wait_login_cac $id;then + echo "unable to boot a working system within time frame, retrying..." >&2 + echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + eval "$(clear_defer)" + else + echo "got a working system" >&2 + fi +done +clear_defer >/dev/null +defer "cac delete $id;$old_trapstr" mkdir -p shared/2configs/temp cac generatenetworking $id > \ -- cgit v1.2.3 From 246116dabbe849e75612fbdb57b01696913ff27e Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 13:29:11 +0100 Subject: m 2 urlwatch: replace url for cvs2svn --- makefu/2configs/urlwatch.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix index eadffa7dd..e4f639d5b 100644 --- a/makefu/2configs/urlwatch.nix +++ b/makefu/2configs/urlwatch.nix @@ -12,7 +12,7 @@ http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release https://pypi.python.org/simple/bepasty/ https://pypi.python.org/simple/xstatic/ - http://cvs2svn.tigris.org/servlets/ProjectDocumentList?folderID=2976 + http://cvs2svn.tigris.org/svn/cvs2svn/tags/ ]; }; } -- cgit v1.2.3 From 5a9ccbef0abe1f3acb16d716a2e1d7faa9bb0af1 Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 28 Dec 2015 19:43:31 +0100 Subject: {tv 2 => krebs 3} backup --- krebs/3modules/backup.nix | 286 +++++++++++++++++++++++++++++++++++++++++++++ krebs/3modules/default.nix | 1 + krebs/4lib/types.nix | 17 +++ tv/1systems/cd.nix | 1 - tv/1systems/xu.nix | 1 - tv/2configs/backup.nix | 268 +++++------------------------------------- tv/2configs/default.nix | 1 + 7 files changed, 333 insertions(+), 242 deletions(-) create mode 100644 krebs/3modules/backup.nix diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix new file mode 100644 index 000000000..01bb16a2b --- /dev/null +++ b/krebs/3modules/backup.nix @@ -0,0 +1,286 @@ +{ config, lib, pkgs, ... }: +with lib; +let + out = { + options.krebs.backup = api; + config = mkIf cfg.enable imp; + }; + + cfg = config.krebs.backup; + + api = { + enable = mkEnableOption "krebs.backup" // { default = true; }; + plans = mkOption { + default = {}; + type = types.attrsOf (types.submodule ({ + # TODO enable = mkEnableOption "TODO" // { default = true; }; + options = { + method = mkOption { + type = types.enum ["pull" "push"]; + }; + name = mkOption { + type = types.str; + }; + src = mkOption { + type = types.krebs.file-location; + }; + dst = mkOption { + type = types.krebs.file-location; + }; + startAt = mkOption { + type = types.str; + }; + snapshots = mkOption { + type = types.attrsOf (types.submodule { + options = { + format = mkOption { + type = types.str; # TODO date's +FORMAT + }; + retain = mkOption { + type = types.nullOr types.int; + default = null; # null = retain all snapshots + }; + }; + }); + }; + }; + })); + }; + }; + + imp = { + users.groups.backup.gid = genid "backup"; + users.users = {} + // { + root.openssh.authorizedKeys.keys = + map (plan: plan.dst.host.ssh.pubkey) + (filter isPullSrc (attrValues cfg.plans)) + ++ + map (plan: plan.src.host.ssh.pubkey) + (filter isPushDst (attrValues cfg.plans)) + ; + } + ; + systemd.services = + flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: { + name = "backup.${name}.pull"; + value = makePullService plan; + }) + // + flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: { + name = "backup.${name}.push"; + value = makePushService plan; + }) + ; + }; + + isPushSrc = plan: + plan.method == "push" && + plan.src.host.name == config.krebs.build.host.name; + + isPullSrc = plan: + plan.method == "pull" && + plan.src.host.name == config.krebs.build.host.name; + + isPushDst = plan: + plan.method == "push" && + plan.dst.host.name == config.krebs.build.host.name; + + isPullDst = plan: + plan.method == "pull" && + plan.dst.host.name == config.krebs.build.host.name; + + # TODO push destination needs this in the dst.user's PATH + service-path = [ + pkgs.coreutils + pkgs.gnused + pkgs.openssh + pkgs.rsync + pkgs.utillinux + ]; + + # TODO if there is plan.user, then use its privkey + makePushService = plan: assert isPushSrc plan; { + path = service-path; + serviceConfig = { + ExecStart = push plan; + Type = "oneshot"; + }; + startAt = plan.startAt; + }; + + makePullService = plan: assert isPullDst plan; { + path = service-path; + serviceConfig = { + ExecStart = pull plan; + Type = "oneshot"; + }; + startAt = plan.startAt; + }; + + push = plan: let + # We use writeDashBin and return the absolute path so systemd will produce + # nice names in the log, i.e. without the Nix store hash. + out = "${main}/bin/${main.name}"; + + main = writeDashBin "backup.${plan.name}.push" '' + set -efu + dst=${shell.escape plan.dst.path} + + mkdir -m 0700 -p "$dst" + exec flock -n "$dst" ${critical-section} + ''; + + critical-section = writeDash "backup.${plan.name}.push.critical-section" '' + # TODO check if there is a previous + set -efu + identity=${shell.escape plan.src.host.ssh.privkey.path} + src=${shell.escape plan.src.path} + dst_target=${shell.escape "root@${getFQDN plan.dst.host}"} + dst_path=${shell.escape plan.dst.path} + dst=$dst_target:$dst_path + + # Export NOW so runtime of rsync doesn't influence snapshot naming. + export NOW + NOW=$(date +%s) + + echo >&2 "update snapshot: current; $src -> $dst" + rsync >&2 \ + -aAXF --delete \ + -e "ssh -F /dev/null -i $identity" \ + --rsync-path ${shell.escape + "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \ + --link-dest="$dst_path/current" \ + "$src/" \ + "$dst/.partial" + + exec ssh -F /dev/null \ + -i "$identity" \ + "$dst_target" \ + -T \ + env NOW="$NOW" /bin/sh < ${remote-snapshot} + EOF + ''; + + remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" '' + set -efu + dst=${shell.escape plan.dst.path} + + if test -e "$dst/current"; then + mv "$dst/current" "$dst/.previous" + fi + mv "$dst/.partial" "$dst/current" + rm -fR "$dst/.previous" + echo >&2 + + (${(take-snapshots plan).text}) + ''; + + in out; + + # TODO admit plan.dst.user and its ssh identity + pull = plan: let + # We use writeDashBin and return the absolute path so systemd will produce + # nice names in the log, i.e. without the Nix store hash. + out = "${main}/bin/${main.name}"; + + main = writeDashBin "backup.${plan.name}.pull" '' + set -efu + dst=${shell.escape plan.dst.path} + + mkdir -m 0700 -p "$dst" + exec flock -n "$dst" ${critical-section} + ''; + + critical-section = writeDash "backup.${plan.name}.pull.critical-section" '' + # TODO check if there is a previous + set -efu + identity=${shell.escape plan.dst.host.ssh.privkey.path} + src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"} + dst=${shell.escape plan.dst.path} + + # Export NOW so runtime of rsync doesn't influence snapshot naming. + export NOW + NOW=$(date +%s) + + echo >&2 "update snapshot: current; $dst <- $src" + mkdir -m 0700 -p ${shell.escape plan.dst.path} + rsync >&2 \ + -aAXF --delete \ + -e "ssh -F /dev/null -i $identity" \ + --link-dest="$dst/current" \ + "$src/" \ + "$dst/.partial" + mv "$dst/current" "$dst/.previous" + mv "$dst/.partial" "$dst/current" + rm -fR "$dst/.previous" + echo >&2 + + exec ${take-snapshots plan} + ''; + in out; + + take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" '' + set -efu + NOW=''${NOW-$(date +%s)} + dst=${shell.escape plan.dst.path} + + snapshot() {( + : $ns $format $retain + name=$(date --date="@$NOW" +"$format") + if ! test -e "$dst/$ns/$name"; then + echo >&2 "create snapshot: $ns/$name" + mkdir -m 0700 -p "$dst/$ns" + rsync >&2 \ + -aAXF --delete \ + --link-dest="$dst/current" \ + "$dst/current/" \ + "$dst/$ns/.partial.$name" + mv "$dst/$ns/.partial.$name" "$dst/$ns/$name" + echo >&2 + fi + case $retain in + ([0-9]*) + delete_from=$(($retain + 1)) + ls -r "$dst/$ns" \ + | sed -n "$delete_from,\$p" \ + | while read old_name; do + echo >&2 "delete snapshot: $ns/$old_name" + rm -fR "$dst/$ns/$old_name" + done + ;; + (ALL) + : + ;; + esac + )} + + ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }: + toString (map shell.escape [ + "ns=${ns}" + "format=${format}" + "retain=${if retain == null then "ALL" else toString retain}" + "snapshot" + ])) + plan.snapshots)} + ''; + + # TODO getFQDN: admit hosts in other domains + getFQDN = host: "${host.name}.${config.krebs.search-domain}"; + + writeDash = name: text: pkgs.writeScript name '' + #! ${pkgs.dash}/bin/dash + ${text} + ''; + + writeDashBin = name: text: pkgs.writeTextFile { + executable = true; + destination = "/bin/${name}"; + name = name; + text = '' + #! ${pkgs.dash}/bin/dash + ${text} + ''; + }; + +in out diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 740ba67b8..deccecc48 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -7,6 +7,7 @@ let out = { imports = [ ./apt-cacher-ng.nix + ./backup.nix ./bepasty-server.nix ./build.nix ./current.nix diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index c52afa246..81ce659bd 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -177,4 +177,21 @@ types // rec { addr6 = str; hostname = str; label = str; + + krebs.file-location = types.submodule { + options = { + # TODO user + host = mkOption { + type = host; + }; + # TODO merge with ssl.privkey.path + path = mkOption { + type = types.either types.path types.str; + apply = x: { + path = toString x; + string = x; + }.${typeOf x}; + }; + }; + }; } diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix index 7cb903a44..8c2a9ae43 100644 --- a/tv/1systems/cd.nix +++ b/tv/1systems/cd.nix @@ -7,7 +7,6 @@ with lib; krebs.build.target = "root@cd.internet"; imports = [ - ../2configs/backup.nix ../2configs/hw/CAC-Developer-2.nix ../2configs/fs/CAC-CentOS-7-64bit.nix #../2configs/consul-server.nix diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index e1a9076dc..1f3e010a4 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -9,7 +9,6 @@ with lib; "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a"; imports = [ - ../2configs/backup.nix # TODO ../2configs/hw/x220.nix #../2configs/consul-client.nix ../2configs/git.nix diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix index 1cef0a6dc..51d3bb8a7 100644 --- a/tv/2configs/backup.nix +++ b/tv/2configs/backup.nix @@ -1,38 +1,22 @@ -{ config, lib, pkgs, ... }: +{ config, lib, ... }: with lib; -let - # Users that are allowed to connect to the backup user. - # Note: the user must own a push plan destination otherwise no rsync. - backup-users = [ - config.krebs.users.tv - ]; - - ## TODO parse.file-location admit user - ## loc has the form : - #parse.file-location = loc: let - # parts = splitString ":" loc; - # host-name = head parts; - # path = concatStringsSep ":" (tail parts); - #in { - # type = "types.krebs.file-location"; - # host = config.krebs.hosts.${host-name}; - # path = path; - #}; - - # TODO assert plan.dst.path & co - plans = with config.krebs.users; with config.krebs.hosts; addNames { +{ + krebs.backup.plans = addNames { xu-test-cd = { method = "push"; - #src = parse.file-location xu:/tmp/xu-test; - #dst = parse.file-location cd:/krebs/backup/xu-test; - src = { user = tv; host = xu; path = "/tmp/xu-test"; }; - dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; }; - startAt = "0,6,12,18:00"; - retain = { - hourly = 4; # sneakily depends on startAt - daily = 7; - weekly = 4; - monthly = 3; + + src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; }; + dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; }; + + #startAt = "0,6,12,18:00"; + startAt = "minutely"; + snapshots = { + minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; }; }; #xu-test-wu = { @@ -41,214 +25,18 @@ let #}; cd-test-xu = { method = "pull"; - #src = parse.file-location cd:/tmp/cd-test; - #dst = parse.file-location xu:/bku/cd-test; - src = { user = tv; host = cd; path = "/tmp/cd-test"; }; - dst = { user = tv; host = xu; path = "/bku/cd-test"; }; + src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; }; + dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; }; + startAt = "minutely"; + snapshots = { + minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; }; }; - - out = { - #options.krebs.backup = api; - config = imp; - }; - - imp = { - users.groups.backup.gid = genid "backup"; - users.users = map makeUser (filter isPushDst (attrValues plans)); - systemd.services = - flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: { - name = "backup.${name}"; - value = makePushService plan; - }); - }; - - - # TODO getFQDN: admit hosts in other domains - getFQDN = host: "${host.name}.${config.krebs.search-domain}"; - - isPushSrc = plan: - plan.method == "push" && - plan.src.host.name == config.krebs.build.host.name; - - makePushService = plan: assert isPushSrc plan; { - startAt = plan.startAt; - serviceConfig.ExecStart = writeSh plan "rsync" '' - exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [ - "-a" - "-e" - "${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}" - "${plan.src.path}" - "${plan.name}@${getFQDN plan.dst.host}::push" - ]} - ''; - }; - - isPushDst = plan: - plan.method == "push" && - plan.dst.host.name == config.krebs.build.host.name; - - makeUser = plan: assert isPushDst plan; rec { - name = plan.name; - uid = genid name; - group = config.users.groups.backup.name; - home = plan.dst.path; - createHome = true; - shell = "${writeSh plan "shell" '' - case $2 in - 'rsync --server --daemon .') - exec ${backup.rsync plan [ "--server" "--daemon" "." ]} - ;; - ''') - echo "ERROR: no command specified" >&2 - exit 23 - ;; - *) - echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2 - exit 23 - ;; - esac - ''}"; - openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ]; - }; - - rsync = plan: args: writeSh plan "rsync" '' - install -v -m 0700 -d ${plan.dst.path}/push >&2 - install -v -m 0700 -d ${plan.dst.path}/list >&2 - - ${pkgs.rsync}/bin/rsync \ - --config=${backup.rsyncd-conf plan { - post-xfer = writeSh plan "rsyncd.post-xfer" '' - case $RSYNC_EXIT_STATUS in 0) - exec ${backup.rsnapshot plan { - preexec = writeSh plan "rsnapshot.preexec" '' - touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL - ''; - postexec = writeSh plan "rsnapshot.postexec" '' - rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL - ''; - }} - esac - ''; - }} \ - ${toString (map shell.escape args)} - - fail=0 - for i in monthly weekly daily hourly; do - if test -e ${plan.dst.path}/rsnapshot.$i; then - rm ${plan.dst.path}/rsnapshot.$i - echo "ERROR: $i snapshot failed" >&2 - fail=1 - fi - done - if test $fail != 0; then - exit -1 - fi - ''; - - rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" '' - fake super = yes - use chroot = no - lock file = ${plan.dst.path}/rsyncd.lock - - [push] - max connections = 1 - path = ${plan.dst.path}/push - write only = yes - read only = no - post-xfer exec = ${conf.post-xfer} - - [list] - path = ${plan.dst.path}/list - read only = yes - write only = no - ''; - - rsnapshot = plan: conf: writeSh plan "rsnapshot" '' - rsnapshot() { - ${pkgs.proot}/bin/proot \ - -b /bin \ - -b /nix \ - -b /run/current-system \ - -b ${plan.dst.path} \ - -r ${plan.dst.path} \ - -w / \ - ${pkgs.rsnapshot}/bin/rsnapshot \ - -c ${pkgs.writeText "${plan.name}.rsnapshot.conf" '' - config_version 1.2 - snapshot_root ${plan.dst.path}/list - cmd_cp ${pkgs.coreutils}/bin/cp - cmd_du ${pkgs.coreutils}/bin/du - #cmd_rm ${pkgs.coreutils}/bin/rm - cmd_rsync ${pkgs.rsync}/bin/rsync - cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff - cmd_preexec ${conf.preexec} - cmd_postexec ${conf.postexec} - retain hourly 4 - retain daily 7 - retain weekly 4 - retain monthly 3 - lockfile ${plan.dst.path}/rsnapshot.pid - link_dest 1 - backup /push ./ - verbose 4 - ''} \ - "$@" - } - - cd ${plan.dst.path}/list/ - - now=$(date +%s) - is_older_than() { - test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \ - -ge $2 - } - - # TODO report stale snapshots - # i.e. there are $interval.$i > $interval.$max - - hour_s=3600 - day_s=86400 - week_s=604800 - month_s=2419200 # 4 weeks - - set -- - - if test -e weekly.3 && is_older_than monthly.0 $month_s; then - set -- "$@" monthly - fi - - if test -e daily.6 && is_older_than weekly.0 $week_s; then - set -- "$@" weekly - fi - - if test -e hourly.3 && is_older_than daily.0 $day_s; then - set -- "$@" daily - fi - - if is_older_than hourly.0 $hour_s; then - set -- "$@" hourly - fi - - - if test $# = 0; then - echo "taking no snapshots" >&2 - else - echo "taking snapshots: $@" >&2 - fi - - export RSNAPSHOT_INTERVAL - for RSNAPSHOT_INTERVAL; do - rsnapshot "$RSNAPSHOT_INTERVAL" - done - ''; - - writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" '' - #! ${pkgs.dash}/bin/dash - set -efu - export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])} - ${text} - ''; - -in out +} diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index 3400c13b6..c300633bb 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -28,6 +28,7 @@ with lib; imports = [ + ./backup.nix ./vim.nix { # stockholm dependencies -- cgit v1.2.3 From 676d0f748138f0e1fa3cb2177b5a08a857f17fac Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 29 Dec 2015 12:52:35 +0100 Subject: k Zhosts: init bobby (miefda) --- krebs/Zhosts/bobby | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 krebs/Zhosts/bobby diff --git a/krebs/Zhosts/bobby b/krebs/Zhosts/bobby new file mode 100644 index 000000000..aac6e377b --- /dev/null +++ b/krebs/Zhosts/bobby @@ -0,0 +1,11 @@ +Subnet = 10.243.111.112/32 +Subnet = 42:0:0:0:0:0:111:112/128 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEA+AScnIqFdzGl+iRZTNZ7r91n/r1H4GzDsrAupUvJ4mi7nDN4eP8s +uLvKtJp22RxfuF3Kf4KhHb8LHQ8bLLN/KDaNDXrCNBc69d7vvLsjoY+wfGLJNu4Y +Ad/8J4r3rdb83mTA3IHb47T/70MERPBr2gF84YiG6ZoQrPQuTk4lHxaI83SOhjny +0F0ucS/rBV6Vv9y5/756TKi1cFPSpY4X+qeWc8xWrBGJcJiiqYb8ZX2o/lkAJ5c+ +jI/VdybGFVGY9+bp4Jw5xBIo5KGuFnm8+blRmSDDl3joRneKQSx9FAu7RUwoajBu +cEbi1529NReQzIFT6Vt22ymbHftxOiuh4QIDAQAB +-----END RSA PUBLIC KEY----- -- cgit v1.2.3