From e09559bb786c7cf639dc1428bbe2ed6ba7ced9fb Mon Sep 17 00:00:00 2001 From: tv Date: Tue, 5 Jan 2016 21:47:55 +0100 Subject: krebs.backup.plans.*.{snapshots,startAt} += default --- krebs/3modules/backup.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 01bb16a2b..20bc5dee0 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -28,9 +28,17 @@ let type = types.krebs.file-location; }; startAt = mkOption { - type = types.str; + default = "hourly"; + type = types.str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { + default = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; type = types.attrsOf (types.submodule { options = { format = mkOption { -- cgit v1.2.3 From b44615eebec6055f3b49a8170354ce4bb3213cb4 Mon Sep 17 00:00:00 2001 From: tv Date: Tue, 5 Jan 2016 22:35:05 +0100 Subject: =?UTF-8?q?tv=20backup:=20wu:/home=20=E2=86=92=20xu:/bku/wu-home?= =?UTF-8?q?=20@=2005:00?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- krebs/3modules/tv/default.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 6fd1c4224..31c1a375a 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -247,6 +247,7 @@ with lib; }; }; secure = true; + ssh.privkey.path = ; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa"; }; xu = { -- cgit v1.2.3 From 8647055f0cc1cd4b4b1b389b3b7c2f4ee2211e93 Mon Sep 17 00:00:00 2001 From: tv Date: Tue, 5 Jan 2016 22:57:16 +0100 Subject: krebs.backup: add TODOs --- krebs/3modules/backup.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 20bc5dee0..a1f335905 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -292,3 +292,15 @@ let }; in out +# TODO ionice +# TODO mail on failed push, pull +# TODO mail on missing push +# TODO don't cancel plans on activation +# also, don't hang while deploying at: +# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer +# TODO make sure /bku is properly mounted +# TODO make sure that secure hosts cannot backup to insecure ones +# TODO optionally only backup when src and dst are near enough :) +# TODO try using btrfs for snapshots (configurable) +# TODO warn if partial snapshots are found +# TODO warn if unknown stuff is found in dst path -- cgit v1.2.3 From 98be119acd7e58782797ed79bc42b9e6a45358ff Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jan 2016 21:42:31 +0100 Subject: krebs.urlwatch: bump compatibility to urlwatch-2 configFile is the default configuration urlwatch generates. --- krebs/3modules/urlwatch.nix | 46 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 31cbfcf6e..7a9fb55fd 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -54,6 +54,10 @@ let example = [ https://nixos.org/channels/nixos-unstable/git-revision ]; + apply = map (x: getAttr (typeOf x) { + set = x; + string.url = x; + }); }; verbose = mkOption { type = types.bool; @@ -64,7 +68,40 @@ let }; }; - urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls); + urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + + configFile = toFile "urlwatch.yaml" (toJSON { + display = { + error = true; + new = true; + unchanged = false; + }; + report = { + email = { + enabled = false; + from = ""; + html = false; + smtp = { + host = "localhost"; + keyring = true; + port = 25; + starttls = true; + }; + subject = "{count} changes: {jobs}"; + to = ""; + }; + html.diff = "unified"; + stdout = { + color = true; + enabled = true; + }; + text = { + details = true; + footer = true; + line_length = 75; + }; + }; + }); imp = { systemd.timers.urlwatch = { @@ -109,10 +146,15 @@ let from=${escapeShellArg cfg.from} mailto=${escapeShellArg cfg.mailto} urlsFile=${escapeShellArg urlsFile} + configFile=${escapeShellArg configFile} cd /tmp - urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || : + urlwatch \ + ${optionalString cfg.verbose "-v"} \ + --urls="$urlsFile" \ + --config="$configFile" \ + > changes || : if test -s changes; then date=$(date -R) -- cgit v1.2.3 From c3c4aac670cf0ac26340061cef1527b3d7f64386 Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 1 Feb 2016 03:00:48 +0100 Subject: make deploy2: deploy using nixos-rebuild switch --- krebs/3modules/build.nix | 109 +++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 37 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..0f8aec89d 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,83 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { - options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - url = mkOption { - type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; + options.krebs.build.source-version = mkOption { + type = types.enum [ 1 2 ]; + default = 1; }; - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe - }; - rev = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; + options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" { + v1 = { + dir = mkOption { + type = let + default-host = config.krebs.current.host; + in types.attrsOf (types.submodule ({ config, ... }: { + options = { + host = mkOption { + type = types.host; + default = default-host; + }; + path = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + url = mkOption { + type = types.str; + default = "file://${config.host.name}${config.path}"; + }; + }; + })); + default = {}; + }; + + git = mkOption { + type = with types; attrsOf (submodule ({ config, ... }: { + options = { + url = mkOption { + type = types.str; # TODO must be shell safe + }; + rev = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + }; + })); + default = {}; + }; + }; + + v2 = let + raw = types.either types.str types.path; + url = types.submodule { + options = { + url = mkOption { + type = types.str; + }; + rev = mkOption { + type = types.str; + }; + dev = mkOption { + type = types.str; + }; }; }; - })); - default = {}; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; + default = {}; + }; }; + }; in out -- cgit v1.2.3 From 68655d1ddf078eb1bb3a48ba7e6e9376d913985e Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 3 Feb 2016 11:32:58 +0100 Subject: krebs.git: remove trailing spaces --- krebs/3modules/git.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index e6267d7e6..0fa6b240b 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -27,7 +27,7 @@ let description = '' Enable cgit. Cgit is an attempt to create a fast web interface for the git version - control system, using a built in cache to decrease pressure on the + control system, using a built in cache to decrease pressure on the git server. cgit in this module is being served via fastcgi nginx.This module deploys a http://cgit. nginx configuration and enables nginx @@ -127,7 +127,7 @@ let git-imp = { system.activationScripts.git-init = "${init-script}"; - + # TODO maybe put all scripts here and then use PATH? environment.etc."${etc-base}".source = scriptFarm "git-ssh-authorizers" { @@ -136,7 +136,7 @@ let (map getName (ensureList repo)) (map getName perm.allow-commands) ]) cfg.rules); - + authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [ (map getName (ensureList user)) (map getName (ensureList repo)) @@ -144,7 +144,7 @@ let (map getName perm.allow-receive-modes) ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); }; - + users.extraUsers = singleton rec { description = "Git repository hosting user"; name = "git"; -- cgit v1.2.3 From cdb590be5072712c9552a98f8979aa94288dcbc8 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 3 Feb 2016 13:36:54 +0100 Subject: krebs.git.rules: specify type --- krebs/3modules/git.nix | 193 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 140 insertions(+), 53 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index 0fa6b240b..7b28ffca8 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -44,48 +44,8 @@ let default = "/etc/git"; }; repos = mkOption { - type = types.attrsOf (types.submodule ({ - options = { - desc = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository description. - ''; - }; - section = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository section. - ''; - }; - name = mkOption { - type = types.str; - description = '' - Repository name. - ''; - }; - hooks = mkOption { - type = types.attrsOf types.str; - default = {}; - description = '' - Repository-specific hooks. - ''; - }; - public = mkOption { - type = types.bool; - default = false; - description = '' - Allow everybody to read the repository via HTTP if cgit enabled. - ''; - # TODO allow every configured user to fetch the repository via SSH. - }; - }; - })); - + type = types.attrsOf subtypes.repo; default = {}; - example = literalExample '' { testing = { @@ -99,7 +59,6 @@ let testing2 = { name = "testing2"; }; } ''; - description = '' Repositories. ''; @@ -121,28 +80,156 @@ let ''; }; rules = mkOption { - type = types.unspecified; + type = types.listOf subtypes.rule; + default = []; + example = literalExample '' + singleton { + user = [ config.krebs.users.tv ]; + repo = [ testing ]; # see literal example of repos + perm = push "refs/*" (with lib.git; [ + non-fast-forward create delete merge + ]); + } + ''; + description = '' + Rules. + ''; }; }; + # TODO put into krebs/4lib/types.nix? + subtypes = { + repo = types.submodule ({ + options = { + collaborators = mkOption { + type = types.listOf types.user; + default = []; + description = '' + List of users that should be able to fetch from this repo. + + This option is currently not used by krebs.git but instead can be + used to create rules. See e.g. for + an example. + ''; + }; + desc = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository description. + ''; + }; + section = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository section. + ''; + }; + name = mkOption { + type = types.str; + description = '' + Repository name. + ''; + }; + hooks = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Repository-specific hooks. + ''; + }; + public = mkOption { + type = types.bool; + default = false; + description = '' + Allow everybody to read the repository via HTTP if cgit enabled. + ''; + # TODO allow every configured user to fetch the repository via SSH. + }; + }; + }); + rule = types.submodule ({ config, ... }: { + options = { + user = mkOption { + type = types.listOf types.user; + description = '' + List of users this rule should apply to. + Checked by authorize-command. + ''; + }; + repo = mkOption { + type = types.listOf subtypes.repo; + description = '' + List of repos this rule should apply to. + Checked by authorize-command. + ''; + }; + perm = mkOption { + type = types.submodule { + # TODO generate enum argument from krebs/4lib/git.nix + options = { + allow-commands = mkOption { + type = types.listOf (types.enum (with git; [ + git-receive-pack + git-upload-pack + ])); + default = []; + description = '' + List of commands the rule's users are allowed to execute. + Checked by authorize-command. + ''; + }; + allow-receive-ref = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Ref that can receive objects. + Checked by authorize-push. + ''; + }; + allow-receive-modes = mkOption { + type = types.listOf (types.enum (with git; [ + fast-forward + non-fast-forward + create + delete + merge + ])); + default = []; + description = '' + List of allowed receive modes. + Checked by pre-receive hook. + ''; + }; + }; + }; + description = '' + Permissions granted. + ''; + }; + }; + }); + }; + git-imp = { system.activationScripts.git-init = "${init-script}"; # TODO maybe put all scripts here and then use PATH? environment.etc."${etc-base}".source = scriptFarm "git-ssh-authorizers" { - authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (map getName perm.allow-commands) + authorize-command = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (map getName rule.perm.allow-commands) ]) cfg.rules); - authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (ensureList perm.allow-receive-ref) - (map getName perm.allow-receive-modes) - ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); + authorize-push = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (ensureList rule.perm.allow-receive-ref) + (map getName rule.perm.allow-receive-modes) + ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules)); }; users.extraUsers = singleton rec { -- cgit v1.2.3 From b857a48632128be0324c68be95bee16fb0f1b15f Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 4 Feb 2016 05:40:26 +0100 Subject: krebs.build.populate: init and drop support for v1 --- krebs/3modules/build.nix | 218 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 147 insertions(+), 71 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 0f8aec89d..00142acdd 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,81 +28,157 @@ let type = types.user; }; - options.krebs.build.source-version = mkOption { - type = types.enum [ 1 2 ]; - default = 1; + options.krebs.build.source = let + raw = types.either types.str types.path; + url = types.submodule { + options = { + url = mkOption { + type = types.str; + }; + rev = mkOption { + type = types.str; + }; + dev = mkOption { + type = types.str; + }; + }; + }; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; + default = {}; }; - options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" { - v1 = { - dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { - options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - url = mkOption { - type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; - }; + options.krebs.build.populate = mkOption { + type = types.str; + default = let + source = config.krebs.build.source; + target-user = maybeEnv "target_user" "root"; + target-host = maybeEnv "target_host" config.krebs.build.host.name; + target-path = maybeEnv "target_path" "/var/src"; + out = '' + #! /bin/sh + set -eu - git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe - }; - rev = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - }; - })); - default = {}; - }; - }; + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } - v2 = let - raw = types.either types.str types.path; - url = types.submodule { - options = { - url = mkOption { - type = types.str; - }; - rev = mkOption { - type = types.str; - }; - dev = mkOption { - type = types.str; - }; - }; - }; - in mkOption { - type = types.attrsOf (types.either types.str url); - apply = let f = mapAttrs (_: value: { - string = value; - path = toString value; - set = f value; - }.${typeOf value}); in f; - default = {}; - }; + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + unset tmpdir + trap ' + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + chmod 0755 "$tmpdir" + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + verbose proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + -f ${shell.escape "P /*"} \ + ${concatMapStringsSep " \\\n " + (name: "-f ${shell.escape "R /${name}"}") + (attrNames file-specs)} \ + --delete \ + -vFrlptD \ + ${shell.escape target-path}/ \ + ${shell.escape "${target-user}@${target-host}:${target-path}"} + ''; + + get-schema = uri: + if substring 0 1 uri == "/" + then "file" + else head (splitString ":" uri); + + has-schema = schema: uri: get-schema uri == schema; + + get-url = spec: { + string = spec; + path = toString spec; + set = get-url spec.url; + }.${typeOf spec}; + + git-specs = + filterAttrs (_: spec: has-schema "https" (get-url spec)) source // + filterAttrs (_: spec: has-schema "http" (get-url spec)) source // + filterAttrs (_: spec: has-schema "git" (get-url spec)) source; + + file-specs = + filterAttrs (_: spec: has-schema "file" (get-url spec)) source; + + symlink-specs = + filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; + + git-script = '' + #! /bin/sh + set -efu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + fetch_git() {( + dst_dir=$1 + src_url=$2 + src_ref=$3 + + if ! test -e "$dst_dir"; then + git clone "$src_url" "$dst_dir" + fi + + cd "$dst_dir" + + if ! url=$(git config remote.origin.url); then + git remote add origin "$src_url" + elif test "$url" != "$src_url"; then + git remote set-url origin "$src_url" + fi + + # TODO resolve src_ref to commit hash + hash=$src_ref + + if ! test "$(git log --format=%H -1)" = "$hash"; then + git fetch origin + git checkout "$hash" -- "$dst_dir" + git checkout "$hash" + fi + + git clean -dxf + )} + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: toString (map shell.escape [ + "verbose" + "fetch_git" + "${target-path}/${name}" + spec.url + spec.rev + ])) + git-specs)} + ''; + in out; }; }; -- cgit v1.2.3