From 8a04a15a7b812ad8c7aac1ae987ac2e970f4c0ae Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 30 Dec 2022 23:36:50 +0100 Subject: l sync-containers3: use networkd --- lass/3modules/sync-containers3.nix | 66 +++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 37 deletions(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 1371d5233..c04a7758b 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -64,7 +64,6 @@ in { privateNetwork = true; hostBridge = "ctr0"; bindMounts = { - "/etc/resolv.conf".hostPath = "/etc/resolv.conf"; "/var/lib/self/disk" = { hostPath = "/var/lib/sync-containers3/${ctr.name}/disk"; isReadOnly = false; @@ -257,44 +256,26 @@ in { }) (lib.mkIf (cfg.containers != {}) { # networking - networking.networkmanager.unmanaged = [ "ctr0" ]; - networking.interfaces.dummy0.virtual = true; - networking.bridges.ctr0.interfaces = [ "dummy0" ]; - networking.interfaces.ctr0.ipv4.addresses = [{ - address = "10.233.0.1"; - prefixLength = 24; - }]; - systemd.services."dhcpd-ctr0" = { - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - serviceConfig = { - Type = "forking"; - Restart = "always"; - DynamicUser = true; - StateDirectory = "dhcpd-ctr0"; - User = "dhcpd-ctr0"; - Group = "dhcpd-ctr0"; - AmbientCapabilities = [ - "CAP_NET_RAW" # to send ICMP messages - "CAP_NET_BIND_SERVICE" # to bind on DHCP port (67) - ]; - ExecStartPre = "${pkgs.coreutils}/bin/touch /var/lib/dhcpd-ctr0/dhcpd.leases"; - ExecStart = "${pkgs.dhcp}/bin/dhcpd -4 -lf /var/lib/dhcpd-ctr0/dhcpd.leases -cf ${pkgs.writeText "dhpd.conf" '' - default-lease-time 600; - max-lease-time 7200; - authoritative; - ddns-update-style interim; - log-facility local1; # see dhcpd.nix - - option subnet-mask 255.255.255.0; - option routers 10.233.0.1; - # option domain-name-servers 8.8.8.8; # TODO configure dns server - subnet 10.233.0.0 netmask 255.255.255.0 { - range 10.233.0.10 10.233.0.250; - } - ''} ctr0"; + systemd.network.networks.ctr0 = { + name = "ctr0"; + address = [ + "10.233.0.1/24" + ]; + networkConfig = { + IPForward = "yes"; + IPMasquerade = "both"; + ConfigureWithoutCarrier = true; + DHCPServer = "yes"; }; }; + systemd.network.netdevs.ctr0.netdevConfig = { + Kind = "bridge"; + Name = "ctr0"; + }; + networking.networkmanager.unmanaged = [ "ctr0" ]; + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-i ctr0"; target = "ACCEPT"; } + ]; }) (lib.mkIf cfg.inContainer.enable { users.groups.container_sync = {}; @@ -308,6 +289,17 @@ in { cfg.inContainer.pubkey ]; }; + + networking.useHostResolvConf = false; + networking.useNetworkd = true; + systemd.network = { + enable = true; + networks.eth0 = { + matchConfig.Name = "eth0"; + DHCP = "yes"; + dhcpV4Config.UseDNS = true; + }; + }; }) ]; } -- cgit v1.2.3 From 3918ad6cf863646d4f27aabfa9e1cd28a612df47 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 30 Dec 2022 23:37:04 +0100 Subject: l sync-containers3: add rsync timeout --- lass/3modules/sync-containers3.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index c04a7758b..16e04d743 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -100,7 +100,7 @@ in { set -efux if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then touch "$HOME"/incomplete - rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk + rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk rm "$HOME"/incomplete fi ''} -- cgit v1.2.3 From 4591d70e1c4478325f41c57bad8f2eeed4e1d722 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 30 Dec 2022 23:37:46 +0100 Subject: l sync-containers3: don't start if consul fails --- lass/3modules/sync-containers3.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 16e04d743..1a006835d 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -135,7 +135,8 @@ in { ;; 200) # echo 'got 200 from kv, will check payload' - export payload=$(consul kv get containers/${ctr.name}) + payload=$(consul kv get containers/${ctr.name}) || continue + export payload if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then # echo 'we are the host, trying to reach container' if $(retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null); then -- cgit v1.2.3 From d5a4c2181dcbd12fbb94d5bd7dcbf7f4f89763a8 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 2 Jan 2023 18:47:14 +0100 Subject: l sync-containers3: create dir, fix symlink --- lass/3modules/sync-containers3.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 1a006835d..e9f708690 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -50,7 +50,8 @@ in { wantedBy = [ "multi-user.target" ]; serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" '' set -efu - ln -frs /var/state/var_src /var/src + mkdir -p /var/state/var_src + ln -Tfrs /var/state/var_src /var/src if test -e /var/src/nixos-config; then /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || : fi -- cgit v1.2.3 From 5415114141754e6e1382e8e6841b902717c6de1f Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 2 Jan 2023 18:47:26 +0100 Subject: l sync-containers3: fix group --- lass/3modules/sync-containers3.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index e9f708690..645809da5 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -247,7 +247,7 @@ in { users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" { }) cfg.containers; users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({ - group = "container_${ctr.name}"; + group = "${ctr.name}_container"; isNormalUser = true; uid = slib.genid_uint31 "container_${ctr.name}"; home = "/var/lib/sync-containers3/${ctr.name}"; -- cgit v1.2.3 From 7bbcac3e5ee3ad65762e54d85a1786d077cca699 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 2 Jan 2023 18:47:49 +0100 Subject: l sync-containers3: add init script --- lass/3modules/sync-containers3.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 645809da5..053175565 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -255,6 +255,24 @@ in { homeMode = "705"; })) cfg.containers; + environment.systemPackages = lib.mapAttrsToList (_: ctr: (pkgs.writers.writeDashBin "${ctr.name}_init" '' + set -efux + export PATH=${lib.makeBinPath [ + pkgs.coreutils + pkgs.cryptsetup + pkgs.libxfs.bin + ]}:$PATH + truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk + cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey} + cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} + mkfs.xfs /dev/mapper/${ctr.name} + mkdir -p /var/lib/sync-containers3/${ctr.name}/state + mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state + /run/current-system/sw/bin/nixos-container start ${ctr.name} + /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" '' + mkdir -p /var/state + ''} + '')) cfg.containers; }) (lib.mkIf (cfg.containers != {}) { # networking -- cgit v1.2.3 From a38c39424f29bbdfe1493061da05326f9d05d4a0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 2 Jan 2023 18:48:12 +0100 Subject: l sync-containers3: allow ctr0 in FORWARD --- lass/3modules/sync-containers3.nix | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 053175565..02ba0a5ff 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -296,6 +296,10 @@ in { krebs.iptables.tables.filter.INPUT.rules = [ { predicate = "-i ctr0"; target = "ACCEPT"; } ]; + krebs.iptables.tables.filter.FORWARD.rules = [ + { predicate = "-i ctr0"; target = "ACCEPT"; } + { predicate = "-o ctr0"; target = "ACCEPT"; } + ]; }) (lib.mkIf cfg.inContainer.enable { users.groups.container_sync = {}; -- cgit v1.2.3 From 32966b38aed6f495e425bc5f1d54761534bc019b Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 2 Jan 2023 23:31:53 +0100 Subject: l sync-containers3: add runContainer option --- lass/3modules/sync-containers3.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 02ba0a5ff..86aa40f03 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -28,6 +28,10 @@ in { type = lib.types.bool; default = false; }; + runContainer = lib.mkOption { + type = lib.types.bool; + default = true; + }; }; })); }; @@ -74,7 +78,7 @@ in { isReadOnly = false; }; }; - }) cfg.containers; + }) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers); systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [ { "${ctr.name}_syncer" = { @@ -108,7 +112,7 @@ in { ''; }; }; } - { "${ctr.name}_watcher" = { + { "${ctr.name}_watcher" = lib.mkIf ctr.runContainer { path = with pkgs; [ coreutils consul @@ -164,7 +168,7 @@ in { ''; }; }; } - { "${ctr.name}_scheduler" = { + { "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer { wantedBy = [ "multi-user.target" ]; path = with pkgs; [ coreutils -- cgit v1.2.3