From d9d7f7fd32295a3208ace7c2c1cdfaf61ab70f04 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 24 May 2017 10:48:14 +0200 Subject: m 2 aralast: init --- makefu/2configs/logging/external/aralast.nix | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 makefu/2configs/logging/external/aralast.nix (limited to 'makefu/2configs/logging') diff --git a/makefu/2configs/logging/external/aralast.nix b/makefu/2configs/logging/external/aralast.nix new file mode 100644 index 000000000..c335db454 --- /dev/null +++ b/makefu/2configs/logging/external/aralast.nix @@ -0,0 +1,38 @@ +{ config, lib, pkgs, ... }: + +with import ; +let + pkg = pkgs.stdenv.mkDerivation { + name = "aralast-master"; + src = pkgs.fetchFromGitHub { + owner = "makefu"; + repo = "aralast"; + rev = "7121598"; + sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m"; + }; + installPhase = '' + install -m755 -D aralast.sh $out/bin/aralast + ''; + }; +in { + systemd.services.aralast = { + description = "periodically fetch aramark"; + path = [ + pkgs.curl + pkgs.gnugrep + pkgs.gnused + ]; + wantedBy = [ "multi-user.target" ]; + environment = { + INFLUX_HOST = "localhost"; + INFLUX_PORT = "8086"; + }; + # every 10 seconds when the cantina is open + startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0/10"; + serviceConfig = { + User = "nobody"; + ExecStart = "${pkg}/bin/aralast"; + PrivateTmp = true; + }; + }; +} -- cgit v1.2.3 From c29ba49d3dba4386ee90e784912d38b37e63efa6 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 24 May 2017 10:49:34 +0200 Subject: m 2 central-stats-server: redirect http://stats.makefu.r to grafana --- makefu/2configs/logging/central-stats-server.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'makefu/2configs/logging') diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix index 4f7961f32..602fcc6d0 100644 --- a/makefu/2configs/logging/central-stats-server.nix +++ b/makefu/2configs/logging/central-stats-server.nix @@ -12,7 +12,9 @@ in { services.grafana.addr = "0.0.0.0"; services.influxdb.enable = true; - + # redirect grafana to stats.makefu.r + services.nginx.enable = true; + services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000"; # forward these via nginx services.influxdb.extraConfig = { meta.hostname = config.krebs.build.host.name; -- cgit v1.2.3 From 0c92dd719a46139523f6e353c354871bd78024a4 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 25 May 2017 23:19:36 +0200 Subject: m 2: rename stats and share --- makefu/2configs/logging/central-logging-client.nix | 32 --------- makefu/2configs/logging/central-logging-server.nix | 23 ------ makefu/2configs/logging/central-stats-client.nix | 60 ---------------- makefu/2configs/logging/central-stats-server.nix | 84 ---------------------- makefu/2configs/logging/client.nix | 32 +++++++++ makefu/2configs/logging/external/aralast.nix | 38 ---------- makefu/2configs/logging/server.nix | 23 ++++++ 7 files changed, 55 insertions(+), 237 deletions(-) delete mode 100644 makefu/2configs/logging/central-logging-client.nix delete mode 100644 makefu/2configs/logging/central-logging-server.nix delete mode 100644 makefu/2configs/logging/central-stats-client.nix delete mode 100644 makefu/2configs/logging/central-stats-server.nix create mode 100644 makefu/2configs/logging/client.nix delete mode 100644 makefu/2configs/logging/external/aralast.nix create mode 100644 makefu/2configs/logging/server.nix (limited to 'makefu/2configs/logging') diff --git a/makefu/2configs/logging/central-logging-client.nix b/makefu/2configs/logging/central-logging-client.nix deleted file mode 100644 index 04d2de0d0..000000000 --- a/makefu/2configs/logging/central-logging-client.nix +++ /dev/null @@ -1,32 +0,0 @@ -{pkgs, buil, config, ...}: -let - log-server = config.makefu.log-server; - log-port = 9200; -in { - services.journalbeat = { - enable = true; - # TODO: filter for certain journal fields, not all - extraConfig = '' - journalbeat: - name: logs-${config.krebs.build.host.name} - seek_position: cursor - cursor_seek_fallback: tail - write_cursor_state: true - cursor_flush_period: 5s - clean_field_names: true - convert_to_numbers: false - move_metadata_to_field: journal - default_type: journal - output.elasticsearch: - enabled: true - hosts: ["${log-server}:${builtins.toString log-port}"] - template.enabled: false - #output.console: - # enabled: true - logging.level: info - logging.to_syslog: true - logging.selectors: ["*"] - - ''; - }; -} diff --git a/makefu/2configs/logging/central-logging-server.nix b/makefu/2configs/logging/central-logging-server.nix deleted file mode 100644 index 90f8e6680..000000000 --- a/makefu/2configs/logging/central-logging-server.nix +++ /dev/null @@ -1,23 +0,0 @@ -{pkgs, config, ...}: - -with import ; -let - es-port = 9200; - kibana-port = 5601; -in { - services.elasticsearch = { - enable = true; - listenAddress = "0.0.0.0"; - port = es-port; - }; - services.kibana = { - enable = true; - listenAddress = "0.0.0.0"; - port = kibana-port; - }; - - networking.firewall.extraCommands = '' - iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT - iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT - ''; -} diff --git a/makefu/2configs/logging/central-stats-client.nix b/makefu/2configs/logging/central-stats-client.nix deleted file mode 100644 index dd6ddddaf..000000000 --- a/makefu/2configs/logging/central-stats-client.nix +++ /dev/null @@ -1,60 +0,0 @@ -{pkgs, config, ...}: -{ - services.collectd = { - enable = true; - autoLoadPlugin = true; - extraConfig = '' - Hostname ${config.krebs.build.host.name} - LoadPlugin load - LoadPlugin disk - LoadPlugin memory - LoadPlugin df - Interval 30.0 - - LoadPlugin interface - - Interface "*Link" - Interface "lo" - Interface "vboxnet*" - Interface "virbr*" - IgnoreSelected true - - - LoadPlugin df - - MountPoint "/nix/store" - # MountPoint "/run*" - # MountPoint "/sys*" - # MountPoint "/dev" - # MountPoint "/dev/shm" - # MountPoint "/tmp" - FSType "tmpfs" - FSType "binfmt_misc" - FSType "debugfs" - FSType "mqueue" - FSType "hugetlbfs" - FSType "systemd-1" - FSType "cgroup" - FSType "securityfs" - FSType "ramfs" - FSType "proc" - FSType "devpts" - FSType "devtmpfs" - MountPoint "/var/lib/docker/devicemapper" - IgnoreSelected true - - - LoadPlugin cpu - - ReportByCpu true - ReportByState true - ValuesPercentage true - - - LoadPlugin network - - Server "${config.makefu.stats-server}" "25826" - - ''; - }; -} diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix deleted file mode 100644 index 602fcc6d0..000000000 --- a/makefu/2configs/logging/central-stats-server.nix +++ /dev/null @@ -1,84 +0,0 @@ -{pkgs, config, ...}: - -with import ; -let - collectd-port = 25826; - influx-port = 8086; - grafana-port = 3000; # TODO nginx forward - db = "collectd_db"; - logging-interface = config.makefu.server.primary-itf; -in { - services.grafana.enable = true; - services.grafana.addr = "0.0.0.0"; - - services.influxdb.enable = true; - # redirect grafana to stats.makefu.r - services.nginx.enable = true; - services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000"; - # forward these via nginx - services.influxdb.extraConfig = { - meta.hostname = config.krebs.build.host.name; - # meta.logging-enabled = true; - http.bind-address = ":${toString influx-port}"; - admin.bind-address = ":8083"; - monitoring = { - enabled = false; - # write-interval = "24h"; - }; - collectd = [{ - enabled = true; - typesdb = "${pkgs.collectd}/share/collectd/types.db"; - database = db; - port = collectd-port; - }]; - }; - krebs.kapacitor = - let - echoToIrc = pkgs.writeDash "echo_irc" '' - set -euf - data="$(${pkgs.jq}/bin/jq -r .message)" - export LOGNAME=malarm - ${pkgs.irc-announce}/bin/irc-announce \ - irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null - ''; - in { - enable = true; - alarms = { - cpu_deadman.database = db; - cpu_deadman.text = '' - var data = batch - |query(${"'''"} - SELECT mean("value") AS mean - FROM "collectd_db"."default"."cpu_value" - WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0) - ${"'''"}) - .period(10m) - .every(1m) - .groupBy('host') - data |alert() - .crit(lambda: "mean" < 50) - .stateChangesOnly() - .exec('${echoToIrc}') - data |deadman(1.0,5m) - .stateChangesOnly() - .exec('${echoToIrc}') - ''; - }; - - }; - networking.firewall.extraCommands = '' - iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT - iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT - iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT - iptables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT - iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT - iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT - - ip6tables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT - ip6tables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT - ip6tables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT - ip6tables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT - ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT - ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT - ''; -} diff --git a/makefu/2configs/logging/client.nix b/makefu/2configs/logging/client.nix new file mode 100644 index 000000000..04d2de0d0 --- /dev/null +++ b/makefu/2configs/logging/client.nix @@ -0,0 +1,32 @@ +{pkgs, buil, config, ...}: +let + log-server = config.makefu.log-server; + log-port = 9200; +in { + services.journalbeat = { + enable = true; + # TODO: filter for certain journal fields, not all + extraConfig = '' + journalbeat: + name: logs-${config.krebs.build.host.name} + seek_position: cursor + cursor_seek_fallback: tail + write_cursor_state: true + cursor_flush_period: 5s + clean_field_names: true + convert_to_numbers: false + move_metadata_to_field: journal + default_type: journal + output.elasticsearch: + enabled: true + hosts: ["${log-server}:${builtins.toString log-port}"] + template.enabled: false + #output.console: + # enabled: true + logging.level: info + logging.to_syslog: true + logging.selectors: ["*"] + + ''; + }; +} diff --git a/makefu/2configs/logging/external/aralast.nix b/makefu/2configs/logging/external/aralast.nix deleted file mode 100644 index c335db454..000000000 --- a/makefu/2configs/logging/external/aralast.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ config, lib, pkgs, ... }: - -with import ; -let - pkg = pkgs.stdenv.mkDerivation { - name = "aralast-master"; - src = pkgs.fetchFromGitHub { - owner = "makefu"; - repo = "aralast"; - rev = "7121598"; - sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m"; - }; - installPhase = '' - install -m755 -D aralast.sh $out/bin/aralast - ''; - }; -in { - systemd.services.aralast = { - description = "periodically fetch aramark"; - path = [ - pkgs.curl - pkgs.gnugrep - pkgs.gnused - ]; - wantedBy = [ "multi-user.target" ]; - environment = { - INFLUX_HOST = "localhost"; - INFLUX_PORT = "8086"; - }; - # every 10 seconds when the cantina is open - startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0/10"; - serviceConfig = { - User = "nobody"; - ExecStart = "${pkg}/bin/aralast"; - PrivateTmp = true; - }; - }; -} diff --git a/makefu/2configs/logging/server.nix b/makefu/2configs/logging/server.nix new file mode 100644 index 000000000..90f8e6680 --- /dev/null +++ b/makefu/2configs/logging/server.nix @@ -0,0 +1,23 @@ +{pkgs, config, ...}: + +with import ; +let + es-port = 9200; + kibana-port = 5601; +in { + services.elasticsearch = { + enable = true; + listenAddress = "0.0.0.0"; + port = es-port; + }; + services.kibana = { + enable = true; + listenAddress = "0.0.0.0"; + port = kibana-port; + }; + + networking.firewall.extraCommands = '' + iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT + iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT + ''; +} -- cgit v1.2.3