From dabd9f0f02b44b048b6355184fa64612201db72d Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Sat, 28 Apr 2018 09:41:43 +0200
Subject: l monitoring: open ports

---
 lass/2configs/monitoring/prometheus-server.nix | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lass/2configs/monitoring/prometheus-server.nix')

diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
index d56d7e552..c5c97412d 100644
--- a/lass/2configs/monitoring/prometheus-server.nix
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -9,6 +9,12 @@
   #  useDHCP = true;
   #};
 
+  krebs.iptables.tables.filter.INPUT.rules = [
+    { predicate = "-i retiolum -p tcp --dport 3000"; target = "ACCEPT"; }
+    { predicate = "-i retiolum -p tcp --dport 9090"; target = "ACCEPT"; }
+    { predicate = "-i retiolum -p tcp --dport 9093"; target = "ACCEPT"; }
+  ];
+
   services = {
     prometheus = {
       enable = true;
-- 
cgit v1.2.3


From a0862fa505ba8fb1d94c8bdac69a2077ba89bcdc Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Sat, 28 Apr 2018 09:43:17 +0200
Subject: l monitoring: monitor more hosts

---
 lass/2configs/monitoring/prometheus-server.nix | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'lass/2configs/monitoring/prometheus-server.nix')

diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
index c5c97412d..92bb0519f 100644
--- a/lass/2configs/monitoring/prometheus-server.nix
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -130,11 +130,10 @@
           static_configs = [
             {
               targets = [
-                "localhost:9100"
-              ];
-              labels = {
-                alias = "prometheus.example.com";
-              };
+              ] ++ map (host: "${host}:9100") (lib.attrNames (lib.filterAttrs (_: host: host.owner.name == "lass" && host.monitoring) config.krebs.hosts));
+              #labels = {
+              #  alias = "prometheus.example.com";
+              #};
             }
           ];
         }
-- 
cgit v1.2.3


From da44ae1115af80bb71f38de20b7421d08e435ea7 Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Sat, 28 Apr 2018 09:43:51 +0200
Subject: l monitoring: print alarms to irc

---
 lass/2configs/monitoring/prometheus-server.nix | 33 ++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

(limited to 'lass/2configs/monitoring/prometheus-server.nix')

diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
index 92bb0519f..1f9419e1a 100644
--- a/lass/2configs/monitoring/prometheus-server.nix
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -181,4 +181,37 @@
       security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
     };
   };
+  services.logstash = {
+    enable = true;
+    inputConfig = ''
+      http {
+        port => 14813
+        host => "127.0.0.1"
+      }
+    '';
+    filterConfig = ''
+      if ([alerts]) {
+        ruby {
+          code => '
+            lines = []
+            event["alerts"].each {|p|
+              lines << "#{p["labels"]["instance"]}#{p["annotations"]["summary"]} #{p["status"]}"
+            }
+            event["output"] = lines.join("\n")
+          '
+        }
+      }
+    '';
+    outputConfig = ''
+      file { path => "/tmp/logs.json" codec => "json_lines" }
+      irc {
+        channels => [ "#noise" ]
+        host => "irc.r"
+        nick => "alarm"
+        codec => "json_lines"
+        format => "%{output}"
+      }
+    '';
+    #plugins = [ ];
+  };
 }
-- 
cgit v1.2.3


From 28d6704a0d617ca7d379b836ab9fdd4d6a0be868 Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Thu, 3 May 2018 18:04:38 +0200
Subject: l monitoring: use correct logstash url

---
 lass/2configs/monitoring/prometheus-server.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lass/2configs/monitoring/prometheus-server.nix')

diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
index 1f9419e1a..e16d421a0 100644
--- a/lass/2configs/monitoring/prometheus-server.nix
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -164,7 +164,7 @@
               ];
               "webhook_configs" = [
                 {
-                  "url" = "https://example.com/prometheus-alerts";
+                  "url" = "http://127.0.0.1:14813/prometheus-alerts";
                   "send_resolved" = true;
                 }
               ];
-- 
cgit v1.2.3