summaryrefslogtreecommitdiffstats
path: root/retiolum/scripts/adv_graphgen
diff options
context:
space:
mode:
Diffstat (limited to 'retiolum/scripts/adv_graphgen')
-rwxr-xr-xretiolum/scripts/adv_graphgen/all_the_graphs.sh9
-rwxr-xr-xretiolum/scripts/adv_graphgen/anonytize.sh2
-rw-r--r--retiolum/scripts/adv_graphgen/find_super.py50
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_anon.py4
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_stats.py18
-rwxr-xr-xretiolum/scripts/adv_graphgen/sanitize.sh2
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats2json (renamed from retiolum/scripts/adv_graphgen/tinc_stats.py)59
7 files changed, 122 insertions, 22 deletions
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 5533c722..d3ce8f86 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -4,11 +4,14 @@
echo "`date` begin all graphs" >> /tmp/build_graph
cd $(dirname $(readlink -f $0))
PATH=$PATH:../../../util/bin/
- export LOG_FILE=/var/log/retiolum.log
+ export LOG_FILE=/var/log/syslog
+ export TINC_LEGACY=true
+ EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
+ INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
begin=`timer`
export GRAPHITE_HOST="no_omo"
- (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
- (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
+ (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
+ (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
# wait
graphitec "retiolum.graph.buildtime" "$(timer $begin)" >> /tmp/build_graph
echo "`date` end all graphs" >> /tmp/build_graph
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index d49793cb..b31f4dbb 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -11,7 +11,7 @@ TYPE2=png
OPENER=/bin/true
DOTFILE=`mktemp`
trap 'rm $DOTFILE' INT TERM
-sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\
+sudo -E python tinc_stats2json |\
python parse_tinc_anon.py> $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py
new file mode 100644
index 00000000..df01734e
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/find_super.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+def find_super(path="/etc/tinc/retiolum/hosts"):
+ import os
+ import re
+
+ needle_addr = re.compile("Address\s*=\s*(.*)")
+ needle_port = re.compile("Port\s*=\s*(.*)")
+ for f in os.listdir(path):
+ with open(path+"/"+f) as of:
+ addrs = []
+ port = "655"
+
+ for line in of.readlines():
+
+ addr_found = needle_addr.match(line)
+ if addr_found:
+ addrs.append(addr_found.group(1))
+
+ port_found = needle_port.match(line)
+ if port_found:
+ port = port_found.group(1)
+
+ if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
+
+def check_super(path="/etc/tinc/retiolum/hosts"):
+ from socket import socket,AF_INET,SOCK_STREAM
+ for host,addrs in find_super(path):
+ valid_addrs = []
+ for addr in addrs:
+ try:
+ s = socket(AF_INET,SOCK_STREAM)
+ s.settimeout(3)
+ s.connect(addr)
+ #print("success connecting %s:%d"%(addr))
+ s.settimeout(None)
+ s.close()
+ valid_addrs.append(addr)
+ except Exception as e:
+ pass
+ #print("cannot connect to %s:%d"%(addr))
+ if valid_addrs: yield (host,valid_addrs)
+
+
+if __name__ == "__main__":
+ """
+ usage
+ """
+ for host,addrs in check_super():
+ print host,addrs
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
index e0bea913..21c36e0f 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
@@ -15,7 +15,7 @@ try:
sys.stderr.write("connecting to %s:%d"%(host,port))
s.connect((host,port))
except Exception as e:
- print >>sys.stderr, "Cannot connect to graphite: " + str(e)
+ sys.stderr.write( "Cannot connect to graphite: " + str(e))
""" TODO: Refactoring needed to pull the edges out of the node structures again,
it should be easier to handle both structures"""
DUMP_FILE = "/krebs/db/availability"
@@ -151,4 +151,4 @@ try:
msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end)
s.send(msg)
s.close()
-except Exception as e: print >>sys.stderr, e
+except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index 16f4f795..76a3ffcd 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -2,6 +2,7 @@
# -*- coding: utf8 -*-
from BackwardsReader import BackwardsReader
import sys,json
+from find_super import check_super
try:
from time import time
import socket
@@ -16,10 +17,13 @@ try:
except Exception as e:
sys.stderr.write("Cannot connect to graphite: %s\n" % str(e))
-supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
+supernodes= [ ]
+for supernode,addr in check_super():
+ supernodes.append(supernode)
""" TODO: Refactoring needed to pull the edges out of the node structures again,
it should be easier to handle both structures"""
DUMP_FILE = "/krebs/db/availability"
+
def write_digraph(nodes):
"""
writes the complete digraph in dot format
@@ -53,8 +57,7 @@ def write_stat_node(nodes):
try:
msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin)
s.send(msg)
- #print >>sys.stderr, msg
- except Exception as e: print sys.stderr,e
+ except Exception as e: pass
#except: pass
for k,v in nodes.iteritems():
num_conns+= len(v['to'])
@@ -82,8 +85,7 @@ def generate_stats(nodes):
jlines.append(jline)
lines_to_use -=1
- except Exception,e:
- sys.stderr.write(str(e))
+ except Exception,e: sys.stderr.write(str(e))
for k,v in nodes.iteritems():
conns = v.get('to',[])
for c in conns: #sanitize weights
@@ -169,7 +171,11 @@ def write_node(k,v):
for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
node += "internal:"+addr+"\\l"
node +="\""
- if k in supernodes:
+
+ # warning if node only has one connection
+ if v['num_conns'] == 1:
+ node += ",fillcolor=red"
+ elif k in supernodes:
node += ",fillcolor=steelblue1"
#node +=",group=\""+v['external-ip'].replace(".","")+"\""
node += "]"
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index c46662f3..45d29a22 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -11,7 +11,7 @@ TYPE2=png
OPENER=/bin/true
DOTFILE=`mktemp`
trap 'rm $DOTFILE' INT TERM
-sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\
+sudo -E python tinc_stats2json |\
python parse_tinc_stats.py > $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats2json
index d0d47aff..ede19b26 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats2json
@@ -1,13 +1,17 @@
#!/usr/bin/python
-from BackwardsReader import BackwardsReader
+import subprocess
import os
import re
import sys
import json
-TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum")
-os.environ["LOG_FILE"]
+
+TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
+
+# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog),
+# or if the new and hip tincctl should be used
+is_legacy= os.environ.get("TINC_LEGACY",False)
SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
@@ -21,11 +25,14 @@ BEGIN_EDGES = "Edges:"
END_EDGES = "End of edges."
def get_tinc_block(log_file):
- """ returns an iterateable block from the given log file (syslog) """
+ """ returns an iterateable block from the given log file (syslog)
+ This function became obsolete with the introduction of tincctl
+ """
+ from BackwardsReader import BackwardsReader
tinc_block = []
in_block = False
bf = BackwardsReader(log_file)
- BOL = re.compile(".*tinc.retiolum\[[0-9]+\]: ")
+ BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
while True:
line = bf.readline()
if not line:
@@ -44,6 +51,37 @@ def get_tinc_block(log_file):
break
return reversed(tinc_block)
+def parse_new_input():
+ nodes = {}
+ pnodes = subprocess.Popen(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0]
+ #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
+ for line in pnodes.split('\n'):
+ if not line: continue
+ l = line.split()
+ nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+ psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"])
+ for line in psubnets.split('\n'):
+ if not line: continue
+ l = line.split()
+ try:
+ if not nodes[l[2]].get('internal-ip',False):
+ nodes[l[2]]['internal-ip'] = []
+ nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+ except KeyError:
+ pass # node does not exist (presumably)
+ pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"])
+ for line in pedges.split('\n'):
+ if not line: continue
+ l = line.split()
+ try:
+ if not nodes[l[0]].has_key('to') :
+ nodes[l[0]]['to'] = []
+ nodes[l[0]]['to'].append(
+ {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+ except KeyError:
+ pass #node does not exist
+ return nodes
+
def parse_input(log_data):
nodes={}
for line in log_data:
@@ -68,7 +106,6 @@ def parse_input(log_data):
if END_EDGES in line :
break
l = line.replace('\n','').split()
-
if not nodes[l[0]].has_key('to') :
nodes[l[0]]['to'] = []
nodes[l[0]]['to'].append(
@@ -78,6 +115,10 @@ def parse_input(log_data):
if __name__ == '__main__':
import subprocess,time
- subprocess.call(["pkill","-SIGUSR2", "tincd"])
- time.sleep(1)
- print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
+ if is_legacy:
+ subprocess.call(["pkill","-SIGUSR2", "tincd"])
+ time.sleep(1)
+ print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
+ else:
+ print json.dumps(parse_new_input())
+
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-23 13:14:39 +0000