diff options
Diffstat (limited to 'retiolum/scripts/adv_graphgen/tinc_stats')
8 files changed, 634 insertions, 0 deletions
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py new file mode 100755 index 00000000..a1ef13f1 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py @@ -0,0 +1,58 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- + +import sys,json +""" TODO: Refactoring needed to pull the edges out of the node structures again, +it should be easier to handle both structures""" +DUMP_FILE = "/krebs/db/availability" + +def get_all_nodes(): + import os + return os.listdir("/etc/tinc/retiolum/hosts") + +def generate_stats(): + """ Generates availability statistics of the network and nodes + """ + import json + jlines = [] + try: + f = open(DUMP_FILE,'r') + for line in f: + jlines.append(json.loads(line)) + f.close() + except Exception,e: + pass + all_nodes = {} + for k in get_all_nodes(): + all_nodes[k] = get_node_availability(k,jlines) + print ( json.dumps(all_nodes)) + +def get_node_availability(name,jlines): + """ calculates the node availability by reading the generated dump file + adding together the uptime of the node and returning the time + parms: + name - node name + jlines - list of already parsed dictionaries node archive + """ + begin = last = current = 0 + uptime = 0 + for stat in jlines: + if not stat['nodes']: + continue + ts = stat['timestamp'] + if not begin: + begin = last = ts + current = ts + if stat['nodes'].get(name,{}).get('to',[]): + uptime += current - last + else: + pass + last = ts + all_the_time = last - begin + try: + return uptime/ all_the_time + except: + return 1 + +if __name__ == "__main__": + generate_stats() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py new file mode 100644 index 00000000..6bdbf43c --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py @@ -0,0 +1,35 @@ +import sys +import os +import string + +class BackwardsReader: + """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """ + def readline(self): + while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): + self.blkcount = self.blkcount + 1 + line = self.data[0] + try: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize) + line, '\n') + except IOError: + self.f.seek(0) + self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n') + + if len(self.data) == 0: + return "" + + line = self.data[-1] + self.data = self.data[:-1] + return line + '\n' + + def __init__(self, file, blksize=4096): + """initialize the internal structures""" + self.size = os.stat(file)[6] + self.blksize = blksize + self.blkcount = 1 + self.f = open(file, 'rb') + if self.size > self.blksize: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize), '\n') + if not self.data[-1]: + self.data = self.data[:-1] diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py new file mode 100755 index 00000000..038ca9c0 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py @@ -0,0 +1,54 @@ +#!/usr/bin/python3 +# -*- coding: utf8 -*- +import sys,json,os +from Graph import delete_unused_nodes,resolve_myself +GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat") + +def add_geo(nodes): + from pygeoip import GeoIP + gi = GeoIP(GEODB) + + for k,v in nodes.iteritems(): + try: + nodes[k].update(gi.record_by_addr(v["external-ip"])) + except Exception as e: + sys.stderr.write(str(e)) + sys.stderr.write("Cannot determine GeoData for %s\n"%k) + + return nodes +def add_coords_to_edges(nodes): + from pygeoip import GeoIP + gi = GeoIP(GEODB) + + for k,v in nodes.iteritems(): + for i,j in enumerate(v.get("to",[])): + data=gi.record_by_addr(j["addr"]) + try: + j["latitude"]=data["latitude"] + j["longitude"]=data["longitude"] + except Exception as e: pass + + return nodes + +def add_jitter(nodes): + from random import random + #add a bit of jitter to all of the coordinates + max_jitter=0.005 + for k,v in nodes.iteritems(): + jitter_lat= max_jitter -random()*max_jitter*2 + jitter_long= max_jitter -random()*max_jitter*2 + try: + v["latitude"]= v["latitude"] + jitter_lat + v["longitude"]= v["longitude"] + jitter_long + for nodek,node in nodes.iteritems(): + for to in node['to']: + if to['name'] == k: + to['latitude'] = v["latitude"] + to['longitude'] = v["longitude"] + except Exception as e: pass + return nodes + +if __name__ == "__main__": + import json + nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) + print (json.dumps(nodes)) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py new file mode 100644 index 00000000..18c3d545 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py @@ -0,0 +1,231 @@ +#!/usr/bin/python +from BackwardsReader import BackwardsReader +from Graphite import GraphiteSender +import sys,json,os +from Supernodes import check_all_the_super +from Availability import get_node_availability +import sys,json +from time import time +DUMP_FILE = "/krebs/db/availability" + + +def resolve_myself(nodes): + #resolve MYSELF to the real ip + for k,v in nodes.iteritems(): + if v["external-ip"] == "MYSELF": + for nodek,node in nodes.iteritems(): + for to in node['to']: + if to['name'] == k: + v["external-ip"] = to["addr"] + return nodes + +def dump_graph(nodes): + from time import time + graph = {} + graph['nodes'] = nodes + graph['timestamp'] = time() + f = open(DUMP_FILE,'a') + json.dump(graph,f) + f.write('\n') + f.close() + +def generate_availability_stats(nodes): + """ generates stats of from availability + """ + jlines = [] + try: + f = BackwardsReader(DUMP_FILE) + lines_to_use = 1000 + while True: + if lines_to_use == 0: break + line = f.readline() + if not line: break + jline = json.loads(line) + if not jline['nodes']: continue + + jlines.append(jline) + lines_to_use -=1 + except Exception,e: sys.stderr.write(str(e)) + + for k,v in nodes.iteritems(): + v['availability'] = get_node_availability(k,jlines) + sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) + +def generate_stats(nodes): + """ Generates some statistics of the network and nodes + """ + for k,v in nodes.iteritems(): + conns = v.get('to',[]) + for c in conns: #sanitize weights + if float(c['weight']) > 9000: c['weight'] = str(9001) + elif float(c['weight']) < 0: c['weight'] = str(0) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) + +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) + +def delete_unused_nodes(nodes): + """ Deletes all the nodes which are currently not connected to the network""" + new_nodes = {} + for k,v in nodes.iteritems(): + if v['external-ip'] == "(null)": + continue + if v.get('to',[]): + new_nodes[k] = v + for k,v in new_nodes.iteritems(): + if not [ i for i in v['to'] if i['name'] in new_nodes]: + del(k) + return new_nodes + +def merge_edges(nodes): + """ merge back and forth edges into one + DESTRUCTS the current structure by deleting "connections" in the nodes + """ + for k,v in nodes.iteritems(): + for con in v.get('to',[]): + for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): + if k == secon['name']: + del (nodes[con['name']]['to'][i]) + con['bidirectional'] = True + + +def print_head(): + print ('digraph retiolum {') + print (' graph [center packMode="clust"]') + print (' node[shape=box,style=filled,fillcolor=grey]') + print (' overlap=false') + +def print_stat_node(nodes): + ''' Write a `stats` node in the corner + This node contains infos about the current number of active nodes and connections inside the network + ''' + from time import localtime,strftime + num_conns = 0 + num_nodes = len(nodes) + try: + msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin) + s.send(msg) + except Exception as e: pass + #except: pass + for k,v in nodes.iteritems(): + num_conns+= len(v['to']) + node_text = " stats_node [label=\"Statistics\\l" + node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime()) + node_text += "Active Nodes: %s\\l" % num_nodes + node_text += "Connections : %s\\l" % num_conns + node_text += "\"" + node_text += ",fillcolor=green" + node_text += "]" + print(node_text) + +def print_node(k,v): + """ writes a single node and its edges + edges are weightet with the informations inside the nodes provided by + tinc + """ + + node = " "+k+"[label=\"" + node += k+"\\l" + node += "availability: %f\\l" % v['availability'] + if v.has_key('num_conns'): + node += "Num Connects:"+str(v['num_conns'])+"\\l" + node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" + for addr in v.get('internal-ip',['dunno lol']): + node += "internal:%s\\l"%addr + node +="\"" + + if v['num_conns'] == 1: + node += ",fillcolor=red" + elif k in supernodes: + node += ",fillcolor=steelblue1" + node += "]" + print node + +def print_anonymous_node(k,v): + """ writes a single node and its edges + edges are weightet with the informations inside the nodes provided by + tinc + """ + + node = " "+k #+"[label=\"" + print node + +def print_edge(k,v): + for con in v.get('to',[]): + label = con['weight'] + w = int(con['weight']) + weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) + + length = str(float(w)/1500) + if float(weight) < 0 : + weight= "1" + + edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight + if con.get('bidirectional',False): + edge += ",dir=both" + edge += "]" + print edge + +def anonymize_nodes(nodes): + #anonymizes all nodes + i = "0" + newnodes = {} + for k,v in nodes.iteritems(): + for nodek,node in nodes.iteritems(): + for to in node['to']: + if to['name'] == k: + to['name'] = i + newnodes[i] = v + i = str(int(i)+1) + return newnodes + +if __name__ == "__main__": + supernodes= [] + try: + gr = GraphiteSender(os.environ.get("GRAPHITE_HOST","localhost")) + begin = time() + except Exception as e: + sys.stderr.write( "Cannot connect to graphite: " + str(e)) + if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]: + print("usage: %s (anonymous|complete)") + sys.exit(1) + + nodes = json.load(sys.stdin) + nodes = delete_unused_nodes(nodes) + print_head() + generate_stats(nodes) + merge_edges(nodes) + + + if sys.argv[1] == "anonymous": + nodes = anonymize_nodes(nodes) + + for k,v in nodes.iteritems(): + print_anonymous_node(k,v) + print_edge(k,v) + + elif sys.argv[1] == "complete": + for supernode,addr in check_all_the_super(): + supernodes.append(supernode) + generate_availability_stats(nodes) + for k,v in nodes.iteritems(): + print_node(k,v) + print_edge(k,v) + try: + dump_graph(nodes) + except Exception,e: + sys.stderr.write("Cannot dump graph: %s" % str(e)) + else: + pass + + print_stat_node(nodes) + print ('}') + try: + gr.send("graph.anon_build_time",(time()-begin)*1000) + except Exception as e: pass diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py new file mode 100644 index 00000000..5002d8e5 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py @@ -0,0 +1,24 @@ +#!/usr/bin/python + +import socket +from time import time + +class GraphiteSender: + def __init__(self,host,port=2003,prefix="retiolum"): + self.host = host + self.port = port + self.prefix = prefix + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((host,port)) + + def send(name,data): + # construct a message for graphite, honor the configured prefix + self.sock.send("%s.%s %d %d\r\n"%(self.prefix,name,data,time())) + + def send_raw(path,data): + #ignore the configured prefix, just it to the path given + self.sock.send("%s %d %d\r\n"%(path,data,time())) + +if __name__ == "__main__": + import sys + GraphiteSender(sys.argv[1]).send_raw(sys.argv[2],sys.argv[3]) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py new file mode 100755 index 00000000..644cbc63 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +import subprocess +import os +import re +import sys +import json + + +TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") + +# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), +# or if the new and hip tincctl should be used + + +# Tags and Delimiters +TINC_TAG="tinc.%s" % TINC_NETWORK +BEGIN_NODES = "Nodes:" +END_NODES = "End of nodes." +BEGIN_SUBNET = "Subnet list:" +END_SUBNET = "End of subnet list" +BEGIN_EDGES = "Edges:" +END_EDGES = "End of edges." +def usage(): + from sys import argv,exit + print("""usage: %s +This tool dumps all tinc node informations as json + +ENVIRONMENT VARIABLES: + TINC_NETWORK The tinc network to dump + (default: retiolum) + LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in + (default: /var/log/everything.log) +""" % argv[0]) + exit(1) +def debug(func): + from functools import wraps + @wraps(func) + def with_debug(*args,**kwargs): + print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs)) + return func(*args,**kwargs) + return with_debug + + +def get_tinc_log_file(): + # TODO parse logfile from somewhere + return os.environ.get("LOG_FILE","/var/log/everything.log") + + +def parse_tinc_stats(): + import subprocess + from time import sleep + from distutils.spawn import find_executable as which + #newest tinc + if which("tinc"): + return parse_new_input("tinc") + #new tinc + elif which("tincctl"): + return parse_new_input("tincctl") + #old tinc + elif which("tincd"): + # TODO refactor me + subprocess.call(["pkill","-SIGUSR2", "tincd"]) + sleep(1) + return parse_input(get_tinc_block(get_tinc_log_file())) + #no tinc + else: + raise Exception("no tinc executable found!") + +#@debug +def get_tinc_block(log_file): + """ returns an iterateable block from the given log file (syslog) + This function became obsolete with the introduction of tincctl + """ + from BackwardsReader import BackwardsReader + tinc_block = [] + in_block = False + bf = BackwardsReader(log_file) + BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) + while True: + line = bf.readline() + if not line: + raise Exception("end of file at log file? This should not happen!") + line = BOL.sub('',line).strip() + + if END_SUBNET in line: + in_block = True + + if not in_block: + continue + tinc_block.append(line) + + if BEGIN_NODES in line: + break + return reversed(tinc_block) + +def parse_new_input(tinc_bin): + nodes = {} + pnodes = subprocess.Popen([tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0] + #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) + for line in pnodes.split('\n'): + if not line: continue + l = line.split() + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + psubnets = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]) + for line in psubnets.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + except KeyError: + pass # node does not exist (presumably) + pedges = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","edges"]) + for line in pedges.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + except KeyError: + pass #node does not exist + return nodes + +#@debug +def parse_input(log_data): + nodes={} + for line in log_data: + if BEGIN_NODES in line : + nodes={} + for line in log_data: + if END_NODES in line : + break + l = line.replace('\n','').split() #TODO unhack me + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + if BEGIN_SUBNET in line : + for line in log_data: + if END_SUBNET in line : + break + l = line.replace('\n','').split() + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + if BEGIN_EDGES in line : + edges = {} + for line in log_data: + if END_EDGES in line : + break + l = line.replace('\n','').split() + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + return nodes + + +if __name__ == '__main__': + # TODO refactor me + from sys import argv + if len(argv) > 1: + usage() + else: + print json.dumps(parse_tinc_stats()) + diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py new file mode 100644 index 00000000..ae0fae8f --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py @@ -0,0 +1,59 @@ +#!/usr/bin/python + +def find_potential_super(path="/etc/tinc/retiolum/hosts"): + import os + import re + + needle_addr = re.compile("Address\s*=\s*(.*)") + needle_port = re.compile("Port\s*=\s*(.*)") + for f in os.listdir(path): + with open(path+"/"+f) as of: + addrs = [] + port = "655" + + for line in of.readlines(): + + addr_found = needle_addr.match(line) + if addr_found: + addrs.append(addr_found.group(1)) + + port_found = needle_port.match(line) + if port_found: + port = port_found.group(1) + + if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + +def try_connect(addr): + try: + from socket import socket,AF_INET,SOCK_STREAM + s = socket(AF_INET,SOCK_STREAM) + s.settimeout(2) + s.connect(addr) + s.settimeout(None) + s.close() + return addr + except Exception as e: + pass + #return () + +def check_one_super(ha): + host,addrs = ha + valid_addrs = [] + for addr in addrs: + ret = try_connect(addr) + if ret: valid_addrs.append(ret) + if valid_addrs: return (host,valid_addrs) + +def check_all_the_super(path="/etc/tinc/retiolum/hosts"): + from multiprocessing import Pool + p = Pool(20) + return filter(None,p.map(check_one_super,find_potential_super(path))) + + + +if __name__ == "__main__": + """ + usage + """ + for host,addrs in check_all_the_super(): + print host,addrs diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py new file mode 100644 index 00000000..d4a686bf --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py @@ -0,0 +1,7 @@ +import Availability +import BackwardsReader +import Log2JSON +import Supernodes +import Geo +import Graph +import Graphite |