From 702cbc1307f55acef930c0413fd3362f1a71a6f3 Mon Sep 17 00:00:00 2001 From: EUcancER Date: Wed, 26 Oct 2011 12:40:22 +0200 Subject: //retiolum/adv_graphgen: rewrite graph generation --- retiolum/scripts/adv_graphgen/parse.py | 41 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py index aae69f0f..ed1ef5b6 100755 --- a/retiolum/scripts/adv_graphgen/parse.py +++ b/retiolum/scripts/adv_graphgen/parse.py @@ -1,23 +1,42 @@ #!/usr/bin/python # -*- coding: utf8 -*- -import sys +import sys,json +supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" - def write_digraph(nodes): """ writes the complete digraph in dot format """ print ('digraph retiolum {') + #print (' graph [center rankdir=LR packMode="clust"]') + print (' graph [center packMode="clust"]') print (' node[shape=box,style=filled,fillcolor=grey]') print (' overlap=false') generate_stats(nodes) nodes = delete_unused_nodes(nodes) merge_edges(nodes) + write_stat_node(nodes) for k,v in nodes.iteritems(): write_node(k,v) print ('}') +def write_stat_node(nodes): + ''' Write a `stats` node in the corner + This node contains infos about the current number of active nodes and connections inside the network + ''' + num_conns = 0 + num_nodes = len(nodes) + for k,v in nodes.iteritems(): + num_conns+= len(v['to']) + node_text = " stats_node [label=\"Statistics\\l" + node_text += "Active Nodes: %s\\l" % num_nodes + node_text += "Connections : %s\\l" % num_conns + node_text += "\"" + node_text += ",fillcolor=green" + node_text += "]" + print(node_text) + def generate_stats(nodes): """ Generates some statistics of the network and nodes """ @@ -62,14 +81,26 @@ def write_node(k,v): for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']): node += "internal:"+addr+"\\l" node +="\"" - if v['external-ip'] == "MYSELF": + if k in supernodes: node += ",fillcolor=steelblue1" - node +=",group=\""+v['external-ip'].replace(".","")+"\"" + #node +=",group=\""+v['external-ip'].replace(".","")+"\"" node += "]" print node for con in v.get('to',[]): - edge = " "+k+ " -> " +con['name'] + "[weight="+str(float(con['weight'])) + label = con['weight'] + w = int(con['weight']) + weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) + + length = str(float(w)/1500) + #weight = "1000" #str(300/float(con['weight'])) + #weight = str((100/float(con['weight']))) + #weight = str(-1 * (200-100000/int(con['weight']))) + if float(weight) < 0 : + weight= "1" + + #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n") + edge = " "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length if con.get('bidirectional',False): edge += ",dir=both" edge += "]" -- cgit v1.2.3 From ad5d0233ce202aeef725f9ede6c164abfa30c6c4 Mon Sep 17 00:00:00 2001 From: EUcancER Date: Wed, 26 Oct 2011 15:21:10 +0200 Subject: //retiolum/adv_graphgen:add availability to parser --- retiolum/scripts/adv_graphgen/parse.py | 48 +++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py index ed1ef5b6..ca45ca15 100755 --- a/retiolum/scripts/adv_graphgen/parse.py +++ b/retiolum/scripts/adv_graphgen/parse.py @@ -5,6 +5,7 @@ import sys,json supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" +DUMP_FILE = "/srv/http/tmp/live/availability" def write_digraph(nodes): """ writes the complete digraph in dot format @@ -15,12 +16,20 @@ def write_digraph(nodes): print (' node[shape=box,style=filled,fillcolor=grey]') print (' overlap=false') generate_stats(nodes) - nodes = delete_unused_nodes(nodes) merge_edges(nodes) write_stat_node(nodes) for k,v in nodes.iteritems(): write_node(k,v) print ('}') +def dump_graph(nodes): + from time import time + graph = {} + graph['nodes'] = nodes + graph['timestamp'] = time() + f = open(DUMP_FILE,'a') + json.dump(graph,f) + f.write('\n') + f.close() def write_stat_node(nodes): ''' Write a `stats` node in the corner This node contains infos about the current number of active nodes and connections inside the network @@ -40,8 +49,39 @@ def write_stat_node(nodes): def generate_stats(nodes): """ Generates some statistics of the network and nodes """ + f = open(DUMP_FILE,'r') + flines = f.readlines() + f.close() for k,v in nodes.iteritems(): v['num_conns'] = len(v.get('to',[])) + v['availability'] = get_node_availability(k,flines) + sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) + +def get_node_availability(name,flines): + """ calculates the node availability by reading the generated dump file + adding together the uptime of the node and returning the time + parms: + name - node name + f - archival dump line array + """ + begin = last = current = 0 + uptime = 0 + #sys.stderr.write ( "Getting Node availability of %s\n" % name) + for line in flines: + stat = json.loads(line) + ts = stat['timestamp'] + if not begin: + begin = last = ts + current = ts + if stat['nodes'].get(name,{}).get('to',[]): + uptime += current - last + else: + pass + #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) + last = ts + all_the_time = last - begin + return uptime/ all_the_time + def delete_unused_nodes(nodes): new_nodes = {} for k,v in nodes.iteritems(): @@ -57,7 +97,6 @@ def delete_unused_nodes(nodes): def merge_edges(nodes): """ merge back and forth edges into one DESTRUCTS the current structure by deleting "connections" in the nodes - """ for k,v in nodes.iteritems(): for con in v.get('to',[]): @@ -75,9 +114,10 @@ def write_node(k,v): node = " "+k+"[label=\"" node += k+"\\l" - node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" + node += "availability: %.2f%%\\l" % (v['availability'] * 100) if v.has_key('num_conns'): node += "Num Connects:"+str(v['num_conns'])+"\\l" + node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']): node += "internal:"+addr+"\\l" node +="\"" @@ -138,4 +178,6 @@ def parse_input(): {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) return nodes nodes = parse_input() +nodes = delete_unused_nodes(nodes) +dump_graph(nodes) write_digraph(nodes) -- cgit v1.2.3 From 88d66c8db7fcb0fa7ecb7924439ededb883b8a66 Mon Sep 17 00:00:00 2001 From: EUcancER Date: Wed, 26 Oct 2011 15:31:15 +0200 Subject: //retiolum/adv_graphgen: update archive path --- retiolum/scripts/adv_graphgen/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py index ca45ca15..715a228a 100755 --- a/retiolum/scripts/adv_graphgen/parse.py +++ b/retiolum/scripts/adv_graphgen/parse.py @@ -5,7 +5,7 @@ import sys,json supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" -DUMP_FILE = "/srv/http/tmp/live/availability" +DUMP_FILE = "/krebs/db/availability" def write_digraph(nodes): """ writes the complete digraph in dot format -- cgit v1.2.3 From 208c4e50751d3fef90b52628302548299674774e Mon Sep 17 00:00:00 2001 From: EUcancER Date: Wed, 26 Oct 2011 21:01:31 +0200 Subject: //retiolum/adv_graphgen: archive file parsed once, not n times the archive file used to calculate the node availability is now parsed only once, not [num nodes] times. this saves quite some time --- retiolum/scripts/adv_graphgen/parse.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py index 715a228a..0e878bd8 100755 --- a/retiolum/scripts/adv_graphgen/parse.py +++ b/retiolum/scripts/adv_graphgen/parse.py @@ -50,25 +50,26 @@ def generate_stats(nodes): """ Generates some statistics of the network and nodes """ f = open(DUMP_FILE,'r') - flines = f.readlines() + jlines = [] + for line in f: + jlines.append(json.loads(line)) f.close() for k,v in nodes.iteritems(): v['num_conns'] = len(v.get('to',[])) - v['availability'] = get_node_availability(k,flines) + v['availability'] = get_node_availability(k,jlines) sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) -def get_node_availability(name,flines): +def get_node_availability(name,jlines): """ calculates the node availability by reading the generated dump file adding together the uptime of the node and returning the time parms: name - node name - f - archival dump line array + jlines - list of already parsed dictionaries node archive """ begin = last = current = 0 uptime = 0 #sys.stderr.write ( "Getting Node availability of %s\n" % name) - for line in flines: - stat = json.loads(line) + for stat in jlines: ts = stat['timestamp'] if not begin: begin = last = ts @@ -114,7 +115,7 @@ def write_node(k,v): node = " "+k+"[label=\"" node += k+"\\l" - node += "availability: %.2f%%\\l" % (v['availability'] * 100) + node += "availability: %f\\l" % v['availability'] if v.has_key('num_conns'): node += "Num Connects:"+str(v['num_conns'])+"\\l" node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" -- cgit v1.2.3 From cf61e00b669842700f7031f1f46add3eecb936f9 Mon Sep 17 00:00:00 2001 From: EUcancER Date: Thu, 27 Oct 2011 15:33:25 +0200 Subject: //retiolum/adv_graphgen: add avg weight to nodes --- retiolum/scripts/adv_graphgen/parse.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py index 0e878bd8..e99a5573 100755 --- a/retiolum/scripts/adv_graphgen/parse.py +++ b/retiolum/scripts/adv_graphgen/parse.py @@ -55,10 +55,18 @@ def generate_stats(nodes): jlines.append(json.loads(line)) f.close() for k,v in nodes.iteritems(): - v['num_conns'] = len(v.get('to',[])) + conns = v.get('to',[]) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) v['availability'] = get_node_availability(k,jlines) sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) - +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) def get_node_availability(name,jlines): """ calculates the node availability by reading the generated dump file adding together the uptime of the node and returning the time @@ -116,6 +124,7 @@ def write_node(k,v): node = " "+k+"[label=\"" node += k+"\\l" node += "availability: %f\\l" % v['availability'] + node += "avg weight: %.2f\\l" % v['avg_weight'] if v.has_key('num_conns'): node += "Num Connects:"+str(v['num_conns'])+"\\l" node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" -- cgit v1.2.3 From 0ed905da208ff9eed13878485325ea26ae2e2c8f Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 4 Nov 2011 14:21:26 +0100 Subject: //retiolum/adv_graphgen: replace old parsing style, refactoring tinc_stats now writes the current tinc state into stdout as json sanitize.sh uses tinc_stats and pipes the output to parse_tinc_stats.py (the name is a lie) --- retiolum/scripts/adv_graphgen/parse.py | 193 --------------------------------- 1 file changed, 193 deletions(-) delete mode 100755 retiolum/scripts/adv_graphgen/parse.py (limited to 'retiolum/scripts/adv_graphgen/parse.py') diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py deleted file mode 100755 index e99a5573..00000000 --- a/retiolum/scripts/adv_graphgen/parse.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf8 -*- - -import sys,json -supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] -""" TODO: Refactoring needed to pull the edges out of the node structures again, -it should be easier to handle both structures""" -DUMP_FILE = "/krebs/db/availability" -def write_digraph(nodes): - """ - writes the complete digraph in dot format - """ - print ('digraph retiolum {') - #print (' graph [center rankdir=LR packMode="clust"]') - print (' graph [center packMode="clust"]') - print (' node[shape=box,style=filled,fillcolor=grey]') - print (' overlap=false') - generate_stats(nodes) - merge_edges(nodes) - write_stat_node(nodes) - for k,v in nodes.iteritems(): - write_node(k,v) - print ('}') -def dump_graph(nodes): - from time import time - graph = {} - graph['nodes'] = nodes - graph['timestamp'] = time() - f = open(DUMP_FILE,'a') - json.dump(graph,f) - f.write('\n') - f.close() -def write_stat_node(nodes): - ''' Write a `stats` node in the corner - This node contains infos about the current number of active nodes and connections inside the network - ''' - num_conns = 0 - num_nodes = len(nodes) - for k,v in nodes.iteritems(): - num_conns+= len(v['to']) - node_text = " stats_node [label=\"Statistics\\l" - node_text += "Active Nodes: %s\\l" % num_nodes - node_text += "Connections : %s\\l" % num_conns - node_text += "\"" - node_text += ",fillcolor=green" - node_text += "]" - print(node_text) - -def generate_stats(nodes): - """ Generates some statistics of the network and nodes - """ - f = open(DUMP_FILE,'r') - jlines = [] - for line in f: - jlines.append(json.loads(line)) - f.close() - for k,v in nodes.iteritems(): - conns = v.get('to',[]) - v['num_conns'] = len(conns) - v['avg_weight'] = get_node_avg_weight(conns) - v['availability'] = get_node_availability(k,jlines) - sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) -def get_node_availability(name,jlines): - """ calculates the node availability by reading the generated dump file - adding together the uptime of the node and returning the time - parms: - name - node name - jlines - list of already parsed dictionaries node archive - """ - begin = last = current = 0 - uptime = 0 - #sys.stderr.write ( "Getting Node availability of %s\n" % name) - for stat in jlines: - ts = stat['timestamp'] - if not begin: - begin = last = ts - current = ts - if stat['nodes'].get(name,{}).get('to',[]): - uptime += current - last - else: - pass - #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) - last = ts - all_the_time = last - begin - return uptime/ all_the_time - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.iteritems(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True - - -def write_node(k,v): - """ writes a single node and its edges - edges are weightet with the informations inside the nodes provided by - tinc - """ - - node = " "+k+"[label=\"" - node += k+"\\l" - node += "availability: %f\\l" % v['availability'] - node += "avg weight: %.2f\\l" % v['avg_weight'] - if v.has_key('num_conns'): - node += "Num Connects:"+str(v['num_conns'])+"\\l" - node += "external:"+v['external-ip']+":"+v['external-port']+"\\l" - for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']): - node += "internal:"+addr+"\\l" - node +="\"" - if k in supernodes: - node += ",fillcolor=steelblue1" - #node +=",group=\""+v['external-ip'].replace(".","")+"\"" - node += "]" - print node - - for con in v.get('to',[]): - label = con['weight'] - w = int(con['weight']) - weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) - - length = str(float(w)/1500) - #weight = "1000" #str(300/float(con['weight'])) - #weight = str((100/float(con['weight']))) - #weight = str(-1 * (200-100000/int(con['weight']))) - if float(weight) < 0 : - weight= "1" - - #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n") - edge = " "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length - if con.get('bidirectional',False): - edge += ",dir=both" - edge += "]" - print edge - -def parse_input(): - nodes={} - for line in sys.stdin: - line = line.replace('\n','') - if line == 'Nodes:': - nodes={} - for line in sys.stdin: - if line == 'End of nodes.\n': - break - l = line.replace('\n','').split() #TODO unhack me - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - if line == 'Subnet list:': - for line in sys.stdin: - if line == 'End of subnet list.\n': - break - l = line.replace('\n','').split() - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - if line == 'Edges:': - edges = {} - for line in sys.stdin: - if line == 'End of edges.\n': - break - l = line.replace('\n','').split() - - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - return nodes -nodes = parse_input() -nodes = delete_unused_nodes(nodes) -dump_graph(nodes) -write_digraph(nodes) -- cgit v1.2.3