From f6219c0bb717533b74c8362dac96636fff866e79 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 26 Oct 2011 12:36:45 +0200
Subject: //retiolum/adv_graphgen: fix generator script

svg is now generated instead of png
dotfile is now a tempfile
---
 retiolum/scripts/adv_graphgen/sanitize.sh | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 1dc43bf4..3fddb605 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -1,20 +1,25 @@
 #!/bin/sh
 HERE=$(dirname `readlink -f $0`)
-TMP=/tmp
 GRAPH_SETTER1=dot
 GRAPH_SETTER2=circo
 GRAPH_SETTER3='neato -Goverlap=prism '
 GRAPH_SETTER4=sfdp
 LOG_FILE=/var/log/syslog
+TYPE=svg
+TYPE2=png
 OPENER=/bin/true
-
+DOTFILE=`mktemp`
+trap 'rm $DOTFILE' SIGINT SIGTERM
 sudo pkill -USR2 tincd
 sudo sed -n '/tinc.retiolum/{s/.*tinc.retiolum\[[0-9]*\]: //gp}' $LOG_FILE |\
-    $HERE/parse.py > $TMP/retiolum.dot
+    $HERE/parse.py > $DOTFILE
 
-$GRAPH_SETTER1 -Tpng -o $1/retiolum_1.png $TMP/retiolum.dot
-$GRAPH_SETTER2 -Tpng -o $1/retiolum_2.png $TMP/retiolum.dot
-$GRAPH_SETTER3 -Tpng -o $1/retiolum_3.png $TMP/retiolum.dot
-$GRAPH_SETTER4 -Tpng -o $1/retiolum_4.png $TMP/retiolum.dot
-$OPENER $HERE/retiolum_1.png &>/dev/null 
-rm $TMP/retiolum.dot
+$GRAPH_SETTER1 -T$TYPE -o $1/retiolum_1.$TYPE $DOTFILE
+$GRAPH_SETTER2 -T$TYPE -o $1/retiolum_2.$TYPE $DOTFILE
+$GRAPH_SETTER3 -T$TYPE -o $1/retiolum_3.$TYPE $DOTFILE
+$GRAPH_SETTER4 -T$TYPE -o $1/retiolum_4.$TYPE $DOTFILE
+#convert -resize 20% $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
+#convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
+#convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
+#convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
+#$OPENER $1/retiolum_1.$TYPE &>/dev/null 
-- 
cgit v1.2.3


From 702cbc1307f55acef930c0413fd3362f1a71a6f3 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 26 Oct 2011 12:40:22 +0200
Subject: //retiolum/adv_graphgen: rewrite graph generation

---
 retiolum/scripts/adv_graphgen/parse.py | 41 +++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
index aae69f0f..ed1ef5b6 100755
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ b/retiolum/scripts/adv_graphgen/parse.py
@@ -1,23 +1,42 @@
 #!/usr/bin/python
 # -*- coding: utf8 -*-
 
-import sys
+import sys,json
+supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
-
 def write_digraph(nodes):
   """
   writes the complete digraph in dot format
   """
   print ('digraph retiolum {')
+  #print ('  graph [center rankdir=LR packMode="clust"]')
+  print ('  graph [center packMode="clust"]')
   print ('  node[shape=box,style=filled,fillcolor=grey]')
   print ('  overlap=false')
   generate_stats(nodes)
   nodes = delete_unused_nodes(nodes)
   merge_edges(nodes)
+  write_stat_node(nodes)
   for k,v in nodes.iteritems():
     write_node(k,v)
   print ('}')
+def write_stat_node(nodes):
+  ''' Write a `stats` node in the corner
+      This node contains infos about the current number of active nodes and connections inside the network
+  '''
+  num_conns = 0
+  num_nodes = len(nodes)
+  for k,v in nodes.iteritems():
+    num_conns+= len(v['to'])
+  node_text = "  stats_node [label=\"Statistics\\l"
+  node_text += "Active Nodes: %s\\l" % num_nodes
+  node_text += "Connections : %s\\l" % num_conns
+  node_text += "\""
+  node_text += ",fillcolor=green"
+  node_text += "]"
+  print(node_text)
+
 def generate_stats(nodes):
   """ Generates some statistics of the network and nodes
   """
@@ -62,14 +81,26 @@ def write_node(k,v):
   for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
     node += "internal:"+addr+"\\l"
   node +="\""
-  if v['external-ip'] == "MYSELF":
+  if k in supernodes:
     node += ",fillcolor=steelblue1"
-  node +=",group=\""+v['external-ip'].replace(".","")+"\""
+  #node +=",group=\""+v['external-ip'].replace(".","")+"\""
   node += "]"
   print node
 
   for con in v.get('to',[]):
-    edge = "  "+k+ " -> " +con['name'] + "[weight="+str(float(con['weight']))
+    label  = con['weight']
+    w = int(con['weight'])
+    weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
+
+    length = str(float(w)/1500)
+    #weight = "1000" #str(300/float(con['weight']))
+    #weight = str((100/float(con['weight'])))
+    #weight = str(-1 * (200-100000/int(con['weight'])))
+    if float(weight) < 0 :
+      weight= "1"
+
+    #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
+    edge = "  "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length
     if con.get('bidirectional',False):
       edge += ",dir=both"
     edge += "]"
-- 
cgit v1.2.3


From ad5d0233ce202aeef725f9ede6c164abfa30c6c4 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 26 Oct 2011 15:21:10 +0200
Subject: //retiolum/adv_graphgen:add availability to parser

---
 retiolum/scripts/adv_graphgen/parse.py | 48 +++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
index ed1ef5b6..ca45ca15 100755
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ b/retiolum/scripts/adv_graphgen/parse.py
@@ -5,6 +5,7 @@ import sys,json
 supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
+DUMP_FILE = "/srv/http/tmp/live/availability"
 def write_digraph(nodes):
   """
   writes the complete digraph in dot format
@@ -15,12 +16,20 @@ def write_digraph(nodes):
   print ('  node[shape=box,style=filled,fillcolor=grey]')
   print ('  overlap=false')
   generate_stats(nodes)
-  nodes = delete_unused_nodes(nodes)
   merge_edges(nodes)
   write_stat_node(nodes)
   for k,v in nodes.iteritems():
     write_node(k,v)
   print ('}')
+def dump_graph(nodes):
+  from time import time
+  graph = {}
+  graph['nodes'] = nodes
+  graph['timestamp'] = time()
+  f = open(DUMP_FILE,'a')
+  json.dump(graph,f)
+  f.write('\n')
+  f.close()
 def write_stat_node(nodes):
   ''' Write a `stats` node in the corner
       This node contains infos about the current number of active nodes and connections inside the network
@@ -40,8 +49,39 @@ def write_stat_node(nodes):
 def generate_stats(nodes):
   """ Generates some statistics of the network and nodes
   """
+  f = open(DUMP_FILE,'r')
+  flines = f.readlines()
+  f.close()
   for k,v in nodes.iteritems():
     v['num_conns'] = len(v.get('to',[]))
+    v['availability'] = get_node_availability(k,flines)
+    sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
+
+def get_node_availability(name,flines):
+  """ calculates the node availability by reading the generated dump file
+  adding together the uptime of the node and returning the time
+	parms:
+          name - node name
+          f - archival dump line array
+  """
+  begin = last = current = 0
+  uptime = 0
+  #sys.stderr.write ( "Getting Node availability of %s\n" % name)
+  for line in flines:
+    stat = json.loads(line)
+    ts = stat['timestamp']
+    if not begin:
+      begin = last = ts
+    current = ts
+    if stat['nodes'].get(name,{}).get('to',[]):
+      uptime += current - last
+    else:
+      pass
+      #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
+    last = ts
+  all_the_time = last - begin
+  return uptime/ all_the_time
+
 def delete_unused_nodes(nodes):
   new_nodes = {}
   for k,v in nodes.iteritems():
@@ -57,7 +97,6 @@ def delete_unused_nodes(nodes):
 def merge_edges(nodes):
   """ merge back and forth edges into one
   DESTRUCTS the current structure by deleting "connections" in the nodes
-
   """
   for k,v in nodes.iteritems():
     for con in v.get('to',[]):
@@ -75,9 +114,10 @@ def write_node(k,v):
 
   node = "  "+k+"[label=\""
   node += k+"\\l"
-  node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
+  node += "availability: %.2f%%\\l" % (v['availability'] * 100)
   if v.has_key('num_conns'):
     node += "Num Connects:"+str(v['num_conns'])+"\\l"
+  node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
   for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
     node += "internal:"+addr+"\\l"
   node +="\""
@@ -138,4 +178,6 @@ def parse_input():
             {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
   return nodes
 nodes = parse_input()
+nodes = delete_unused_nodes(nodes)
+dump_graph(nodes)
 write_digraph(nodes)
-- 
cgit v1.2.3


From 88d66c8db7fcb0fa7ecb7924439ededb883b8a66 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 26 Oct 2011 15:31:15 +0200
Subject: //retiolum/adv_graphgen: update archive path

---
 retiolum/scripts/adv_graphgen/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
index ca45ca15..715a228a 100755
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ b/retiolum/scripts/adv_graphgen/parse.py
@@ -5,7 +5,7 @@ import sys,json
 supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
-DUMP_FILE = "/srv/http/tmp/live/availability"
+DUMP_FILE = "/krebs/db/availability"
 def write_digraph(nodes):
   """
   writes the complete digraph in dot format
-- 
cgit v1.2.3


From 208c4e50751d3fef90b52628302548299674774e Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 26 Oct 2011 21:01:31 +0200
Subject: //retiolum/adv_graphgen: archive file parsed once, not n times

the archive file used to calculate the node availability is now parsed only once, not [num nodes] times.
this saves quite some time
---
 retiolum/scripts/adv_graphgen/parse.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
index 715a228a..0e878bd8 100755
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ b/retiolum/scripts/adv_graphgen/parse.py
@@ -50,25 +50,26 @@ def generate_stats(nodes):
   """ Generates some statistics of the network and nodes
   """
   f = open(DUMP_FILE,'r')
-  flines = f.readlines()
+  jlines = []
+  for line in f:
+    jlines.append(json.loads(line))
   f.close()
   for k,v in nodes.iteritems():
     v['num_conns'] = len(v.get('to',[]))
-    v['availability'] = get_node_availability(k,flines)
+    v['availability'] = get_node_availability(k,jlines)
     sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
 
-def get_node_availability(name,flines):
+def get_node_availability(name,jlines):
   """ calculates the node availability by reading the generated dump file
   adding together the uptime of the node and returning the time
 	parms:
           name - node name
-          f - archival dump line array
+          jlines - list of already parsed dictionaries node archive
   """
   begin = last = current = 0
   uptime = 0
   #sys.stderr.write ( "Getting Node availability of %s\n" % name)
-  for line in flines:
-    stat = json.loads(line)
+  for stat in jlines:
     ts = stat['timestamp']
     if not begin:
       begin = last = ts
@@ -114,7 +115,7 @@ def write_node(k,v):
 
   node = "  "+k+"[label=\""
   node += k+"\\l"
-  node += "availability: %.2f%%\\l" % (v['availability'] * 100)
+  node += "availability: %f\\l" % v['availability'] 
   if v.has_key('num_conns'):
     node += "Num Connects:"+str(v['num_conns'])+"\\l"
   node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
-- 
cgit v1.2.3


From cf61e00b669842700f7031f1f46add3eecb936f9 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Thu, 27 Oct 2011 15:33:25 +0200
Subject: //retiolum/adv_graphgen: add avg weight to nodes

---
 retiolum/scripts/adv_graphgen/parse.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
index 0e878bd8..e99a5573 100755
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ b/retiolum/scripts/adv_graphgen/parse.py
@@ -55,10 +55,18 @@ def generate_stats(nodes):
     jlines.append(json.loads(line))
   f.close()
   for k,v in nodes.iteritems():
-    v['num_conns'] = len(v.get('to',[]))
+    conns = v.get('to',[])
+    v['num_conns'] = len(conns)
+    v['avg_weight'] = get_node_avg_weight(conns)
     v['availability'] = get_node_availability(k,jlines)
     sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
-
+def get_node_avg_weight(conns):
+  """ calculates the average weight for the given connections """
+  if not conns:
+    sys.syderr.write("get_node_avg_weight: connection parameter empty")
+    return 9001
+  else:
+    return sum([float(c['weight']) for c in conns])/len(conns)
 def get_node_availability(name,jlines):
   """ calculates the node availability by reading the generated dump file
   adding together the uptime of the node and returning the time
@@ -116,6 +124,7 @@ def write_node(k,v):
   node = "  "+k+"[label=\""
   node += k+"\\l"
   node += "availability: %f\\l" % v['availability'] 
+  node += "avg weight: %.2f\\l" % v['avg_weight'] 
   if v.has_key('num_conns'):
     node += "Num Connects:"+str(v['num_conns'])+"\\l"
   node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
-- 
cgit v1.2.3


From 822c43a763aa61c1accce3768090d066048faaff Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 3 Nov 2011 20:14:06 +0100
Subject: //retiolum//tinc_stats: initial commit

tinc_stats writes the current TINC statistics into stdout
---
 retiolum/scripts/adv_graphgen/tinc_stats.py | 77 +++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100755 retiolum/scripts/adv_graphgen/tinc_stats.py

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py
new file mode 100755
index 00000000..0d3ebb11
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+from BackwardsReader import BackwardsReader
+import os
+import re
+
+
+TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum")
+SYSLOG_FILE = "/var/log/everything.log"
+
+
+# Tags and Delimiters
+TINC_TAG="tinc.%s" % TINC_NETWORK
+BEGIN_NODES = "Nodes:"
+END_NODES = "End of nodes."
+BEGIN_SUBNET = "Subnet list:"
+END_SUBNET = "End of subnet list"
+BEGIN_EDGES = "Edges:"
+END_EDGES = "End of edges."
+
+def get_tinc_block(log_file):
+  """ returns an iterateable block from the given log file (syslog) """
+  tinc_block = []
+  in_block = False
+  bf = BackwardsReader(log_file)
+  BOL = re.compile(".*tinc.retiolum\[[0-9]+\]: ")
+  while True:
+    line = bf.readline()
+    if not line:
+      raise Exception("end of file at log file? This should not happen!")
+    line = BOL.sub('',line).strip()
+
+    if END_SUBNET in line:
+      print("Found end of block")
+      in_block = True
+
+    if not in_block:
+      continue
+
+    tinc_block.append(line)
+
+    if BEGIN_NODES in line:
+      print("Found begin of block")
+      break
+  return reversed(tinc_block)
+
+def parse_input(log_data):
+  nodes={}
+  for line in log_data:
+    if BEGIN_NODES in line :
+      nodes={}
+      for line in log_data:
+        if END_NODES in line :
+          break
+        l = line.replace('\n','').split() #TODO unhack me
+        nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+    if BEGIN_SUBNET in line :
+      for line in log_data:
+        if END_SUBNET in line :
+          break
+        l = line.replace('\n','').split() 
+        if not nodes[l[2]].get('internal-ip',False):
+           nodes[l[2]]['internal-ip'] = []
+        nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+    if BEGIN_EDGES in line :
+      edges = {}
+      for line in log_data:
+        if END_EDGES in line :
+          break
+        l = line.replace('\n','').split() 
+
+        if not nodes[l[0]].has_key('to') :
+          nodes[l[0]]['to'] = []
+        nodes[l[0]]['to'].append(
+            {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+  return nodes
+
+print parse_input((get_tinc_block(SYSLOG_FILE)))
-- 
cgit v1.2.3


From fc5b5f2562cbfd9028f6b77269d123261dae89e9 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Fri, 4 Nov 2011 13:41:01 +0100
Subject: //retiolum/adv_graphgen: bug-fixing

debian uses features of adv_graphgen which hasn't been tested until now. Bugs has been fixed concerning usage of LOG_FILE environ
and many more

fix minor calculation bug in availability calucluation (delete faulty archive lines)
---
 retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 10 ++++++----
 retiolum/scripts/adv_graphgen/sanitize.sh         |  7 +++----
 retiolum/scripts/adv_graphgen/tinc_stats.py       |  3 ++-
 3 files changed, 11 insertions(+), 9 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index c02c9a84..410e5229 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -49,14 +49,14 @@ def write_stat_node(nodes):
 def generate_stats(nodes):
   """ Generates some statistics of the network and nodes
   """
+  jlines = []
   try:
     f = open(DUMP_FILE,'r')
+    for line in f:
+      jlines.append(json.loads(line))
     f.close()
   except Exception,e:
-    f = []
-  jlines = []
-  for line in f:
-    jlines.append(json.loads(line))
+    pass
   for k,v in nodes.iteritems():
     conns = v.get('to',[])
     v['num_conns'] = len(conns)
@@ -81,6 +81,8 @@ def get_node_availability(name,jlines):
   uptime = 0
   #sys.stderr.write ( "Getting Node availability of %s\n" % name)
   for stat in jlines:
+    if not stat['nodes']:
+      continue
     ts = stat['timestamp']
     if not begin:
       begin = last = ts
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 16479304..402ce256 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -4,14 +4,13 @@ GRAPH_SETTER1=dot
 GRAPH_SETTER2=circo
 GRAPH_SETTER3='neato -Goverlap=prism '
 GRAPH_SETTER4=sfdp
-#LOG_FILE=/var/log/syslog
+LOG_FILE=/var/log/syslog
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp`
-trap 'rm $DOTFILE' SIGINT SIGTERM
-sudo pkill -USR2 tincd
-sudo python tinc_stats.py |\
+trap 'rm $DOTFILE' SIGTERM
+sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\
     python parse_tinc_stats.py > $DOTFILE
 
 $GRAPH_SETTER1 -T$TYPE -o $1/retiolum_1.$TYPE $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py
index fb238abe..370c59f7 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats.py
@@ -7,7 +7,8 @@ import json
 
 
 TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum")
-SYSLOG_FILE = "/var/log/everything.log"
+os.environ["LOG_FILE"]
+SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
 
 
 # Tags and Delimiters
-- 
cgit v1.2.3


From 0ed905da208ff9eed13878485325ea26ae2e2c8f Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Fri, 4 Nov 2011 14:21:26 +0100
Subject: //retiolum/adv_graphgen: replace old parsing style, refactoring

tinc_stats now writes the current tinc state into stdout as json
sanitize.sh uses tinc_stats and pipes the output to parse_tinc_stats.py (the name is a lie)
---
 retiolum/scripts/adv_graphgen/parse.py            | 193 ----------------------
 retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 173 +++++++++++++++++++
 retiolum/scripts/adv_graphgen/sanitize.sh         |   8 +-
 retiolum/scripts/adv_graphgen/tinc_stats.py       |   8 +-
 4 files changed, 182 insertions(+), 200 deletions(-)
 delete mode 100755 retiolum/scripts/adv_graphgen/parse.py
 create mode 100755 retiolum/scripts/adv_graphgen/parse_tinc_stats.py

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
deleted file mode 100755
index e99a5573..00000000
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ /dev/null
@@ -1,193 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf8 -*-
-
-import sys,json
-supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
-""" TODO: Refactoring needed to pull the edges out of the node structures again,
-it should be easier to handle both structures"""
-DUMP_FILE = "/krebs/db/availability"
-def write_digraph(nodes):
-  """
-  writes the complete digraph in dot format
-  """
-  print ('digraph retiolum {')
-  #print ('  graph [center rankdir=LR packMode="clust"]')
-  print ('  graph [center packMode="clust"]')
-  print ('  node[shape=box,style=filled,fillcolor=grey]')
-  print ('  overlap=false')
-  generate_stats(nodes)
-  merge_edges(nodes)
-  write_stat_node(nodes)
-  for k,v in nodes.iteritems():
-    write_node(k,v)
-  print ('}')
-def dump_graph(nodes):
-  from time import time
-  graph = {}
-  graph['nodes'] = nodes
-  graph['timestamp'] = time()
-  f = open(DUMP_FILE,'a')
-  json.dump(graph,f)
-  f.write('\n')
-  f.close()
-def write_stat_node(nodes):
-  ''' Write a `stats` node in the corner
-      This node contains infos about the current number of active nodes and connections inside the network
-  '''
-  num_conns = 0
-  num_nodes = len(nodes)
-  for k,v in nodes.iteritems():
-    num_conns+= len(v['to'])
-  node_text = "  stats_node [label=\"Statistics\\l"
-  node_text += "Active Nodes: %s\\l" % num_nodes
-  node_text += "Connections : %s\\l" % num_conns
-  node_text += "\""
-  node_text += ",fillcolor=green"
-  node_text += "]"
-  print(node_text)
-
-def generate_stats(nodes):
-  """ Generates some statistics of the network and nodes
-  """
-  f = open(DUMP_FILE,'r')
-  jlines = []
-  for line in f:
-    jlines.append(json.loads(line))
-  f.close()
-  for k,v in nodes.iteritems():
-    conns = v.get('to',[])
-    v['num_conns'] = len(conns)
-    v['avg_weight'] = get_node_avg_weight(conns)
-    v['availability'] = get_node_availability(k,jlines)
-    sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
-def get_node_avg_weight(conns):
-  """ calculates the average weight for the given connections """
-  if not conns:
-    sys.syderr.write("get_node_avg_weight: connection parameter empty")
-    return 9001
-  else:
-    return sum([float(c['weight']) for c in conns])/len(conns)
-def get_node_availability(name,jlines):
-  """ calculates the node availability by reading the generated dump file
-  adding together the uptime of the node and returning the time
-	parms:
-          name - node name
-          jlines - list of already parsed dictionaries node archive
-  """
-  begin = last = current = 0
-  uptime = 0
-  #sys.stderr.write ( "Getting Node availability of %s\n" % name)
-  for stat in jlines:
-    ts = stat['timestamp']
-    if not begin:
-      begin = last = ts
-    current = ts
-    if stat['nodes'].get(name,{}).get('to',[]):
-      uptime += current - last
-    else:
-      pass
-      #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
-    last = ts
-  all_the_time = last - begin
-  return uptime/ all_the_time
-
-def delete_unused_nodes(nodes):
-  new_nodes = {}
-  for k,v in nodes.iteritems():
-    if v['external-ip'] == "(null)":
-      continue
-    if v.get('to',[]):
-      new_nodes[k] = v
-  for k,v in new_nodes.iteritems():
-    if not [ i for i in v['to'] if i['name'] in new_nodes]:
-      #del(new_nodes[k])
-      del(k)
-  return new_nodes
-def merge_edges(nodes):
-  """ merge back and forth edges into one
-  DESTRUCTS the current structure by deleting "connections" in the nodes
-  """
-  for k,v in nodes.iteritems():
-    for con in v.get('to',[]):
-      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
-        if k == secon['name']:
-          del (nodes[con['name']]['to'][i])
-          con['bidirectional'] = True
-
-
-def write_node(k,v):
-  """ writes a single node and its edges 
-      edges are weightet with the informations inside the nodes provided by
-      tinc
-  """
-
-  node = "  "+k+"[label=\""
-  node += k+"\\l"
-  node += "availability: %f\\l" % v['availability'] 
-  node += "avg weight: %.2f\\l" % v['avg_weight'] 
-  if v.has_key('num_conns'):
-    node += "Num Connects:"+str(v['num_conns'])+"\\l"
-  node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
-  for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
-    node += "internal:"+addr+"\\l"
-  node +="\""
-  if k in supernodes:
-    node += ",fillcolor=steelblue1"
-  #node +=",group=\""+v['external-ip'].replace(".","")+"\""
-  node += "]"
-  print node
-
-  for con in v.get('to',[]):
-    label  = con['weight']
-    w = int(con['weight'])
-    weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
-
-    length = str(float(w)/1500)
-    #weight = "1000" #str(300/float(con['weight']))
-    #weight = str((100/float(con['weight'])))
-    #weight = str(-1 * (200-100000/int(con['weight'])))
-    if float(weight) < 0 :
-      weight= "1"
-
-    #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
-    edge = "  "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length
-    if con.get('bidirectional',False):
-      edge += ",dir=both"
-    edge += "]"
-    print edge
-
-def parse_input():
-  nodes={}
-  for line in sys.stdin:
-    line = line.replace('\n','')
-    if line == 'Nodes:':
-      nodes={}
-      for line in sys.stdin:
-        if line == 'End of nodes.\n':
-          break
-        l = line.replace('\n','').split() #TODO unhack me
-        nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
-    if line == 'Subnet list:':
-      for line in sys.stdin:
-        if line == 'End of subnet list.\n':
-          break
-        l = line.replace('\n','').split() 
-        if not nodes[l[2]].get('internal-ip',False):
-           nodes[l[2]]['internal-ip'] = []
-        nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
-    if line == 'Edges:':
-      edges = {}
-      for line in sys.stdin:
-        if line == 'End of edges.\n':
-          break
-        l = line.replace('\n','').split() 
-
-        if not nodes[l[0]].has_key('to') :
-          nodes[l[0]]['to'] = []
-        nodes[l[0]]['to'].append(
-            {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
-  return nodes
-nodes = parse_input()
-nodes = delete_unused_nodes(nodes)
-dump_graph(nodes)
-write_digraph(nodes)
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
new file mode 100755
index 00000000..c02c9a84
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -0,0 +1,173 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+
+import sys,json
+supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
+""" TODO: Refactoring needed to pull the edges out of the node structures again,
+it should be easier to handle both structures"""
+DUMP_FILE = "/krebs/db/availability"
+def write_digraph(nodes):
+  """
+  writes the complete digraph in dot format
+  """
+  print ('digraph retiolum {')
+  #print ('  graph [center rankdir=LR packMode="clust"]')
+  print ('  graph [center packMode="clust"]')
+  print ('  node[shape=box,style=filled,fillcolor=grey]')
+  print ('  overlap=false')
+  generate_stats(nodes)
+  merge_edges(nodes)
+  write_stat_node(nodes)
+  for k,v in nodes.iteritems():
+    write_node(k,v)
+  print ('}')
+def dump_graph(nodes):
+  from time import time
+  graph = {}
+  graph['nodes'] = nodes
+  graph['timestamp'] = time()
+  f = open(DUMP_FILE,'a')
+  json.dump(graph,f)
+  f.write('\n')
+  f.close()
+def write_stat_node(nodes):
+  ''' Write a `stats` node in the corner
+      This node contains infos about the current number of active nodes and connections inside the network
+  '''
+  num_conns = 0
+  num_nodes = len(nodes)
+  for k,v in nodes.iteritems():
+    num_conns+= len(v['to'])
+  node_text = "  stats_node [label=\"Statistics\\l"
+  node_text += "Active Nodes: %s\\l" % num_nodes
+  node_text += "Connections : %s\\l" % num_conns
+  node_text += "\""
+  node_text += ",fillcolor=green"
+  node_text += "]"
+  print(node_text)
+
+def generate_stats(nodes):
+  """ Generates some statistics of the network and nodes
+  """
+  try:
+    f = open(DUMP_FILE,'r')
+    f.close()
+  except Exception,e:
+    f = []
+  jlines = []
+  for line in f:
+    jlines.append(json.loads(line))
+  for k,v in nodes.iteritems():
+    conns = v.get('to',[])
+    v['num_conns'] = len(conns)
+    v['avg_weight'] = get_node_avg_weight(conns)
+    v['availability'] = get_node_availability(k,jlines)
+    sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
+def get_node_avg_weight(conns):
+  """ calculates the average weight for the given connections """
+  if not conns:
+    sys.syderr.write("get_node_avg_weight: connection parameter empty")
+    return 9001
+  else:
+    return sum([float(c['weight']) for c in conns])/len(conns)
+def get_node_availability(name,jlines):
+  """ calculates the node availability by reading the generated dump file
+  adding together the uptime of the node and returning the time
+	parms:
+          name - node name
+          jlines - list of already parsed dictionaries node archive
+  """
+  begin = last = current = 0
+  uptime = 0
+  #sys.stderr.write ( "Getting Node availability of %s\n" % name)
+  for stat in jlines:
+    ts = stat['timestamp']
+    if not begin:
+      begin = last = ts
+    current = ts
+    if stat['nodes'].get(name,{}).get('to',[]):
+      uptime += current - last
+    else:
+      pass
+      #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
+    last = ts
+  all_the_time = last - begin
+  try:
+    return uptime/ all_the_time
+  except:
+    return 1
+
+def delete_unused_nodes(nodes):
+  new_nodes = {}
+  for k,v in nodes.iteritems():
+    if v['external-ip'] == "(null)":
+      continue
+    if v.get('to',[]):
+      new_nodes[k] = v
+  for k,v in new_nodes.iteritems():
+    if not [ i for i in v['to'] if i['name'] in new_nodes]:
+      #del(new_nodes[k])
+      del(k)
+  return new_nodes
+def merge_edges(nodes):
+  """ merge back and forth edges into one
+  DESTRUCTS the current structure by deleting "connections" in the nodes
+  """
+  for k,v in nodes.iteritems():
+    for con in v.get('to',[]):
+      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
+        if k == secon['name']:
+          del (nodes[con['name']]['to'][i])
+          con['bidirectional'] = True
+
+
+def write_node(k,v):
+  """ writes a single node and its edges 
+      edges are weightet with the informations inside the nodes provided by
+      tinc
+  """
+
+  node = "  "+k+"[label=\""
+  node += k+"\\l"
+  node += "availability: %f\\l" % v['availability'] 
+  node += "avg weight: %.2f\\l" % v['avg_weight'] 
+  if v.has_key('num_conns'):
+    node += "Num Connects:"+str(v['num_conns'])+"\\l"
+  node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
+  for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
+    node += "internal:"+addr+"\\l"
+  node +="\""
+  if k in supernodes:
+    node += ",fillcolor=steelblue1"
+  #node +=",group=\""+v['external-ip'].replace(".","")+"\""
+  node += "]"
+  print node
+
+  for con in v.get('to',[]):
+    label  = con['weight']
+    w = int(con['weight'])
+    weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
+
+    length = str(float(w)/1500)
+    #weight = "1000" #str(300/float(con['weight']))
+    #weight = str((100/float(con['weight'])))
+    #weight = str(-1 * (200-100000/int(con['weight'])))
+    if float(weight) < 0 :
+      weight= "1"
+
+    #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
+    edge = "  "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length
+    if con.get('bidirectional',False):
+      edge += ",dir=both"
+    edge += "]"
+    print edge
+
+def decode_input(FILE):
+  return json.load(FILE)
+nodes = decode_input(sys.stdin)
+nodes = delete_unused_nodes(nodes)
+try:
+  dump_graph(nodes)
+except Exception,e:
+  sys.stderr.write("Cannot dump graph: %s" % str(e))
+write_digraph(nodes)
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 3fddb605..16479304 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -1,18 +1,18 @@
 #!/bin/sh
-HERE=$(dirname `readlink -f $0`)
+cd $(dirname `readlink -f $0`)
 GRAPH_SETTER1=dot
 GRAPH_SETTER2=circo
 GRAPH_SETTER3='neato -Goverlap=prism '
 GRAPH_SETTER4=sfdp
-LOG_FILE=/var/log/syslog
+#LOG_FILE=/var/log/syslog
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp`
 trap 'rm $DOTFILE' SIGINT SIGTERM
 sudo pkill -USR2 tincd
-sudo sed -n '/tinc.retiolum/{s/.*tinc.retiolum\[[0-9]*\]: //gp}' $LOG_FILE |\
-    $HERE/parse.py > $DOTFILE
+sudo python tinc_stats.py |\
+    python parse_tinc_stats.py > $DOTFILE
 
 $GRAPH_SETTER1 -T$TYPE -o $1/retiolum_1.$TYPE $DOTFILE
 $GRAPH_SETTER2 -T$TYPE -o $1/retiolum_2.$TYPE $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py
index 0d3ebb11..fb238abe 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats.py
@@ -2,6 +2,8 @@
 from BackwardsReader import BackwardsReader
 import os
 import re
+import sys
+import json
 
 
 TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum")
@@ -30,7 +32,6 @@ def get_tinc_block(log_file):
     line = BOL.sub('',line).strip()
 
     if END_SUBNET in line:
-      print("Found end of block")
       in_block = True
 
     if not in_block:
@@ -39,7 +40,6 @@ def get_tinc_block(log_file):
     tinc_block.append(line)
 
     if BEGIN_NODES in line:
-      print("Found begin of block")
       break
   return reversed(tinc_block)
 
@@ -74,4 +74,6 @@ def parse_input(log_data):
             {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
   return nodes
 
-print parse_input((get_tinc_block(SYSLOG_FILE)))
+
+if __name__ == '__main__':
+  print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
-- 
cgit v1.2.3


From 193eaae037623b8843449114a22fc9a25778bcc5 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Fri, 4 Nov 2011 14:27:34 +0100
Subject: //retiolum/adv_graphgen: add BackwardsReader

dependency for tinc_stats.py
---
 retiolum/scripts/adv_graphgen/BackwardsReader.py | 35 ++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 retiolum/scripts/adv_graphgen/BackwardsReader.py

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/BackwardsReader.py
new file mode 100644
index 00000000..6bdbf43c
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/BackwardsReader.py
@@ -0,0 +1,35 @@
+import sys
+import os
+import string
+
+class BackwardsReader:
+  """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """
+  def readline(self):
+    while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size):
+      self.blkcount = self.blkcount + 1
+      line = self.data[0]
+      try:
+        self.f.seek(-self.blksize * self.blkcount, 2) 
+        self.data = string.split(self.f.read(self.blksize) + line, '\n')
+      except IOError:  
+        self.f.seek(0)
+        self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n')
+
+    if len(self.data) == 0:
+      return ""
+
+    line = self.data[-1]
+    self.data = self.data[:-1]
+    return line + '\n'
+
+  def __init__(self, file, blksize=4096):
+    """initialize the internal structures"""
+    self.size = os.stat(file)[6]
+    self.blksize = blksize
+    self.blkcount = 1
+    self.f = open(file, 'rb')
+    if self.size > self.blksize:
+      self.f.seek(-self.blksize * self.blkcount, 2) 
+    self.data = string.split(self.f.read(self.blksize), '\n')
+    if not self.data[-1]:
+      self.data = self.data[:-1]
-- 
cgit v1.2.3


From 0dd85c8b1acaebb18a648b0c0b5d4c48b69903d9 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Fri, 4 Nov 2011 14:43:58 +0100
Subject: //retiolum/adv_graphgen: tinc_stats now invokes USR2 against tinc
 deamon

---
 retiolum/scripts/adv_graphgen/tinc_stats.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py
index fb238abe..21d03cc4 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats.py
@@ -76,4 +76,7 @@ def parse_input(log_data):
 
 
 if __name__ == '__main__':
+  import subprocess,time
+  subprocess.popen("pkill -SIGUSR2 tincd")
+  time.sleep(1)
   print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
-- 
cgit v1.2.3