begin packaging of tinc_stats

8 files changed, 0 insertions, 605 deletions

diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
deleted file mode 100755
index 66defa44..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf8 -*-
-
-import sys,json
-""" TODO: Refactoring needed to pull the edges out of the node structures again,
-it should be easier to handle both structures"""
-DUMP_FILE = "/krebs/db/availability"
-
-def get_all_nodes():
-  import os
-  return os.listdir("/etc/tinc/retiolum/hosts")
-
-def generate_stats():
-  """ Generates availability statistics of the network and nodes
-  """
-  import json
-  jlines = []
-  try:
-    f = open(DUMP_FILE,'r')
-    for line in f:
-      jlines.append(json.loads(line))
-    f.close()
-  except Exception as e:
-    pass
-  all_nodes = {}
-  for k in get_all_nodes():
-    all_nodes[k] = get_node_availability(k,jlines)
-  print ( json.dumps(all_nodes))
-
-def get_node_availability(name,jlines):
-  """ calculates the node availability by reading the generated dump file
-  adding together the uptime of the node and returning the time
-	parms:
-          name - node name
-          jlines - list of already parsed dictionaries node archive
-  """
-  begin = last = current = 0
-  uptime = 0
-  for stat in jlines:
-    if not stat['nodes']:
-      continue
-    ts = stat['timestamp']
-    if not begin:
-      begin = last = ts
-    current = ts
-    if stat['nodes'].get(name,{}).get('to',[]):
-      uptime += current - last
-    else:
-      pass
-    last = ts
-  all_the_time = last - begin
-  try:
-    return uptime/ all_the_time
-  except:
-    return 1
-
-if __name__ == "__main__":
-  generate_stats()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
deleted file mode 100644
index 6bdbf43c..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import sys
-import os
-import string
-
-class BackwardsReader:
-  """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """
-  def readline(self):
-    while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size):
-      self.blkcount = self.blkcount + 1
-      line = self.data[0]
-      try:
-        self.f.seek(-self.blksize * self.blkcount, 2) 
-        self.data = string.split(self.f.read(self.blksize) + line, '\n')
-      except IOError:  
-        self.f.seek(0)
-        self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n')
-
-    if len(self.data) == 0:
-      return ""
-
-    line = self.data[-1]
-    self.data = self.data[:-1]
-    return line + '\n'
-
-  def __init__(self, file, blksize=4096):
-    """initialize the internal structures"""
-    self.size = os.stat(file)[6]
-    self.blksize = blksize
-    self.blkcount = 1
-    self.f = open(file, 'rb')
-    if self.size > self.blksize:
-      self.f.seek(-self.blksize * self.blkcount, 2) 
-    self.data = string.split(self.f.read(self.blksize), '\n')
-    if not self.data[-1]:
-      self.data = self.data[:-1]
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
deleted file mode 100755
index 038ca9c0..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf8 -*-
-import sys,json,os
-from Graph import delete_unused_nodes,resolve_myself
-GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat")
-
-def add_geo(nodes):
-  from pygeoip import GeoIP
-  gi = GeoIP(GEODB)
-
-  for k,v in nodes.iteritems():
-    try:
-      nodes[k].update(gi.record_by_addr(v["external-ip"]))
-    except Exception as e:
-      sys.stderr.write(str(e))
-      sys.stderr.write("Cannot determine GeoData for %s\n"%k)
-
-  return nodes
-def add_coords_to_edges(nodes):
-  from pygeoip import GeoIP
-  gi = GeoIP(GEODB)
-
-  for k,v in nodes.iteritems():
-    for i,j in enumerate(v.get("to",[])):
-      data=gi.record_by_addr(j["addr"])
-      try:
-        j["latitude"]=data["latitude"]
-        j["longitude"]=data["longitude"]
-      except Exception as e: pass
-
-  return nodes
-
-def add_jitter(nodes):
-  from random import random
-  #add a bit of jitter to all of the coordinates
-  max_jitter=0.005
-  for k,v in nodes.iteritems():
-    jitter_lat= max_jitter -random()*max_jitter*2
-    jitter_long= max_jitter -random()*max_jitter*2
-    try:
-      v["latitude"]= v["latitude"] + jitter_lat
-      v["longitude"]= v["longitude"] + jitter_long
-      for nodek,node in nodes.iteritems():
-        for to in node['to']:
-          if to['name'] == k:
-            to['latitude'] = v["latitude"]
-            to['longitude'] = v["longitude"]
-    except Exception as e: pass
-  return nodes
-
-if __name__ == "__main__":
-  import json
-  nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
-  print (json.dumps(nodes))
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
deleted file mode 100755
index 2fb09a58..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
+++ /dev/null
@@ -1,250 +0,0 @@
-#!/usr/bin/python
-from BackwardsReader import BackwardsReader
-import sys,json,os
-from Supernodes import check_all_the_super
-from Services import add_services
-from Availability import get_node_availability
-import sys,json
-from time import time
-DUMP_FILE = "/krebs/db/availability"
-
-
-def resolve_myself(nodes):
-  #resolve MYSELF to the real ip
-  for k,v in nodes.items():
-    if v["external-ip"] == "MYSELF":
-      for nodek,node in nodes.items():
-         for to in node['to']:
-           if to['name'] == k:
-             v["external-ip"] = to["addr"]
-  return nodes
-
-def dump_graph(nodes):
-  from time import time
-  graph = {}
-  graph['nodes'] = nodes
-  graph['timestamp'] = time()
-  f = open(DUMP_FILE,'a')
-  json.dump(graph,f)
-  f.write('\n')
-  f.close()
-
-def generate_availability_stats(nodes):
-  """ generates stats of from availability
-  """
-  jlines = []
-  try:
-    f = BackwardsReader(DUMP_FILE)
-    lines_to_use = 1000
-    while True:
-      if lines_to_use == 0: break
-      line = f.readline()
-      if not line: break
-      jline = json.loads(line)
-      if not jline['nodes']: continue
-
-      jlines.append(jline)
-      lines_to_use -=1
-  except Exception as e: sys.stderr.write(str(e))
-
-  for k,v in nodes.items():
-    v['availability'] = get_node_availability(k,jlines)
-    sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
-
-def generate_stats(nodes):
-  """ Generates some statistics of the network and nodes
-  """
-  for k,v in nodes.items():
-    conns = v.get('to',[])
-    for c in conns: #sanitize weights
-      if float(c['weight']) > 9000: c['weight'] = str(9001)
-      elif float(c['weight']) < 0: c['weight'] = str(0)
-    v['num_conns'] = len(conns)
-    v['avg_weight'] = get_node_avg_weight(conns)
-
-def get_node_avg_weight(conns):
-  """ calculates the average weight for the given connections """
-  if not conns:
-    sys.syderr.write("get_node_avg_weight: connection parameter empty")
-    return 9001
-  else:
-    return sum([float(c['weight']) for c in conns])/len(conns)
-
-def delete_unused_nodes(nodes):
-  """ Deletes all the nodes which are currently not connected to the network"""
-  new_nodes = {}
-  for k,v in nodes.items():
-    if v['external-ip'] == "(null)":
-      continue
-    if v.get('to',[]):
-      new_nodes[k] = v
-  for k,v in new_nodes.items():
-    if not [ i for i in v['to'] if i['name'] in new_nodes]:
-      del(k)
-  return new_nodes
-
-def merge_edges(nodes):
-  """ merge back and forth edges into one
-  DESTRUCTS the current structure by deleting "connections" in the nodes
-  """
-  for k,v in nodes.items():
-    for con in v.get('to',[]):
-      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
-        if k == secon['name']:
-          del (nodes[con['name']]['to'][i])
-          con['bidirectional'] = True
-
-
-def print_head():
-  print ('digraph retiolum {')
-  print ('  graph [center=true packMode="clust"]')
-  print ('  node[shape=box,style=filled,fillcolor=grey]')
-  print ('  overlap=false')
-
-def print_stat_node(nodes):
-  ''' Write a `stats` node in the corner
-      This node contains infos about the current number of active nodes and connections inside the network
-  '''
-  from time import localtime,strftime
-  num_conns = 0
-  num_nodes = len(nodes)
-  try: 
-    msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin)
-    s.send(msg)
-  except Exception as e: pass
-  for k,v in nodes.items():
-    num_conns+= len(v['to'])
-  node_text = "  stats_node [label=\"Statistics\\l"
-  node_text += "Build Date  : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
-  node_text += "Active Nodes: %s\\l" % num_nodes
-  node_text += "Connections : %s\\l" % num_conns
-  node_text += "\""
-  node_text += ",fillcolor=green"
-  node_text += "]"
-  print(node_text)
-
-def print_node(k,v):
-  """ writes a single node and its edges 
-      edges are weightet with the informations inside the nodes provided by
-      tinc
-  """
-
-  node = "  "+k+"[label=<<TABLE border='0' title='%s' cellborder='1' >" %k
-  node += "<TR><TD colspan='2'><B>%s</B></TD></TR>"%k
-  if 'availability' in v:
-    node += "<TR><TD>availability:</TD><TD>%f</TD></TR>" % v['availability'] 
-
-  if 'num_conns' in v:
-    node += "<TR><TD>Num Connects:</TD><TD>%s</TD></TR>"%str(v['num_conns'])
-
-  node += "<TR><TD>external:</TD><TD>"+v['external-ip']+":"+v['external-port']+"</TD></TR>"
-  for addr in v.get('internal-ip',['dunno lol']): 
-    node += "<TR><TD>internal:</TD><TD>%s</TD></TR>"%addr
-
-  if 'services' in v:
-    node +="<TR><TD colspan='2'><B>Services:</B></TD></TR>"
-    for service in v['services']:
-      try:uri,comment = service.split(" ",1)
-      except: 
-        uri = service
-        comment =""
-      node +="<TR >"
-      uri_proto=uri.split(":")[0]
-      uri_rest = uri.split(":")[1] 
-      if not uri_rest:
-        node +="<TD title='{0}' align='left' colspan='2' \
-href='{0}'><font color='darkred'>{0}</font>".format(uri)
-      else:
-        node +="<TD title='{0}' align='left' colspan='2' \
-href='{0}'><U>{0}</U>".format(uri)
-      if comment:
-        node += "<br align='left'/>    <I>{0}</I>".format(comment)
-      node +="</TD></TR>"
-  # end label
-  node +="</TABLE>>"
-
-  if v['num_conns'] == 1:
-    node += ",fillcolor=red"
-  elif k in supernodes:
-    node += ",fillcolor=steelblue1"
-  node += "]"
-
-  print(node)
-
-def print_anonymous_node(k,v):
-  """ writes a single node and its edges 
-      edges are weightet with the informations inside the nodes provided by
-      tinc
-  """
-  
-  node = "  "+k #+"[label=\""
-  print(node)
-
-def print_edge(k,v):
-  for con in v.get('to',[]):
-    label  = con['weight']
-    w = int(con['weight'])
-    weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
-
-    length = str(float(w)/1500)
-    if float(weight) < 0 :
-      weight= "1"
-
-    edge = "  "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight 
-    if con.get('bidirectional',False):
-      edge += ",dir=both"
-    edge += "]"
-    print(edge)
-
-def anonymize_nodes(nodes):
-  #anonymizes all nodes
-  i = "0"
-  newnodes = {}
-  for k,v in nodes.items():
-    for nodek,node in nodes.items():
-      for to in node['to']:
-        if to['name'] == k:
-          to['name'] = i
-    newnodes[i] = v
-    i = str(int(i)+1)
-  return newnodes
-
-if __name__ == "__main__":
-  supernodes= []
-  if len(sys.argv) != 2 or  sys.argv[1] not in ["anonymous","complete"]: 
-    print("usage: %s (anonymous|complete)")
-    sys.exit(1)
-
-  nodes = json.load(sys.stdin)
-  nodes = delete_unused_nodes(nodes)
-  print_head()
-  generate_stats(nodes)
-  merge_edges(nodes)
-
-
-  if sys.argv[1] == "anonymous":
-    nodes = anonymize_nodes(nodes)
-
-    for k,v in nodes.items():
-      print_anonymous_node(k,v)
-      print_edge(k,v)
-
-  elif sys.argv[1] == "complete":
-    for supernode,addr in check_all_the_super():
-      supernodes.append(supernode)
-
-    generate_availability_stats(nodes)
-    add_services(nodes)
-    for k,v in nodes.items():
-      print_node(k,v)
-      print_edge(k,v)
-    try:
-      dump_graph(nodes)
-    except Exception as e:
-      sys.stderr.write("Cannot dump graph: %s" % str(e))
-  else:
-    pass
-
-  print_stat_node(nodes)
-  print ('}')
-# vim: set sw=2:ts=2
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
deleted file mode 100755
index a81e2bef..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/python
-import subprocess
-import os
-import re
-import sys
-import json
-
-
-TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
-
-# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), 
-# or if the new and hip tincctl should be used
-
-
-# Tags and Delimiters
-TINC_TAG="tinc.%s" % TINC_NETWORK
-BEGIN_NODES = "Nodes:"
-END_NODES = "End of nodes."
-BEGIN_SUBNET = "Subnet list:"
-END_SUBNET = "End of subnet list"
-BEGIN_EDGES = "Edges:"
-END_EDGES = "End of edges."
-def usage():
-  from sys import argv,exit
-  print("""usage: %s
-This tool dumps all tinc node informations as json
-
-ENVIRONMENT VARIABLES:
-  TINC_NETWORK   The tinc network to dump 
-                      (default: retiolum)
-  LOG_FILE       If legacy tinc is used, defines the log file where tinc stats are dumped in
-                      (default: /var/log/everything.log)
-""" % argv[0])
-  exit(1)
-def debug(func):
-  from functools import wraps
-  @wraps(func)
-  def with_debug(*args,**kwargs):
-    print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs))
-    return func(*args,**kwargs)
-  return with_debug
-
-
-def parse_tinc_stats():
-  import subprocess
-  from time import sleep
-  from distutils.spawn import find_executable as which
-  #newest tinc
-  if which("tinc"):
-    return parse_new_input("tinc")
-  #new tinc
-  elif which("tincctl"):
-    return parse_new_input("tincctl")
-  #old tinc
-  else:
-    raise Exception("no tinc executable found!")
-  
-#@debug
-def get_tinc_block(log_file):
-  """ returns an iterateable block from the given log file (syslog) 
-      This function became obsolete with the introduction of tincctl
-  """
-  from BackwardsReader import BackwardsReader
-  tinc_block = []
-  in_block = False
-  bf = BackwardsReader(log_file)
-  BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
-  while True:
-    line = bf.readline()
-    if not line:
-      raise Exception("end of file at log file? This should not happen!")
-    line = BOL.sub('',line).strip()
-
-    if END_SUBNET in line:
-      in_block = True
-
-    if not in_block:
-      continue
-    tinc_block.append(line)
-
-    if BEGIN_NODES in line:
-      break
-  return reversed(tinc_block)
-
-def parse_new_input(tinc_bin):
-  nodes = {} 
-  pnodes = subprocess.Popen(
-          [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"],
-          stdout=subprocess.PIPE).communicate()[0].decode()
-  #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
-  for line in pnodes.split('\n'):
-    if not line: continue
-    l = line.split()
-    nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
-  psubnets = subprocess.check_output(
-          [tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode()
-  for line in psubnets.split('\n'):
-    if not line: continue
-    l = line.split()
-    try:
-      if not nodes[l[2]].get('internal-ip',False):
-         nodes[l[2]]['internal-ip'] = []
-      nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
-    except KeyError:
-      pass # node does not exist (presumably)
-  pedges = subprocess.check_output(
-          [tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode()
-  for line in pedges.split('\n'):
-    if not line: continue
-    l = line.split()
-    try:
-      if not 'to' in nodes[l[0]] :
-        nodes[l[0]]['to'] = []
-      nodes[l[0]]['to'].append(
-          {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
-    except KeyError:
-      pass #node does not exist
-  return nodes
-
-if __name__ == '__main__':
-  from sys import argv
-  if len(argv) > 1:
-    usage()
-  else:
-    print (json.dumps(parse_tinc_stats()))
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_stats/Services.py
deleted file mode 100644
index 6752e116..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py
+++ /dev/null
@@ -1,17 +0,0 @@
-services_dir="/home/makefu/r/services"
-def add_services(nodes):
-    for k,v in nodes.items():
-        n = nodes[k]
-        try:
-            with open("{0}/{1}".format(services_dir,k)) as f:
-                n["services"] = []
-                for line in f.readlines():
-                    n["services"].append(line.strip())
-        except Exception as e:
-            n["services"] = ["Error: No Service File!"]
-    return nodes
-if __name__ == "__main__":
-    import json,sys
-    nodes = add_services(json.load(sys.stdin))
-    print (json.dumps(nodes,indent=4))
-# vim: set expandtab:ts=4:sw=4
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
deleted file mode 100755
index 7e1f4dae..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python3
-
-def find_potential_super(path="/etc/tinc/retiolum/hosts"):
-  import os
-  import re
-
-  needle_addr = re.compile("Address\s*=\s*(.*)")
-  needle_port = re.compile("Port\s*=\s*(.*)")
-  for f in os.listdir(path):
-    with open(path+"/"+f) as of:
-      addrs = []
-      port = "655"
-
-      for line in of.readlines():
-
-        addr_found = needle_addr.match(line)
-        if addr_found:
-          addrs.append(addr_found.group(1))
-
-        port_found = needle_port.match(line)
-        if port_found:
-          port = port_found.group(1)
-      
-      if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
-
-def try_connect(addr):
-  try:
-    from socket import socket,AF_INET,SOCK_STREAM
-    s = socket(AF_INET,SOCK_STREAM)
-    s.settimeout(2)
-    s.connect(addr)
-    s.settimeout(None)
-    s.close()
-    return addr
-  except Exception as e:
-    pass
-    #return ()
-
-def check_one_super(ha):
-    host,addrs = ha
-    valid_addrs = []
-    for addr in addrs:
-      ret = try_connect(addr)
-      if ret: valid_addrs.append(ret)
-    if valid_addrs: return (host,valid_addrs)
-
-def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
-  from multiprocessing import Pool
-  p = Pool(20)
-  return filter(None,p.map(check_one_super,find_potential_super(path)))
-
-
-
-if __name__ == "__main__":
-  """
-  usage
-  """
-  for host,addrs in check_all_the_super():
-    print("%s %s" %(host,str(addrs)))
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
deleted file mode 100644
index f0fc8520..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import Availability
-import BackwardsReader
-import Log2JSON
-import Supernodes
-import Geo
-import Graph
-import Services