summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xretiolum/scripts/adv_graphgen/anonytize.sh2
-rw-r--r--retiolum/scripts/adv_graphgen/find_super.py59
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_anon.py79
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_stats.py65
-rwxr-xr-xretiolum/scripts/adv_graphgen/sanitize.sh2
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Availability.py (renamed from retiolum/scripts/adv_graphgen/find_legacy_hosts.py)9
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py (renamed from retiolum/scripts/adv_graphgen/BackwardsReader.py)0
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Geo.py (renamed from retiolum/scripts/adv_graphgen/nodes_geoip.py)15
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/Graph.py44
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats2json)0
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py (renamed from retiolum/scripts/adv_graphgen/find-all-nodes.py)0
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/__init__.py6
12 files changed, 75 insertions, 206 deletions
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index 4aad6993..4ed5c8db 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -11,7 +11,7 @@ TYPE2=png
OPENER=/bin/true
DOTFILE=`mktemp --suffix=anon`
trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats2json |\
+sudo -E python tinc_stats/Log2JSON.py |\
python parse_tinc_anon.py> $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py
deleted file mode 100644
index ae0fae8f..00000000
--- a/retiolum/scripts/adv_graphgen/find_super.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-
-def find_potential_super(path="/etc/tinc/retiolum/hosts"):
- import os
- import re
-
- needle_addr = re.compile("Address\s*=\s*(.*)")
- needle_port = re.compile("Port\s*=\s*(.*)")
- for f in os.listdir(path):
- with open(path+"/"+f) as of:
- addrs = []
- port = "655"
-
- for line in of.readlines():
-
- addr_found = needle_addr.match(line)
- if addr_found:
- addrs.append(addr_found.group(1))
-
- port_found = needle_port.match(line)
- if port_found:
- port = port_found.group(1)
-
- if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
-
-def try_connect(addr):
- try:
- from socket import socket,AF_INET,SOCK_STREAM
- s = socket(AF_INET,SOCK_STREAM)
- s.settimeout(2)
- s.connect(addr)
- s.settimeout(None)
- s.close()
- return addr
- except Exception as e:
- pass
- #return ()
-
-def check_one_super(ha):
- host,addrs = ha
- valid_addrs = []
- for addr in addrs:
- ret = try_connect(addr)
- if ret: valid_addrs.append(ret)
- if valid_addrs: return (host,valid_addrs)
-
-def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
- from multiprocessing import Pool
- p = Pool(20)
- return filter(None,p.map(check_one_super,find_potential_super(path)))
-
-
-
-if __name__ == "__main__":
- """
- usage
- """
- for host,addrs in check_all_the_super():
- print host,addrs
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
index 82ee2f2e..9041fb35 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
@@ -1,6 +1,7 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
-from BackwardsReader import BackwardsReader
+from tinc_stats.BackwardsReader import BackwardsReader
+from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges
import sys,json
#supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
try:
@@ -19,6 +20,7 @@ except Exception as e:
""" TODO: Refactoring needed to pull the edges out of the node structures again,
it should be easier to handle both structures"""
DUMP_FILE = "/krebs/db/availability"
+
def write_digraph(nodes):
"""
writes the complete digraph in dot format
@@ -37,6 +39,7 @@ def write_digraph(nodes):
print ('}')
def anon_nodes(nodes):
+ #anonymizes all nodes
i = "0"
newnodes = {}
for k,v in nodes.iteritems():
@@ -66,46 +69,6 @@ def write_stat_node(nodes):
node_text += "]"
print(node_text)
-def generate_stats(nodes):
- """ Generates some statistics of the network and nodes
- """
- for k,v in nodes.iteritems():
- conns = v.get('to',[])
- for c in conns: #sanitize weights
- if float(c['weight']) > 9000: c['weight'] = str(9001)
- elif float(c['weight']) < 0: c['weight'] = str(0)
- v['num_conns'] = len(conns)
- v['avg_weight'] = get_node_avg_weight(conns)
-def get_node_avg_weight(conns):
- """ calculates the average weight for the given connections """
- if not conns:
- sys.syderr.write("get_node_avg_weight: connection parameter empty")
- return 9001
- else:
- return sum([float(c['weight']) for c in conns])/len(conns)
-
-def delete_unused_nodes(nodes):
- new_nodes = {}
- for k,v in nodes.iteritems():
- if v['external-ip'] == "(null)":
- continue
- if v.get('to',[]):
- new_nodes[k] = v
- for k,v in new_nodes.iteritems():
- if not [ i for i in v['to'] if i['name'] in new_nodes]:
- #del(new_nodes[k])
- del(k)
- return new_nodes
-def merge_edges(nodes):
- """ merge back and forth edges into one
- DESTRUCTS the current structure by deleting "connections" in the nodes
- """
- for k,v in nodes.iteritems():
- for con in v.get('to',[]):
- for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
- if k == secon['name']:
- del (nodes[con['name']]['to'][i])
- con['bidirectional'] = True
def write_node(k,v):
@@ -115,13 +78,6 @@ def write_node(k,v):
"""
node = " "+k #+"[label=\""
- #node += k+"\\l"
- #node += "avg weight: %.2f\\l" % v['avg_weight']
- #if v.has_key('num_conns'):
- # node += "Conns:"+str(v['num_conns'])+"\\l"
- #node +="\""
- #node +=",group=\""+v['external-ip'].replace(".","") + "\""
- #node += "]"
print node
for con in v.get('to',[]):
@@ -130,27 +86,20 @@ def write_node(k,v):
weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
length = str(float(w)/1500)
- #weight = "1000" #str(300/float(con['weight']))
- #weight = str((100/float(con['weight'])))
- #weight = str(-1 * (200-100000/int(con['weight'])))
if float(weight) < 0 :
weight= "1"
- #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
- edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight #+ " minlen="+length
+ edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight
if con.get('bidirectional',False):
edge += ",dir=both"
edge += "]"
print edge
-
-def decode_input(FILE):
- return json.load(FILE)
-nodes = decode_input(sys.stdin)
-nodes = delete_unused_nodes(nodes)
-write_digraph(nodes)
-try:
- end = time()
- msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end)
- s.send(msg)
- s.close()
-except Exception as e: pass
+if __name__ == "__main__":
+ nodes = delete_unused_nodes(json.load(sys.stdin))
+ write_digraph(nodes)
+ try:
+ end = time()
+ msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end)
+ s.send(msg)
+ s.close()
+ except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index e6a67e2c..4383b356 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -1,8 +1,10 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
-from BackwardsReader import BackwardsReader
+from tinc_stats.BackwardsReader import BackwardsReader
+from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges,get_node_avg_weight
+from tinc_stats.Supernodes import check_all_the_super
+from tinc_stats.Availability import get_node_availability
import sys,json
-from find_super import check_all_the_super
try:
from time import time
import socket
@@ -100,65 +102,6 @@ def generate_stats(nodes):
v['availability'] = get_node_availability(k,jlines)
sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
-def get_node_avg_weight(conns):
- """ calculates the average weight for the given connections """
- if not conns:
- sys.syderr.write("get_node_avg_weight: connection parameter empty")
- return 9001
- else:
- return sum([float(c['weight']) for c in conns])/len(conns)
-def get_node_availability(name,jlines):
- """ calculates the node availability by reading the generated dump file
- adding together the uptime of the node and returning the time
- parms:
- name - node name
- jlines - list of already parsed dictionaries node archive
- """
- begin = last = current = 0
- uptime = 0
- #sys.stderr.write ( "Getting Node availability of %s\n" % name)
- for stat in jlines:
- if not stat['nodes']:
- continue
- ts = stat['timestamp']
- if not begin:
- begin = last = ts
- current = ts
- if stat['nodes'].get(name,{}).get('to',[]):
- uptime += current - last
- else:
- pass
- #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
- last = ts
- all_the_time = last - begin
- try:
- return uptime/ all_the_time
- except:
- return 1
-
-def delete_unused_nodes(nodes):
- new_nodes = {}
- for k,v in nodes.iteritems():
- if v['external-ip'] == "(null)":
- continue
- if v.get('to',[]):
- new_nodes[k] = v
- for k,v in new_nodes.iteritems():
- if not [ i for i in v['to'] if i['name'] in new_nodes]:
- #del(new_nodes[k])
- del(k)
- return new_nodes
-
-def merge_edges(nodes):
- """ merge back and forth edges into one
- DESTRUCTS the current structure by deleting "connections" in the nodes
- """
- for k,v in nodes.iteritems():
- for con in v.get('to',[]):
- for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
- if k == secon['name']:
- del (nodes[con['name']]['to'][i])
- con['bidirectional'] = True
def write_node(k,v):
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index c8071dc3..7d556c68 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -11,7 +11,7 @@ TYPE2=png
OPENER=/bin/true
DOTFILE=`mktemp --suffix=san`
trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats2json |\
+sudo -E python tinc_stats/Log2JSON.py |\
python parse_tinc_stats.py > $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
index 52388b6d..a1ef13f1 100755
--- a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
@@ -9,8 +9,9 @@ DUMP_FILE = "/krebs/db/availability"
def get_all_nodes():
import os
return os.listdir("/etc/tinc/retiolum/hosts")
+
def generate_stats():
- """ Generates some statistics of the network and nodes
+ """ Generates availability statistics of the network and nodes
"""
import json
jlines = []
@@ -35,7 +36,6 @@ def get_node_availability(name,jlines):
"""
begin = last = current = 0
uptime = 0
- #sys.stderr.write ( "Getting Node availability of %s\n" % name)
for stat in jlines:
if not stat['nodes']:
continue
@@ -47,7 +47,6 @@ def get_node_availability(name,jlines):
uptime += current - last
else:
pass
- #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
last = ts
all_the_time = last - begin
try:
@@ -55,5 +54,5 @@ def get_node_availability(name,jlines):
except:
return 1
-
-generate_stats()
+if __name__ == "__main__":
+ generate_stats()
diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
index 6bdbf43c..6bdbf43c 100644
--- a/retiolum/scripts/adv_graphgen/BackwardsReader.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
diff --git a/retiolum/scripts/adv_graphgen/nodes_geoip.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
index 7f12d213..ba3f6738 100755
--- a/retiolum/scripts/adv_graphgen/nodes_geoip.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
@@ -1,25 +1,12 @@
#!/usr/bin/python3
# -*- coding: utf8 -*-
import sys,json
-
-def delete_unused_nodes(nodes):
- new_nodes = {}
- for k,v in nodes.iteritems():
- if v['external-ip'] == "(null)":
- continue
- if v.get('to',[]):
- new_nodes[k] = v
- for k,v in new_nodes.iteritems():
- if not [ i for i in v['to'] if i['name'] in new_nodes]:
- #del(new_nodes[k])
- del(k)
- return new_nodes
+from Graph import delete_unused_nodes
if __name__ == "__main__":
from pygeoip import GeoIP
gi = GeoIP("GeoLiteCity.dat")
for node,data in delete_unused_nodes(json.load(sys.stdin)).items():
-
try:
print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"]))
except:
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
new file mode 100644
index 00000000..26db3030
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+def generate_stats(nodes):
+ """ Generates some statistics of the network and nodes
+ """
+ for k,v in nodes.iteritems():
+ conns = v.get('to',[])
+ for c in conns: #sanitize weights
+ if float(c['weight']) > 9000: c['weight'] = str(9001)
+ elif float(c['weight']) < 0: c['weight'] = str(0)
+ v['num_conns'] = len(conns)
+ v['avg_weight'] = get_node_avg_weight(conns)
+
+def get_node_avg_weight(conns):
+ """ calculates the average weight for the given connections """
+ if not conns:
+ sys.syderr.write("get_node_avg_weight: connection parameter empty")
+ return 9001
+ else:
+ return sum([float(c['weight']) for c in conns])/len(conns)
+
+def delete_unused_nodes(nodes):
+ """ Deletes all the nodes which are currently not connected to the network"""
+ new_nodes = {}
+ for k,v in nodes.iteritems():
+ if v['external-ip'] == "(null)":
+ continue
+ if v.get('to',[]):
+ new_nodes[k] = v
+ for k,v in new_nodes.iteritems():
+ if not [ i for i in v['to'] if i['name'] in new_nodes]:
+ del(k)
+ return new_nodes
+
+def merge_edges(nodes):
+ """ merge back and forth edges into one
+ DESTRUCTS the current structure by deleting "connections" in the nodes
+ """
+ for k,v in nodes.iteritems():
+ for con in v.get('to',[]):
+ for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
+ if k == secon['name']:
+ del (nodes[con['name']]['to'][i])
+ con['bidirectional'] = True
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
index 644cbc63..644cbc63 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats2json
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
diff --git a/retiolum/scripts/adv_graphgen/find-all-nodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
index ae0fae8f..ae0fae8f 100644
--- a/retiolum/scripts/adv_graphgen/find-all-nodes.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
new file mode 100644
index 00000000..62f541d1
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
@@ -0,0 +1,6 @@
+import Availability
+import BackwardsReader
+import Log2JSON
+import Supernodes
+import Geo
+import Graph