diff options
19 files changed, 192 insertions, 141 deletions
diff --git a/retiolum/scripts/adv_graphgen/.gitignore b/retiolum/scripts/adv_graphgen/.gitignore new file mode 100644 index 00000000..fbee25a5 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/.gitignore @@ -0,0 +1,5 @@ +bin/ +include/ +lib/ +*.egg-info/ +GeoLiteCity.dat diff --git a/retiolum/scripts/adv_graphgen/README b/retiolum/scripts/adv_graphgen/README.md index 082e0f2b..082e0f2b 100644 --- a/retiolum/scripts/adv_graphgen/README +++ b/retiolum/scripts/adv_graphgen/README.md diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh deleted file mode 100755 index 9f3bf82b..00000000 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -( - echo "`date` begin all graphs" >> /tmp/build_graph - cd $(dirname $(readlink -f $0)) - PATH=$PATH:../../../util/bin/ - EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum - INTERNAL_FOLDER=/var/www/euer/graphs/retiolum - begin=`timer` - export GEOCTIYDB="$PWD/GeoLiteCity.dat" - (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)& - (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)& - (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)& -# wait - echo "`date` end all graphs" >> /tmp/build_graph -)& diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh deleted file mode 100755 index 04a68869..00000000 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=anon` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py anonymous> $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh deleted file mode 100755 index f7d0e7e7..00000000 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=san` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py complete > $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done - -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 -rm $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/scripts/all-the-graphs b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs new file mode 100755 index 00000000..3368d145 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs @@ -0,0 +1,13 @@ +#!/bin/bash + +EXTERNAL_FOLDER=${EXTERNAL_FOLDER:-/var/www/euer.krebsco.de/graphs/retiolum} +INTERNAL_FOLDER=${INTERNAL_FOLDER:-/var/www/euer/graphs/retiolum} +GEOCTIYDB="${GEOCITYDB:-}" + +if test -n "$GEOCITYDB";then + # creates a marker file + # TODO: copy map.html from + tinc-stats2json | add-geodata > "$INTERNAL_FOLDER/marker.json" +fi +build-graphs anonymous "$EXTERNAL_FOLDER" +build-graphs complete "$INTERNAL_FOLDER" diff --git a/retiolum/scripts/adv_graphgen/scripts/build-graphs b/retiolum/scripts/adv_graphgen/scripts/build-graphs new file mode 100755 index 00000000..3eb675ec --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/build-graphs @@ -0,0 +1,27 @@ +#!/bin/sh +set -euf +mode=${1?arg1 must be either 'anonymous' or 'complete'} +out=${2?arg2 must be set to an outdir} +# for creating all the graphs +TYPE=svg +# for creating real pictures +TYPE2=png + +DOTFILE=`mktemp` +trap 'rm "$DOTFILE"' INT TERM KILL EXIT + +tinc-stats2json | tinc-build-graph "$mode" > "$DOTFILE" + +i=1 +for setter in dot circo 'neato -Goverlap=prism ' sfdp +do + tmpgraph=`mktemp --tmpdir=$1` + # first build, then move to avoid half-built graphs + "$setter" -T$TYPE -o "$tmpgraph" "$DOTFILE" ||: + chmod go+rx "$tmpgraph" + mv "$tmpgraph" "$1/retiolum_$i.$TYPE" + i=`expr $i + 1` +done + +convert -limit memory 32Mib -limit map 64Mib "$out/retiolum_1.$TYPE" "$out/retiolum_1.$TYPE2" +rm "$DOTFILE" diff --git a/retiolum/scripts/adv_graphgen/setup.py b/retiolum/scripts/adv_graphgen/setup.py new file mode 100644 index 00000000..84597def --- /dev/null +++ b/retiolum/scripts/adv_graphgen/setup.py @@ -0,0 +1,41 @@ +import sys +from setuptools import setup + +setup( + name='tinc_graphs', + version='0.2.3', + + description='Create Graphs from tinc Stats', + long_description=open("README.md").read(), + license='WTFPL', + url='http://krebsco.de/', + download_url='https://pypi.python.org/pypi/tinc_graphs/', + + author='krebs', + author_email='spam@krebsco.de', + # you will also need graphviz and imagemagick + install_requires = [ 'pygeoip' ], + + packages=['tinc_graphs'], + entry_points={ + 'console_scripts' : [ + 'tinc-stats2json = tinc_graphs.Log2JSON:main', + 'tinc-build-graph = tinc_graphs.Graph:main', + 'add-geodata = tinc_graphs.Geo:main', + 'tinc-availability-stats = tinc_graphs.Availability:generate_stats', + ] + }, + + classifiers=[ + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Development Status :: 3 - Alpha", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: Implementation :: CPython", + ], +) + diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs.nix b/retiolum/scripts/adv_graphgen/tinc_graphs.nix new file mode 100644 index 00000000..b45fde13 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs.nix @@ -0,0 +1,25 @@ +with import <nixpkgs> {}; +# nix-build Reaktor.nix +# result/bin/reaktor +## or in your env +# nix-env -i -f tinc_graphs.nix + +buildPythonPackage rec { + name = "tinc_graphs-${version}"; + version = "0.2.6"; + propagatedBuildInputs = with pkgs;[ + pythonPackages.docopt + graphviz + imagemagick + pythonPackages.pygeoip + ]; + src = fetchurl { + url = ""; + sha256 = "1dksw1s1n2hxvnga6pygkr174dywncr0wiggkrkn1srbn2amh1c2"; + }; + meta = { + homepage = http://krebsco.de/; + description = "Create Graphs from Tinc Stats"; + license = stdenv.lib.licenses.wtfpl; + }; +} diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py index 66defa44..42f28ef9 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py @@ -1,14 +1,14 @@ #!/usr/bin/python # -*- coding: utf8 -*- -import sys,json +import sys,json,os """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" -DUMP_FILE = "/krebs/db/availability" +DUMP_FILE = os.environment.get("AVAILABILITY_FILE","tinc-availability.json") +hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") def get_all_nodes(): - import os - return os.listdir("/etc/tinc/retiolum/hosts") + return os.listdir(hostpath) def generate_stats(): """ Generates availability statistics of the network and nodes @@ -21,11 +21,13 @@ def generate_stats(): jlines.append(json.loads(line)) f.close() except Exception as e: - pass + print("Unable to open and parse Availability DB: {} (override with AVAILABILITY_FILE)".format(DUMP_FILE) + sys.exit(1) + all_nodes = {} for k in get_all_nodes(): all_nodes[k] = get_node_availability(k,jlines) - print ( json.dumps(all_nodes)) + print (json.dumps(all_nodes)) def get_node_availability(name,jlines): """ calculates the node availability by reading the generated dump file diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py index 6bdbf43c..6bdbf43c 100644 --- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py index 038ca9c0..78add918 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py @@ -1,9 +1,20 @@ #!/usr/bin/python3 # -*- coding: utf8 -*- import sys,json,os -from Graph import delete_unused_nodes,resolve_myself +from .Graph import delete_unused_nodes,resolve_myself GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat") +def copy_map(): + from shutil import copytree + from os.path import dirname,join,realpath + if len(sys.argv) != 2 or sys.argv[1] == "--help" : + print("usage: {} <destination>".format(sys.argv[0]) + print(" copies the map.html file to the <destination>") + sys.exit(1) + dstdir=sys.argv[1] + copytree(realpath(join(dirname(__file__),'static/map.html')),dstdir) + + def add_geo(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) @@ -14,8 +25,8 @@ def add_geo(nodes): except Exception as e: sys.stderr.write(str(e)) sys.stderr.write("Cannot determine GeoData for %s\n"%k) - return nodes + def add_coords_to_edges(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) @@ -48,7 +59,19 @@ def add_jitter(nodes): except Exception as e: pass return nodes -if __name__ == "__main__": +def main(): import json - nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) - print (json.dumps(nodes)) + try: + with open(GEODB) as f: f.read() + except: + print("cannot open {} (GEODB in env)".format(GEODB)) + sys.exit(1) + try: + nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) + print (json.dumps(nodes)) + except Exception as e: + print("cannot parse data received via stdin") + print(e) + +if __name__ == "__main__": + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py index 2fb09a58..b001780d 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py @@ -1,12 +1,12 @@ #!/usr/bin/python -from BackwardsReader import BackwardsReader +from .BackwardsReader import BackwardsReader import sys,json,os -from Supernodes import check_all_the_super -from Services import add_services -from Availability import get_node_availability +from .Supernodes import check_all_the_super +from .Services import add_services +from .Availability import get_node_availability import sys,json from time import time -DUMP_FILE = "/krebs/db/availability" +DUMP_FILE = os.environment.get("AVAILABILITY_FILE", "tinc-availability.json") def resolve_myself(nodes): @@ -19,6 +19,7 @@ def resolve_myself(nodes): v["external-ip"] = to["addr"] return nodes + def dump_graph(nodes): from time import time graph = {} @@ -209,7 +210,7 @@ def anonymize_nodes(nodes): i = str(int(i)+1) return newnodes -if __name__ == "__main__": +def main(): supernodes= [] if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]: print("usage: %s (anonymous|complete)") @@ -247,4 +248,8 @@ if __name__ == "__main__": print_stat_node(nodes) print ('}') + +if __name__ == "__main__": + main() + # vim: set sw=2:ts=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py index a81e2bef..7df042fd 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py @@ -28,10 +28,9 @@ This tool dumps all tinc node informations as json ENVIRONMENT VARIABLES: TINC_NETWORK The tinc network to dump (default: retiolum) - LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in - (default: /var/log/everything.log) """ % argv[0]) exit(1) + def debug(func): from functools import wraps @wraps(func) @@ -52,42 +51,18 @@ def parse_tinc_stats(): elif which("tincctl"): return parse_new_input("tincctl") #old tinc + elif which("tincd"): + print("old tincd not supported") + sys.exit(1) else: raise Exception("no tinc executable found!") - -#@debug -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) - This function became obsolete with the introduction of tincctl - """ - from BackwardsReader import BackwardsReader - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - tinc_block.append(line) - if BEGIN_NODES in line: - break - return reversed(tinc_block) def parse_new_input(tinc_bin): nodes = {} pnodes = subprocess.Popen( [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0].decode() - #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) for line in pnodes.split('\n'): if not line: continue l = line.split() @@ -117,9 +92,12 @@ def parse_new_input(tinc_bin): pass #node does not exist return nodes -if __name__ == '__main__': +def main(): from sys import argv if len(argv) > 1: usage() else: print (json.dumps(parse_tinc_stats())) + +if __name__ == '__main__': + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py index 6752e116..227e5aba 100644 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py @@ -1,4 +1,5 @@ -services_dir="/home/makefu/r/services" +import os,sys +services_dir=os.environment.get("SERIVCES_DIR","/home/reaktor/nag.services") def add_services(nodes): for k,v in nodes.items(): n = nodes[k] @@ -10,8 +11,15 @@ def add_services(nodes): except Exception as e: n["services"] = ["Error: No Service File!"] return nodes -if __name__ == "__main__": +def main(): import json,sys - nodes = add_services(json.load(sys.stdin)) - print (json.dumps(nodes,indent=4)) + try: + nodes = add_services(json.load(sys.stdin)) + print (json.dumps(nodes,indent=4)) + except: + print("unable to parse json data from stdin") + sys.exit(1) + +if __name__ == "__main__": + main() # vim: set expandtab:ts=4:sw=4 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py index 7e1f4dae..c8514c66 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py @@ -23,6 +23,7 @@ def find_potential_super(path="/etc/tinc/retiolum/hosts"): if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + def try_connect(addr): try: from socket import socket,AF_INET,SOCK_STREAM @@ -34,7 +35,7 @@ def try_connect(addr): return addr except Exception as e: pass - #return () + def check_one_super(ha): host,addrs = ha @@ -44,16 +45,21 @@ def check_one_super(ha): if ret: valid_addrs.append(ret) if valid_addrs: return (host,valid_addrs) -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): + +def check_all_the_super(path): from multiprocessing import Pool p = Pool(20) return filter(None,p.map(check_one_super,find_potential_super(path))) +def main(): + import os + hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): + for host,addrs in check_all_the_super(hostpath): print("%s %s" %(host,str(addrs))) + +if __name__ == "__main__": + main() + +# vim: set expandtab:ts=:sw=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py new file mode 100644 index 00000000..414ffe99 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py @@ -0,0 +1 @@ +__version__="0.2.3" diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html index ef8a0565..ef8a0565 100644 --- a/retiolum/scripts/adv_graphgen/map.html +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py deleted file mode 100644 index f0fc8520..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import Availability -import BackwardsReader -import Log2JSON -import Supernodes -import Geo -import Graph -import Services |