diff options
| -rwxr-xr-x | Reaktor/IRC/asybot.py | 2 | ||||
| -rwxr-xr-x | assets/hooks/pre-commit | 19 | ||||
| -rw-r--r-- | infest/profiles/makefu/aliases | 4 | ||||
| -rw-r--r-- | infest/profiles/makefu/zshrc | 4 | ||||
| -rwxr-xr-x | punani/bin/punani | 4 | ||||
| -rw-r--r-- | punani/db/punani | 12 | ||||
| -rwxr-xr-x | punani/index.py | 5 | ||||
| -rw-r--r-- | retiolum/hosts/skirfir | 2 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/all_the_graphs.sh | 5 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/anonytize.sh | 31 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/parse_tinc_anon.py | 135 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 19 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/sanitize.sh | 6 | ||||
| -rwxr-xr-x | retiolum/scripts/routing/defaultroute.sh | 16 |
14 files changed, 221 insertions, 43 deletions
diff --git a/Reaktor/IRC/asybot.py b/Reaktor/IRC/asybot.py index 99cb2fe6..657cee40 100755 --- a/Reaktor/IRC/asybot.py +++ b/Reaktor/IRC/asybot.py @@ -78,7 +78,7 @@ class asybot(asychat): if command == 'PING': self.push('PONG :%s' % rest) - log.info("Replying to servers PING with PONG :%s" %rest) + log.debug("Replying to servers PING with PONG :%s" %rest) elif command == 'PRIVMSG': self.on_privmsg(prefix, command, params, rest) diff --git a/assets/hooks/pre-commit b/assets/hooks/pre-commit index 76677a76..6ad1ca2a 100755 --- a/assets/hooks/pre-commit +++ b/assets/hooks/pre-commit @@ -1,19 +1,8 @@ #!/bin/sh set -euf -DB="assets.json" -TMP=`mktemp` - -echo "** checking $DB for sanity" -cat $DB | python -m json.tool > $TMP -RET=$? -if [ "$RET" = "0" ]; -then - echo "** Database sanity verified" - #git add $DB - #mv $TMP $DB +if which truth2json;then + truth2json >/dev/null && echo "db verified" else - echo "!! Database insane" - cat $TMP + echo "cannot verify as truth2json is not installed" fi -rm $TMP -exit $RET + diff --git a/infest/profiles/makefu/aliases b/infest/profiles/makefu/aliases new file mode 100644 index 00000000..bb01aad9 --- /dev/null +++ b/infest/profiles/makefu/aliases @@ -0,0 +1,4 @@ +alias s=sudo +alias t=task +alias ll="ls -al" +alias psg="ps aux | grep" diff --git a/infest/profiles/makefu/zshrc b/infest/profiles/makefu/zshrc index a60313e5..db83c42a 100644 --- a/infest/profiles/makefu/zshrc +++ b/infest/profiles/makefu/zshrc @@ -12,10 +12,10 @@ export DISABLE_AUTO_UPDATE="true" plugins=(git ssh-agent) source $ZSH/oh-my-zsh.sh -test -e $HOME/.bash_alias && source $HOME/.bash_alias +test -e $HOME/.aliases && source $HOME/.aliases # Customize to your needs... -export PATH=$PATH:/krebs/bin:$HOME/bin +export PATH=$PATH:/sbin:/usr/sbin:/krebs/bin:$HOME/bin HISTFILE=~/.histfile HISTSIZE=900001 SAVEHIST=900001 diff --git a/punani/bin/punani b/punani/bin/punani index 3f11cec8..97c7ac85 100755 --- a/punani/bin/punani +++ b/punani/bin/punani @@ -25,8 +25,8 @@ elif for PACKER_CMD in bauerbill packer yaourt pacman elif for PACKER_CMD in aptitude apt-get do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then - INSTALL_PARAM='--yes install' - REMOVE_PARAM='--yes remove' + INSTALL_PARAM='-y install' + REMOVE_PARAM='-y remove' else echo "Error 2: no known package manager found; no punani for you!" >&2 diff --git a/punani/db/punani b/punani/db/punani index df471f3a..ad477dab 100644 --- a/punani/db/punani +++ b/punani/db/punani @@ -7,12 +7,24 @@ "super-packer" : { "yaourt" : "pacman" }, + "nano" : { + "apt-get" : "nano", + "pacman" : "nano" + }, "vim" : { "apt-get" : "vim", "pacman" : "vim", "brew" : "vim", "yum" : "vim" }, + "zsh" : { + "pacman" : "zsh", + "apt-get" : "zsh" + }, + "git" : { + "pacman" : "git", + "apt-get" : "git-core" + }, "unison" : { "apt-get" : "unison", "pacman" : "unison" diff --git a/punani/index.py b/punani/index.py index ac19b2fb..ff483d37 100755 --- a/punani/index.py +++ b/punani/index.py @@ -3,6 +3,7 @@ import web import json import os +import sys from bot import * urls = ( '/', 'Index', @@ -12,7 +13,7 @@ urls = ( ) -PDB_FILE="db/punani" +PDB_FILE=os.path.dirname(os.path.abspath(sys.argv[0])) + "/db/punani" PORT="9111" CHANNEL="#retiolum" f = open(PDB_FILE) @@ -70,7 +71,7 @@ class ArchFinder: if __name__ == "__main__": import sys # Set IRC connection parameters. - irc_servers = [('supernode', 6667)] + irc_servers = [('supernode.retiolum', 6667)] irc_channels = [('#retiolum','')] # Prepare and start IRC bot. diff --git a/retiolum/hosts/skirfir b/retiolum/hosts/skirfir index 880376c9..0214e7a1 100644 --- a/retiolum/hosts/skirfir +++ b/retiolum/hosts/skirfir @@ -1,4 +1,4 @@ -Subnet = 42.0.0.222 +Subnet = 10.243.0.18/32 Subnet = 42:423b:0f94:6b03:7c3c:593e:67e8:c857/128 -----BEGIN RSA PUBLIC KEY----- diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh new file mode 100755 index 00000000..36b37f03 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh @@ -0,0 +1,5 @@ +#!/bin/sh +echo "`date` begin all graphs" >> /tmp/build_graph +cd $(dirname $(readlink -f $0)) +(./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)& +(./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)& diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh new file mode 100755 index 00000000..1ebfe972 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -euf +cd $(dirname `readlink -f $0`) +GRAPH_SETTER1=dot +GRAPH_SETTER2=circo +GRAPH_SETTER3='neato -Goverlap=prism ' +GRAPH_SETTER4=sfdp +LOG_FILE=/var/log/syslog +TYPE=svg +TYPE2=png +OPENER=/bin/true +DOTFILE=`mktemp` +trap 'rm $DOTFILE' INT TERM +sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\ + python parse_tinc_anon.py> $DOTFILE + + +i=1 +for setter in dot circo 'neato -Goverlap=prism ' sfdp +do + tmpgraph=`mktemp --tmpdir=$1` + $setter -T$TYPE -o $tmpgraph $DOTFILE + chmod go+rx $tmpgraph + mv $tmpgraph $1/retiolum_$i.$TYPE + i=`expr $i + 1` +done +#convert -resize 20% $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 +#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 +#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 +#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 +rm $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py new file mode 100755 index 00000000..3b0383da --- /dev/null +++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py @@ -0,0 +1,135 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +from BackwardsReader import BackwardsReader +import sys,json +#supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] +""" TODO: Refactoring needed to pull the edges out of the node structures again, +it should be easier to handle both structures""" +DUMP_FILE = "/krebs/db/availability" +def write_digraph(nodes): + """ + writes the complete digraph in dot format + """ + print ('digraph retiolum {') + #print (' graph [center rankdir=LR packMode="clust"]') + print (' graph [center packMode="clust"]') + print (' node[shape=circle,style=filled,fillcolor=grey]') + print (' overlap=false') + generate_stats(nodes) + merge_edges(nodes) + nodes = anon_nodes(nodes) + for k,v in nodes.iteritems(): + write_node(k,v) + write_stat_node(nodes) + print ('}') + +def anon_nodes(nodes): + i = "0" + newnodes = {} + for k,v in nodes.iteritems(): + for nodek,node in nodes.iteritems(): + for to in node['to']: + if to['name'] == k: + to['name'] = i + newnodes[i] = v + i = str(int(i)+1) + return newnodes + +def write_stat_node(nodes): + ''' Write a `stats` node in the corner + This node contains infos about the current number of active nodes and connections inside the network + ''' + num_conns = 0 + num_nodes = len(nodes) + for k,v in nodes.iteritems(): + num_conns+= len(v['to']) + node_text = " stats_node [shape=box,label=\"Statistics\\l" + node_text += "Active Nodes: %s\\l" % num_nodes + node_text += "Connections : %s\\l" % num_conns + node_text += "\"" + node_text += ",fillcolor=green" + node_text += "]" + print(node_text) + +def generate_stats(nodes): + """ Generates some statistics of the network and nodes + """ + for k,v in nodes.iteritems(): + conns = v.get('to',[]) + for c in conns: #sanitize weights + if float(c['weight']) > 9000: c['weight'] = str(9001) + elif float(c['weight']) < 0: c['weight'] = str(0) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) + +def delete_unused_nodes(nodes): + new_nodes = {} + for k,v in nodes.iteritems(): + if v['external-ip'] == "(null)": + continue + if v.get('to',[]): + new_nodes[k] = v + for k,v in new_nodes.iteritems(): + if not [ i for i in v['to'] if i['name'] in new_nodes]: + #del(new_nodes[k]) + del(k) + return new_nodes +def merge_edges(nodes): + """ merge back and forth edges into one + DESTRUCTS the current structure by deleting "connections" in the nodes + """ + for k,v in nodes.iteritems(): + for con in v.get('to',[]): + for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): + if k == secon['name']: + del (nodes[con['name']]['to'][i]) + con['bidirectional'] = True + + +def write_node(k,v): + """ writes a single node and its edges + edges are weightet with the informations inside the nodes provided by + tinc + """ + + node = " "+k #+"[label=\"" + #node += k+"\\l" + #node += "avg weight: %.2f\\l" % v['avg_weight'] + #if v.has_key('num_conns'): + # node += "Conns:"+str(v['num_conns'])+"\\l" + #node +="\"" + #node +=",group=\""+v['external-ip'].replace(".","") + "\"" + #node += "]" + print node + + for con in v.get('to',[]): + label = con['weight'] + w = int(con['weight']) + weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) + + length = str(float(w)/1500) + #weight = "1000" #str(300/float(con['weight'])) + #weight = str((100/float(con['weight']))) + #weight = str(-1 * (200-100000/int(con['weight']))) + if float(weight) < 0 : + weight= "1" + + #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n") + edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight #+ " minlen="+length + if con.get('bidirectional',False): + edge += ",dir=both" + edge += "]" + print edge + +def decode_input(FILE): + return json.load(FILE) +nodes = decode_input(sys.stdin) +nodes = delete_unused_nodes(nodes) +write_digraph(nodes) diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index b8054b0d..54dcc4ab 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -*- coding: utf8 -*- - +from BackwardsReader import BackwardsReader import sys,json supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] """ TODO: Refactoring needed to pull the edges out of the node structures again, @@ -51,12 +51,19 @@ def generate_stats(nodes): """ jlines = [] try: - f = open(DUMP_FILE,'r') - for line in f: - jlines.append(json.loads(line)) - f.close() + f = BackwardsReader(DUMP_FILE) + lines_to_use = 1000 + while True: + if lines_to_use == 0: break + line = f.readline() + if not line: break + jline = json.loads(line) + if not jline['nodes']: continue + + jlines.append(jline) + lines_to_use -=1 except Exception,e: - pass + sys.stderr.write(str(e)) for k,v in nodes.iteritems(): conns = v.get('to',[]) for c in conns: #sanitize weights diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index 5572b9d9..50f1659f 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -xeuf +set -euf cd $(dirname `readlink -f $0`) GRAPH_SETTER1=dot GRAPH_SETTER2=circo @@ -21,10 +21,12 @@ do tmpgraph=`mktemp --tmpdir=$1` $setter -T$TYPE -o $tmpgraph $DOTFILE chmod go+rx $tmpgraph - mv $tmpgraph $1retiolum_$i.$TYPE + mv $tmpgraph $1/retiolum_$i.$TYPE i=`expr $i + 1` done + #convert -resize 20% $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 #convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 #convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 #convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 +rm $DOTFILE diff --git a/retiolum/scripts/routing/defaultroute.sh b/retiolum/scripts/routing/defaultroute.sh index a9e227a6..f5a1a465 100755 --- a/retiolum/scripts/routing/defaultroute.sh +++ b/retiolum/scripts/routing/defaultroute.sh @@ -7,7 +7,7 @@ usage() echo "-d deactivate routing" } -defaultroute=$(route -n | grep 'UG[ \t]' | awk '{print $2}') +defaultroute=$(ip route show | grep default | awk '{ print $3 }') tincdir="/etc/tinc/retiolum" if [[ $(id -u) -gt 0 ]]; then @@ -30,17 +30,9 @@ case "$1" in exit 1;; esac - -cat $tincdir/hosts/* | grep Address | cut -b 11- | +cat $tincdir/tinc.conf | grep ConnectTo | cut -b 13- | while read host do - if [ "$(echo $host | sed 's/[0-9]*//g' | sed 's/\.//g')" = '' ]; then - route $command $host gw $defaultroute - else - host -4 $host | grep "has address" | awk '{ print $4 }' | - while read addr - do - route $command $addr gw $defaultroute && echo "$command routing to $addr via $defaultroute" - done - fi + addr=$(cat $tincdir/hosts/$host | grep Address | cut -b 11-) + echo route $command $addr gw $defaultroute done |