From abd83a5b91c00390300de40b8ada651212efe9a6 Mon Sep 17 00:00:00 2001
From: tv <tv@nomic.retiolum>
Date: Mon, 14 Jan 2013 15:14:46 +0100
Subject: retiolum find_super: parallel lookup

---
 retiolum/scripts/adv_graphgen/find_super | 37 +++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 13 deletions(-)
 mode change 100644 => 100755 retiolum/scripts/adv_graphgen/find_super

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super b/retiolum/scripts/adv_graphgen/find_super
old mode 100644
new mode 100755
index c89a94fb..e971e204
--- a/retiolum/scripts/adv_graphgen/find_super
+++ b/retiolum/scripts/adv_graphgen/find_super
@@ -1,14 +1,25 @@
-#!/bin/sh
-cd /etc/tinc/retiolum/hosts
-ls -1  . | while read fname;
-do
-    for i in `sed -n 's/Address\s*=\s*\(.*\)/\1/p' $fname`;do
-        if nc -zw 5 $i 655 2>/dev/null; then
-            echo "$fname - $i reachable"
-        #else
-            #echo -n
-            #echo "$fname - $i unreachable"
+#! /bin/dash
+set -euf
+case "${1-all}" in
+  (all)
+    find /etc/tinc/retiolum/hosts -type f |
+    xargs -P 423 -n 1 "$0" one
+    ;;
+  (one)
+    hosts_file=$2
+    name=$(basename $hosts_file)
 
-        fi
-    done
-done
+    if grep -q Address $hosts_file; then
+      script="$(sed -n '
+        s/[[:space:]]*//g
+        s/^\(Address\|Port\)=\(.*\)/\1="${\1+$\1 }\2"/p
+      ' $hosts_file)"
+      eval "$script"
+      for address in ${Address-}; do
+        if nc -zw 2 $address 655 2>/dev/null; then
+          echo $name - $address reachable
+        fi &
+      done
+      wait
+    fi
+esac
-- 
cgit v1.2.3


From 3b6a2e967d075145613faffa63ea9399fb6f1800 Mon Sep 17 00:00:00 2001
From: tv <tv@nomic.retiolum>
Date: Mon, 14 Jan 2013 16:05:20 +0100
Subject: retiolum find_super: replace xargs with for&wait

---
 retiolum/scripts/adv_graphgen/find_super | 44 +++++++++++++++-----------------
 1 file changed, 20 insertions(+), 24 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super b/retiolum/scripts/adv_graphgen/find_super
index e971e204..17a0c8c0 100755
--- a/retiolum/scripts/adv_graphgen/find_super
+++ b/retiolum/scripts/adv_graphgen/find_super
@@ -1,25 +1,21 @@
 #! /bin/dash
-set -euf
-case "${1-all}" in
-  (all)
-    find /etc/tinc/retiolum/hosts -type f |
-    xargs -P 423 -n 1 "$0" one
-    ;;
-  (one)
-    hosts_file=$2
-    name=$(basename $hosts_file)
-
-    if grep -q Address $hosts_file; then
-      script="$(sed -n '
-        s/[[:space:]]*//g
-        s/^\(Address\|Port\)=\(.*\)/\1="${\1+$\1 }\2"/p
-      ' $hosts_file)"
-      eval "$script"
-      for address in ${Address-}; do
-        if nc -zw 2 $address 655 2>/dev/null; then
-          echo $name - $address reachable
-        fi &
-      done
-      wait
-    fi
-esac
+set -eu
+cd /etc/tinc/retiolum/hosts
+for name in `
+  grep '^[[:space:]]*Address[[:space:]]*=' * |
+    cut -d: -f1 | sort | uniq
+`; do
+  if eval "`sed -n '
+      s/[[:space:]]\+//g
+      s/^\(Address\|Port\)=\(.*\)/\1="\${\1+\$\1\n}\2"/p
+    ' $name`"; then
+    port=${Port-655}
+    for host in $Address; do
+      if nc -zw 2 $host $port 2>/dev/null; then
+        echo $name - $host reachable
+      fi &
+    done
+    wait
+  fi &
+done
+wait
-- 
cgit v1.2.3


From bce6236eb787c047cfee496640533676d6dfeb45 Mon Sep 17 00:00:00 2001
From: tv <tv@nomic.retiolum>
Date: Mon, 14 Jan 2013 16:06:08 +0100
Subject: retiolum find_super: use same output format as find_super.py

---
 retiolum/scripts/adv_graphgen/find_super | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super b/retiolum/scripts/adv_graphgen/find_super
index 17a0c8c0..2c316d0e 100755
--- a/retiolum/scripts/adv_graphgen/find_super
+++ b/retiolum/scripts/adv_graphgen/find_super
@@ -12,7 +12,7 @@ for name in `
     port=${Port-655}
     for host in $Address; do
       if nc -zw 2 $host $port 2>/dev/null; then
-        echo $name - $host reachable
+        echo "$name [('$host', $port)]"
       fi &
     done
     wait
-- 
cgit v1.2.3


From 0373e52441bad0fcf37ad5f953ecde51886bfce0 Mon Sep 17 00:00:00 2001
From: makefu <root@pigstarter.de>
Date: Tue, 15 Jan 2013 16:12:34 +0000
Subject: adv_graphgen: remove temporary data properly

---
 retiolum/scripts/adv_graphgen/anonytize.sh | 6 +++---
 retiolum/scripts/adv_graphgen/sanitize.sh  | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index b31f4dbb..2e2045e4 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -9,8 +9,8 @@ LOG_FILE=${LOG_FILE:-/var/log/syslog}
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
-DOTFILE=`mktemp`
-trap 'rm $DOTFILE' INT TERM
+DOTFILE=`mktemp --suffix=anon`
+trap 'rm $DOTFILE' INT TERM KILL
 sudo -E python tinc_stats2json |\
     python parse_tinc_anon.py> $DOTFILE
 
@@ -24,7 +24,7 @@ do
   mv $tmpgraph $1/retiolum_$i.$TYPE
   i=`expr $i + 1`
 done
-#convert -resize 20% $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
+convert   $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
 #convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
 #convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
 #convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 45d29a22..846cc549 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -9,8 +9,8 @@ LOG_FILE=${LOG_FILE:-/var/log/syslog}
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
-DOTFILE=`mktemp`
-trap 'rm $DOTFILE' INT TERM
+DOTFILE=`mktemp --suffix=san`
+trap 'rm $DOTFILE' INT TERM KILL
 sudo -E python tinc_stats2json |\
     python parse_tinc_stats.py > $DOTFILE
 
@@ -25,7 +25,7 @@ do
   i=`expr $i + 1`
 done
 
-#convert -resize 20% $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
+convert $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
 #convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
 #convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
 #convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
-- 
cgit v1.2.3


From 0c1883da9a8667595c528130cb9ddc7b0c4a04b2 Mon Sep 17 00:00:00 2001
From: makefu <root@pigstarter.de>
Date: Thu, 24 Jan 2013 02:12:00 +0000
Subject: find_super is now using a threadpool

---
 retiolum/scripts/adv_graphgen/find_super.py | 39 ++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 15 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py
index df01734e..99548f1c 100644
--- a/retiolum/scripts/adv_graphgen/find_super.py
+++ b/retiolum/scripts/adv_graphgen/find_super.py
@@ -23,23 +23,32 @@ def find_super(path="/etc/tinc/retiolum/hosts"):
       
       if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
 
-def check_super(path="/etc/tinc/retiolum/hosts"):
-  from socket import socket,AF_INET,SOCK_STREAM
-  for host,addrs in find_super(path):
+def try_connect(addr):
+  try:
+    from socket import socket,AF_INET,SOCK_STREAM
+    s = socket(AF_INET,SOCK_STREAM)
+    s.settimeout(2)
+    s.connect(addr)
+    s.settimeout(None)
+    s.close()
+    return addr
+  except Exception as e:
+    pass
+    #return ()
+
+def check_one_super(ha):
+    host,addrs = ha
     valid_addrs = []
     for addr in addrs:
-      try:
-        s = socket(AF_INET,SOCK_STREAM)
-        s.settimeout(3)
-        s.connect(addr)
-        #print("success connecting %s:%d"%(addr))
-        s.settimeout(None)
-        s.close()
-        valid_addrs.append(addr)
-      except Exception as e:
-        pass
-        #print("cannot connect to %s:%d"%(addr))
-    if valid_addrs: yield (host,valid_addrs)
+      ret = try_connect(addr)
+      if ret: valid_addrs.append(ret)
+    if valid_addrs: return (host,valid_addrs)
+
+def check_super(path="/etc/tinc/retiolum/hosts"):
+  from multiprocessing import Pool
+  p = Pool(20)
+  return filter(None,p.map(check_one_super,find_super(path)))
+
 
 
 if __name__ == "__main__":
-- 
cgit v1.2.3


From 55d7d06d41fc47d32e4c138cb5aa7127cfd4864b Mon Sep 17 00:00:00 2001
From: makefu <root@pigstarter.de>
Date: Thu, 24 Jan 2013 02:24:16 +0000
Subject: fix misleading function name in find_super

---
 retiolum/scripts/adv_graphgen/find_super.py       | 8 ++++----
 retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 8 ++++++--
 2 files changed, 10 insertions(+), 6 deletions(-)

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py
index 99548f1c..ae0fae8f 100644
--- a/retiolum/scripts/adv_graphgen/find_super.py
+++ b/retiolum/scripts/adv_graphgen/find_super.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-def find_super(path="/etc/tinc/retiolum/hosts"):
+def find_potential_super(path="/etc/tinc/retiolum/hosts"):
   import os
   import re
 
@@ -44,10 +44,10 @@ def check_one_super(ha):
       if ret: valid_addrs.append(ret)
     if valid_addrs: return (host,valid_addrs)
 
-def check_super(path="/etc/tinc/retiolum/hosts"):
+def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
   from multiprocessing import Pool
   p = Pool(20)
-  return filter(None,p.map(check_one_super,find_super(path)))
+  return filter(None,p.map(check_one_super,find_potential_super(path)))
 
 
 
@@ -55,5 +55,5 @@ if __name__ == "__main__":
   """
   usage
   """
-  for host,addrs in check_super():
+  for host,addrs in check_all_the_super():
     print host,addrs
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index 76a3ffcd..e5bd96a8 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 from BackwardsReader import BackwardsReader
 import sys,json
-from find_super import check_super
+from find_super import check_all_the_super
 try:
   from time import time
   import socket
@@ -18,7 +18,7 @@ except Exception as e:
   sys.stderr.write("Cannot connect to graphite: %s\n" % str(e))
 
 supernodes= [ ]
-for supernode,addr in check_super():
+for supernode,addr in check_all_the_super():
   supernodes.append(supernode)
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
@@ -39,6 +39,7 @@ def write_digraph(nodes):
   for k,v in nodes.iteritems():
     write_node(k,v)
   print ('}')
+
 def dump_graph(nodes):
   from time import time
   graph = {}
@@ -48,6 +49,7 @@ def dump_graph(nodes):
   json.dump(graph,f)
   f.write('\n')
   f.close()
+
 def write_stat_node(nodes):
   ''' Write a `stats` node in the corner
       This node contains infos about the current number of active nodes and connections inside the network
@@ -95,6 +97,7 @@ def generate_stats(nodes):
     v['avg_weight'] = get_node_avg_weight(conns)
     v['availability'] = get_node_availability(k,jlines)
     sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
+
 def get_node_avg_weight(conns):
   """ calculates the average weight for the given connections """
   if not conns:
@@ -143,6 +146,7 @@ def delete_unused_nodes(nodes):
       #del(new_nodes[k])
       del(k)
   return new_nodes
+
 def merge_edges(nodes):
   """ merge back and forth edges into one
   DESTRUCTS the current structure by deleting "connections" in the nodes
-- 
cgit v1.2.3


From 59ccc0fd871c4bbd12066f2f6aacf835a839885d Mon Sep 17 00:00:00 2001
From: makefu <root@pigstarter.de>
Date: Fri, 8 Feb 2013 10:10:43 +0000
Subject: add nodes-to-json: parses retiolum/hosts folder into json

---
 retiolum/scripts/adv_graphgen/find_super | 21 ---------------------
 1 file changed, 21 deletions(-)
 delete mode 100755 retiolum/scripts/adv_graphgen/find_super

(limited to 'retiolum/scripts/adv_graphgen')

diff --git a/retiolum/scripts/adv_graphgen/find_super b/retiolum/scripts/adv_graphgen/find_super
deleted file mode 100755
index 2c316d0e..00000000
--- a/retiolum/scripts/adv_graphgen/find_super
+++ /dev/null
@@ -1,21 +0,0 @@
-#! /bin/dash
-set -eu
-cd /etc/tinc/retiolum/hosts
-for name in `
-  grep '^[[:space:]]*Address[[:space:]]*=' * |
-    cut -d: -f1 | sort | uniq
-`; do
-  if eval "`sed -n '
-      s/[[:space:]]\+//g
-      s/^\(Address\|Port\)=\(.*\)/\1="\${\1+\$\1\n}\2"/p
-    ' $name`"; then
-    port=${Port-655}
-    for host in $Address; do
-      if nc -zw 2 $host $port 2>/dev/null; then
-        echo "$name [('$host', $port)]"
-      fi &
-    done
-    wait
-  fi &
-done
-wait
-- 
cgit v1.2.3