Merge branch 'jerlbeck/wip/stats'

* This adds a new counter type (to measure time or delay)
* A statsd reporting backend. This can be fed into graphite
or similar tools.
* A periodic log backend for performance values

29 files changed, 2135 insertions, 27 deletions

diff --git a/.gitignore b/.gitignore
index 24c3af7a..598f88af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,7 @@ tests/testsuite.dir/
 tests/testsuite.log
 
 tests/utils/utils_test
+tests/stats/stats_test
 tests/kasumi/kasumi_test
 tests/sms/sms_test
 tests/timer/timer_test
diff --git a/TODO-RELEASE b/TODO-RELEASE
index 43b1e8ef..4d22f958 100644
--- a/TODO-RELEASE
+++ b/TODO-RELEASE
@@ -1 +1,3 @@
 #library	what		description / commit summary line
+libosmovty	abi-change	stats/vty: Add stats configuration (enum node_type has changed)
+libosmovty	abi-change	vty: Add reserved nodes to enum node_type
diff --git a/include/Makefile.am b/include/Makefile.am
index 52c6a38f..20735800 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -22,12 +22,14 @@ nobase_include_HEADERS = \
                        osmocom/core/linuxrbtree.h \
                        osmocom/core/logging.h \
                        osmocom/core/loggingrb.h \
+                       osmocom/core/stats.h \
                        osmocom/core/macaddr.h \
                        osmocom/core/msgb.h \
                        osmocom/core/panic.h \
                        osmocom/core/prim.h \
                        osmocom/core/process.h \
                        osmocom/core/rate_ctr.h \
+                       osmocom/core/stat_item.h \
                        osmocom/core/select.h \
                        osmocom/core/signal.h \
                        osmocom/core/socket.h \
@@ -112,6 +114,7 @@ nobase_include_HEADERS += \
                           osmocom/vty/buffer.h \
                           osmocom/vty/command.h \
                           osmocom/vty/logging.h \
+                          osmocom/vty/stats.h \
                           osmocom/vty/misc.h \
                           osmocom/vty/telnet_interface.h \
                           osmocom/vty/vector.h \
diff --git a/include/osmocom/core/logging.h b/include/osmocom/core/logging.h
index ba41762f..1c159d0b 100644
--- a/include/osmocom/core/logging.h
+++ b/include/osmocom/core/logging.h
@@ -69,7 +69,8 @@ void logp(int subsys, const char *file, int line, int cont, const char *format,
 #define DLSMS		-7
 #define DLCTRL		-8
 #define DLGTP		-9
-#define OSMO_NUM_DLIB	9
+#define DLSTATS		-10
+#define OSMO_NUM_DLIB	10
 
 struct log_category {
 	uint8_t loglevel;
diff --git a/include/osmocom/core/rate_ctr.h b/include/osmocom/core/rate_ctr.h
index 821c7cfd..03b1bfbe 100644
--- a/include/osmocom/core/rate_ctr.h
+++ b/include/osmocom/core/rate_ctr.h
@@ -30,6 +30,7 @@ struct rate_ctr_per_intv {
 /*! \brief data we keep for each actual value */
 struct rate_ctr {
 	uint64_t current;	/*!< \brief current value */
+	uint64_t previous;	/*!< \brief previous value, used for delta */
 	/*! \brief per-interval data */
 	struct rate_ctr_per_intv intv[RATE_CTR_INTV_NUM];
 };
@@ -46,6 +47,8 @@ struct rate_ctr_group_desc {
 	const char *group_name_prefix;
 	/*! \brief The human-readable description of the group */
 	const char *group_description;
+	/*! \brief The class to which this group belongs */
+	int class_id;
 	/*! \brief The number of counters in this group */
 	const unsigned int num_ctr;
 	/*! \brief Pointer to array of counter names */
@@ -78,9 +81,27 @@ static inline void rate_ctr_inc(struct rate_ctr *ctr)
 	rate_ctr_add(ctr, 1);
 }
 
+/*! \brief Return the counter difference since the last call to this function */
+int64_t rate_ctr_difference(struct rate_ctr *ctr);
+
 int rate_ctr_init(void *tall_ctx);
 
 struct rate_ctr_group *rate_ctr_get_group_by_name_idx(const char *name, const unsigned int idx);
 const struct rate_ctr *rate_ctr_get_by_name(const struct rate_ctr_group *ctrg, const char *name);
 
+typedef int (*rate_ctr_handler_t)(
+	struct rate_ctr_group *, struct rate_ctr *,
+	const struct rate_ctr_desc *, void *);
+typedef int (*rate_ctr_group_handler_t)(struct rate_ctr_group *, void *);
+
+
+/*! \brief Iterate over all counters
+ *  \param[in] handle_item Call-back function, aborts if rc < 0
+ *  \param[in] data Private data handed through to \a handle_counter
+ */
+int rate_ctr_for_each_counter(struct rate_ctr_group *ctrg,
+	rate_ctr_handler_t handle_counter, void *data);
+
+int rate_ctr_for_each_group(rate_ctr_group_handler_t handle_group, void *data);
+
 /*! @} */
diff --git a/include/osmocom/core/stat_item.h b/include/osmocom/core/stat_item.h
new file mode 100644
index 00000000..c2ad8cfd
--- /dev/null
+++ b/include/osmocom/core/stat_item.h
@@ -0,0 +1,130 @@
+#pragma once
+
+/*! \defgroup osmo_stat_item Statistics value item
+ *  @{
+ */
+
+/*! \file stat_item.h */
+
+#include <stdint.h>
+
+#include <osmocom/core/linuxlist.h>
+
+struct osmo_stat_item_desc;
+
+#define STAT_ITEM_NOVALUE_ID 0
+
+struct osmo_stat_item_value {
+	int32_t id;
+	int32_t value;
+};
+
+/*! \brief data we keep for each actual value */
+struct osmo_stat_item {
+	const struct osmo_stat_item_desc *desc;
+	/*! \brief the index of the freshest value */
+	int32_t last_value_index;
+	/*! \brief offset to the freshest value in the value fifo */
+	int16_t last_offs;
+	/*! \brief value fifo */
+	struct osmo_stat_item_value values[0];
+};
+
+/*! \brief statistics value description */
+struct osmo_stat_item_desc {
+	const char *name;	/*!< \brief name of the item */
+	const char *description;/*!< \brief description of the item */
+	const char *unit;	/*!< \brief unit of a value */
+	unsigned int num_values;/*!< \brief number of values to store */
+	int32_t default_value;
+};
+
+/*! \brief description of a statistics value group */
+struct osmo_stat_item_group_desc {
+	/*! \brief The prefix to the name of all values in this group */
+	const char *group_name_prefix;
+	/*! \brief The human-readable description of the group */
+	const char *group_description;
+	/*! \brief The class to which this group belongs */
+	int class_id;
+	/*! \brief The number of values in this group */
+	const unsigned int num_items;
+	/*! \brief Pointer to array of value names */
+	const struct osmo_stat_item_desc *item_desc;
+};
+
+/*! \brief One instance of a counter group class */
+struct osmo_stat_item_group {
+	/*! \brief Linked list of all value groups in the system */
+	struct llist_head list;
+	/*! \brief Pointer to the counter group class */
+	const struct osmo_stat_item_group_desc *desc;
+	/*! \brief The index of this value group within its class */
+	unsigned int idx;
+	/*! \brief Actual counter structures below */
+	struct osmo_stat_item *items[0];
+};
+
+struct osmo_stat_item_group *osmo_stat_item_group_alloc(
+	void *ctx,
+	const struct osmo_stat_item_group_desc *desc,
+	unsigned int idx);
+
+void osmo_stat_item_group_free(struct osmo_stat_item_group *statg);
+
+void osmo_stat_item_set(struct osmo_stat_item *item, int32_t value);
+
+int osmo_stat_item_init(void *tall_ctx);
+
+struct osmo_stat_item_group *osmo_stat_item_get_group_by_name_idx(
+	const char *name, const unsigned int idx);
+
+const struct osmo_stat_item *osmo_stat_item_get_by_name(
+	const struct osmo_stat_item_group *statg, const char *name);
+
+/*! \brief Retrieve the next value from the osmo_stat_item object.
+ * If a new value has been set, it is returned. The idx is used to decide
+ * which value to return.
+ * On success, *idx is updated to refer to the next unread value. If
+ * values have been missed due to FIFO overflow, *idx is incremented by
+ * (1 + num_lost).
+ * This way, the osmo_stat_item object can be kept stateless from the reader's
+ * perspective and therefore be used by several backends simultaneously.
+ *
+ * \param val	the osmo_stat_item object
+ * \param idx	identifies the next value to be read
+ * \param value	a pointer to store the value
+ * \returns  the increment of the index (0: no value has been read,
+ *           1: one value has been taken,
+ *           (1+n): n values have been skipped, one has been taken)
+ */
+int osmo_stat_item_get_next(const struct osmo_stat_item *item, int32_t *idx, int32_t *value);
+
+/*! \brief Get the last (freshest) value */
+static int32_t osmo_stat_item_get_last(const struct osmo_stat_item *item);
+
+/*! \brief Skip all values of the item and update idx accordingly */
+int osmo_stat_item_discard(const struct osmo_stat_item *item, int32_t *idx);
+
+/*! \brief Skip all values of all items and update idx accordingly */
+int osmo_stat_item_discard_all(int32_t *idx);
+
+typedef int (*osmo_stat_item_handler_t)(
+	struct osmo_stat_item_group *, struct osmo_stat_item *, void *);
+
+typedef int (*osmo_stat_item_group_handler_t)(struct osmo_stat_item_group *, void *);
+
+/*! \brief Iteate over all items
+ *  \param[in] handle_item Call-back function, aborts if rc < 0
+ *  \param[in] data Private data handed through to \a handle_item
+ */
+int osmo_stat_item_for_each_item(struct osmo_stat_item_group *statg,
+	osmo_stat_item_handler_t handle_item, void *data);
+
+int osmo_stat_item_for_each_group(osmo_stat_item_group_handler_t handle_group, void *data);
+
+static inline int32_t osmo_stat_item_get_last(const struct osmo_stat_item *item)
+{
+	return item->values[item->last_offs].value;
+}
+/*! @} */
diff --git a/include/osmocom/core/statistics.h b/include/osmocom/core/statistics.h
index de250bec..1e472ffd 100644
--- a/include/osmocom/core/statistics.h
+++ b/include/osmocom/core/statistics.h
@@ -9,6 +9,7 @@ struct osmo_counter {
 	const char *name;		/*!< \brief human-readable name */
 	const char *description;	/*!< \brief humn-readable description */
 	unsigned long value;		/*!< \brief current value */
+	unsigned long previous;		/*!< \brief previous value */
 };
 
 /*! \brief Increment counter */
@@ -37,8 +38,8 @@ struct osmo_counter *osmo_counter_alloc(const char *name);
  */
 void osmo_counter_free(struct osmo_counter *ctr);
 
-/*! \brief Iteate over all counters
- *  \param[in] handle_counter Call-back function
+/*! \brief Iterate over all counters
+ *  \param[in] handle_counter Call-back function, aborts if rc < 0
  *  \param[in] data Private dtata handed through to \a handle_counter
  */
 int osmo_counters_for_each(int (*handle_counter)(struct osmo_counter *, void *), void *data);
@@ -48,3 +49,6 @@ int osmo_counters_for_each(int (*handle_counter)(struct osmo_counter *, void *),
  *  \returns pointer to counter (\ref osmo_counter) or NULL otherwise
  */
 struct osmo_counter *osmo_counter_get_by_name(const char *name);
+
+/*! \brief Return the counter difference since the last call to this function */
+int osmo_counter_difference(struct osmo_counter *ctr);
diff --git a/include/osmocom/core/stats.h b/include/osmocom/core/stats.h
new file mode 100644
index 00000000..731fdb9b
--- /dev/null
+++ b/include/osmocom/core/stats.h
@@ -0,0 +1,110 @@
+/* (C) 2015 by Sysmocom s.f.m.c. GmbH
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+#pragma once
+
+#include <sys/socket.h>
+#include <osmocom/core/linuxlist.h>
+
+struct msgb;
+struct osmo_stat_item_group;
+struct osmo_stat_item_desc;
+struct rate_ctr_group;
+struct rate_ctr_desc;
+
+enum osmo_stats_class {
+	OSMO_STATS_CLASS_UNKNOWN,
+	OSMO_STATS_CLASS_GLOBAL,
+	OSMO_STATS_CLASS_PEER,
+	OSMO_STATS_CLASS_SUBSCRIBER,
+};
+
+enum osmo_stats_reporter_type {
+	OSMO_STATS_REPORTER_STATSD,
+	OSMO_STATS_REPORTER_LOG,
+};
+
+struct osmo_stats_reporter {
+	enum osmo_stats_reporter_type type;
+	char *name;
+
+	unsigned int have_net_config : 1;
+
+	/* config */
+	int enabled;
+	char *name_prefix;
+	char *dest_addr_str;
+	char *bind_addr_str;
+	int dest_port;
+	int mtu;
+	enum osmo_stats_class max_class;
+
+	/* state */
+	int running;
+	struct sockaddr dest_addr;
+	int dest_addr_len;
+	struct sockaddr bind_addr;
+	int bind_addr_len;
+	int fd;
+	struct msgb *buffer;
+	int agg_enabled;
+
+	struct llist_head list;
+	int (*open)(struct osmo_stats_reporter *srep);
+	int (*close)(struct osmo_stats_reporter *srep);
+	int (*send_counter)(struct osmo_stats_reporter *srep,
+		const struct rate_ctr_group *ctrg,
+		const struct rate_ctr_desc *desc,
+		int64_t value, int64_t delta);
+	int (*send_item)(struct osmo_stats_reporter *srep,
+		const struct osmo_stat_item_group *statg,
+		const struct osmo_stat_item_desc *desc,
+		int32_t value);
+};
+
+struct osmo_stats_config {
+	int interval;
+};
+
+extern struct osmo_stats_config *osmo_stats_config;
+
+void osmo_stats_init(void *ctx);
+int osmo_stats_report();
+
+int osmo_stats_set_interval(int interval);
+
+struct osmo_stats_reporter *osmo_stats_reporter_alloc(enum osmo_stats_reporter_type type,
+	const char *name);
+void osmo_stats_reporter_free(struct osmo_stats_reporter *srep);
+
+struct osmo_stats_reporter *osmo_stats_reporter_create_statsd(const char *name);
+struct osmo_stats_reporter *osmo_stats_reporter_create_log(const char *name);
+
+struct osmo_stats_reporter *osmo_stats_reporter_find(enum osmo_stats_reporter_type type,
+	const char *name);
+
+int osmo_stats_reporter_set_remote_addr(struct osmo_stats_reporter *srep, const char *addr);
+int osmo_stats_reporter_set_remote_port(struct osmo_stats_reporter *srep, int port);
+int osmo_stats_reporter_set_local_addr(struct osmo_stats_reporter *srep, const char *addr);
+int osmo_stats_reporter_set_mtu(struct osmo_stats_reporter *srep, int mtu);
+int osmo_stats_reporter_set_max_class(struct osmo_stats_reporter *srep,
+	enum osmo_stats_class class_id);
+int osmo_stats_reporter_set_name_prefix(struct osmo_stats_reporter *srep, const char *prefix);
+int osmo_stats_reporter_enable(struct osmo_stats_reporter *srep);
+int osmo_stats_reporter_disable(struct osmo_stats_reporter *srep);
diff --git a/include/osmocom/gprs/gprs_ns.h b/include/osmocom/gprs/gprs_ns.h
index d5a605df..7c3b23c1 100644
--- a/include/osmocom/gprs/gprs_ns.h
+++ b/include/osmocom/gprs/gprs_ns.h
@@ -118,6 +118,7 @@ struct gprs_nsvc {
 
 	struct osmo_timer_list timer;
 	enum nsvc_timer_mode timer_mode;
+	struct timeval timer_started;
 	int alive_retries;
 
 	unsigned int remote_end_is_sgsn:1;
@@ -125,6 +126,7 @@ struct gprs_nsvc {
 	unsigned int nsvci_is_valid:1;
 
 	struct rate_ctr_group *ctrg;
+	struct osmo_stat_item_group *statg;
 
 	/*! \brief which link-layer are we based on? */
 	enum gprs_ns_ll ll;
diff --git a/include/osmocom/vty/command.h b/include/osmocom/vty/command.h
index 4eb519f6..2ef4109e 100644
--- a/include/osmocom/vty/command.h
+++ b/include/osmocom/vty/command.h
@@ -75,6 +75,7 @@ enum node_type {
 	SERVICE_NODE,		/*!< \brief Service node. */
 	DEBUG_NODE,		/*!< \brief Debug node. */
 	CFG_LOG_NODE,		/*!< \brief Configure the logging */
+	CFG_STATS_NODE,		/*!< \brief Configure the statistics */
 
 	VTY_NODE,		/*!< \brief Vty node. */
 
@@ -83,6 +84,15 @@ enum node_type {
 	L_NS_NODE,		/*!< \brief NS node in libosmo-gb. */
 	L_BSSGP_NODE,		/*!< \brief BSSGP node in libosmo-gb. */
 
+	/*
+	 * When adding new nodes to the libosmocore project, these nodes can be
+	 * used to avoid ABI changes for unrelated projects.
+	 */
+	RESERVED1_NODE,		/*!< \brief Reserved for later extensions */
+	RESERVED2_NODE,		/*!< \brief Reserved for later extensions */
+	RESERVED3_NODE,		/*!< \brief Reserved for later extensions */
+	RESERVED4_NODE,		/*!< \brief Reserved for later extensions */
+
 	_LAST_OSMOVTY_NODE
 };
 
diff --git a/include/osmocom/vty/misc.h b/include/osmocom/vty/misc.h
index db552e77..f3b46dbd 100644
--- a/include/osmocom/vty/misc.h
+++ b/include/osmocom/vty/misc.h
@@ -2,6 +2,7 @@
 
 #include <osmocom/vty/vty.h>
 #include <osmocom/core/rate_ctr.h>
+#include <osmocom/core/stat_item.h>
 #include <osmocom/core/utils.h>
 
 #define VTY_DO_LOWER		1
@@ -10,7 +11,12 @@ char *vty_cmd_string_from_valstr(void *ctx, const struct value_string *vals,
 				 const char *end, int do_lower);
 
 void vty_out_rate_ctr_group(struct vty *vty, const char *prefix,
-                            struct rate_ctr_group *ctrg);
+			    struct rate_ctr_group *ctrg);
+
+void vty_out_stat_item_group(struct vty *vty, const char *prefix,
+			     struct osmo_stat_item_group *statg);
+
+void vty_out_statistics_full(struct vty *vty, const char *prefix);
 
 int osmo_vty_write_config_file(const char *filename);
 int osmo_vty_save_config_file(void);
diff --git a/include/osmocom/vty/stats.h b/include/osmocom/vty/stats.h
new file mode 100644
index 00000000..3851b4df
--- /dev/null
+++ b/include/osmocom/vty/stats.h
@@ -0,0 +1,3 @@
+#pragma once
+
+void osmo_stats_vty_add_cmds();
diff --git a/src/Makefile.am b/src/Makefile.am
index 4bf3408e..7aa6a78a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -15,7 +15,7 @@ libosmocore_la_SOURCES = timer.c select.c signal.c msgb.c bits.c \
 			 gsmtap_util.c crc16.c panic.c backtrace.c \
 			 conv.c application.c rbtree.c strrb.c \
 			 loggingrb.c crc8gen.c crc16gen.c crc32gen.c crc64gen.c \
-			 macaddr.c
+			 macaddr.c stat_item.c stats.c
 
 BUILT_SOURCES = crc8gen.c crc16gen.c crc32gen.c crc64gen.c
 
diff --git a/src/gb/gprs_bssgp.c b/src/gb/gprs_bssgp.c
index fe4fccae..e3e69c9c 100644
--- a/src/gb/gprs_bssgp.c
+++ b/src/gb/gprs_bssgp.c
@@ -31,6 +31,7 @@
 #include <osmocom/gsm/tlv.h>
 #include <osmocom/core/talloc.h>
 #include <osmocom/core/rate_ctr.h>
+#include <osmocom/core/stats.h>
 
 #include <osmocom/gprs/gprs_bssgp.h>
 #include <osmocom/gprs/gprs_ns.h>
@@ -54,6 +55,7 @@ static const struct rate_ctr_group_desc bssgp_ctrg_desc = {
 	.group_description = "BSSGP Peer Statistics",
 	.num_ctr = ARRAY_SIZE(bssgp_ctr_description),
 	.ctr_desc = bssgp_ctr_description,
+	.class_id = OSMO_STATS_CLASS_PEER,
 };
 
 LLIST_HEAD(bssgp_bvc_ctxts);
diff --git a/src/gb/gprs_ns.c b/src/gb/gprs_ns.c
index 827d09d7..2b189cd3 100644
--- a/src/gb/gprs_ns.c
+++ b/src/gb/gprs_ns.c
@@ -75,6 +75,8 @@
 #include <osmocom/core/talloc.h>
 #include <osmocom/core/select.h>
 #include <osmocom/core/rate_ctr.h>
+#include <osmocom/core/stat_item.h>
+#include <osmocom/core/stats.h>
 #include <osmocom/core/socket.h>
 #include <osmocom/core/signal.h>
 #include <osmocom/gprs/gprs_ns.h>
@@ -104,6 +106,8 @@ enum ns_ctr {
 	NS_CTR_NSEI_CHG,
 	NS_CTR_INV_VCI,
 	NS_CTR_INV_NSEI,
+	NS_CTR_LOST_ALIVE,
+	NS_CTR_LOST_RESET,
 };
 
 static const struct rate_ctr_desc nsvc_ctr_description[] = {
@@ -117,6 +121,8 @@ static const struct rate_ctr_desc nsvc_ctr_description[] = {
 	{ "nsei-chg",	"NS-VC changed NSEI count  " },
 	{ "inv-nsvci",	"NS-VCI was invalid count  " },
 	{ "inv-nsei",	"NSEI was invalid count    " },
+	{ "lost.alive",	"ALIVE ACK missing count   " },
+	{ "lost.reset",	"RESET ACK missing count   " },
 };
 
 static const struct rate_ctr_group_desc nsvc_ctrg_desc = {
@@ -126,6 +132,22 @@ static const struct rate_ctr_group_desc nsvc_ctrg_desc = {
 	.ctr_desc = nsvc_ctr_description,
 };
 
+enum ns_stat {
+	NS_STAT_ALIVE_DELAY,
+};
+
+static const struct osmo_stat_item_desc nsvc_stat_description[] = {
+	{ "alive.delay", "ALIVE reponse time        ", "ms", 16, 0 },
+};
+
+static const struct osmo_stat_item_group_desc nsvc_statg_desc = {
+	.group_name_prefix = "ns.nsvc",
+	.group_description = "NSVC Peer Statistics",
+	.num_items = ARRAY_SIZE(nsvc_stat_description),
+	.item_desc = nsvc_stat_description,
+	.class_id = OSMO_STATS_CLASS_PEER,
+};
+
 #define CHECK_TX_RC(rc, nsvc) \
 		if (rc < 0)							\
 			LOGP(DNS, LOGL_ERROR, "TX failed (%d) to peer %s\n",	\
@@ -218,6 +240,7 @@ struct gprs_nsvc *gprs_nsvc_create(struct gprs_ns_inst *nsi, uint16_t nsvci)
 	nsvc->timer.cb = gprs_ns_timer_cb;
 	nsvc->timer.data = nsvc;
 	nsvc->ctrg = rate_ctr_group_alloc(nsvc, &nsvc_ctrg_desc, nsvci);
+	nsvc->statg = osmo_stat_item_group_alloc(nsvc, &nsvc_statg_desc, nsvci);
 
 	llist_add(&nsvc->list, &nsi->gprs_nsvcs);
 
@@ -531,10 +554,20 @@ static void nsvc_start_timer(struct gprs_nsvc *nsvc, enum nsvc_timer_mode mode)
 	if (osmo_timer_pending(&nsvc->timer))
 		osmo_timer_del(&nsvc->timer);
 
+	gettimeofday(&nsvc->timer_started, NULL);
 	nsvc->timer_mode = mode;
 	osmo_timer_schedule(&nsvc->timer, seconds, 0);
 }
 
+static int nsvc_timer_elapsed_ms(struct gprs_nsvc *nsvc)
+{
+	struct timeval now, elapsed;
+	gettimeofday(&now, NULL);
+	timersub(&now, &nsvc->timer_started, &elapsed);
+
+	return 1000 * elapsed.tv_sec + elapsed.tv_usec / 1000;
+}
+
 static void gprs_ns_timer_cb(void *data)
 {
 	struct gprs_nsvc *nsvc = data;
@@ -549,6 +582,7 @@ static void gprs_ns_timer_cb(void *data)
 	switch (nsvc->timer_mode) {
 	case NSVC_TIMER_TNS_ALIVE:
 		/* Tns-alive case: we expired without response ! */
+		rate_ctr_inc(&nsvc->ctrg->ctr[NS_CTR_LOST_ALIVE]);
 		nsvc->alive_retries++;
 		if (nsvc->alive_retries >
 			nsvc->nsi->timeout[NS_TOUT_TNS_ALIVE_RETRIES]) {
@@ -578,6 +612,7 @@ static void gprs_ns_timer_cb(void *data)
 		nsvc_start_timer(nsvc, NSVC_TIMER_TNS_ALIVE);
 		break;
 	case NSVC_TIMER_TNS_RESET:
+		rate_ctr_inc(&nsvc->ctrg->ctr[NS_CTR_LOST_RESET]);
 		/* Chapter 7.3: Re-send the RESET */
 		gprs_ns_tx_reset(nsvc, NS_CAUSE_OM_INTERVENTION);
 		/* Re-start Tns-reset timer */
@@ -1272,6 +1307,9 @@ int gprs_ns_process_msg(struct gprs_ns_inst *nsi, struct msgb *msg,
 			rc = gprs_ns_tx_alive_ack(*nsvc);
 		break;
 	case NS_PDUT_ALIVE_ACK:
+		if ((*nsvc)->timer_mode == NSVC_TIMER_TNS_ALIVE)
+			osmo_stat_item_set((*nsvc)->statg->items[NS_STAT_ALIVE_DELAY],
+				nsvc_timer_elapsed_ms(*nsvc));
 		/* stop Tns-alive and start Tns-test */
 		nsvc_start_timer(*nsvc, NSVC_TIMER_TNS_TEST);
 		if ((*nsvc)->remote_end_is_sgsn) {
diff --git a/src/gb/gprs_ns_vty.c b/src/gb/gprs_ns_vty.c
index 155e1e97..5a951dca 100644
--- a/src/gb/gprs_ns_vty.c
+++ b/src/gb/gprs_ns_vty.c
@@ -167,8 +167,10 @@ static void dump_nse(struct vty *vty, struct gprs_nsvc *nsvc, int stats)
 			inet_ntoa(nsvc->ip.bts_addr.sin_addr),
 			ntohs(nsvc->ip.bts_addr.sin_port));
 	vty_out(vty, "%s", VTY_NEWLINE);
-	if (stats)
+	if (stats) {
 		vty_out_rate_ctr_group(vty, " ", nsvc->ctrg);
+		vty_out_stat_item_group(vty, " ", nsvc->statg);
+	}
 }
 
 static void dump_ns(struct vty *vty, struct gprs_ns_inst *nsi, int stats)
diff --git a/src/logging.c b/src/logging.c
index 20b0596b..876964ae 100644
--- a/src/logging.c
+++ b/src/logging.c
@@ -117,6 +117,11 @@ static const struct log_info_cat internal_cat[OSMO_NUM_DLIB] = {
 		.description = "GPRS GTP library",
 		.enabled = 1, .loglevel = LOGL_NOTICE,
 	},
+	[INT2IDX(DLSTATS)] = {
+		.name = "DLSTATS",
+		.description = "Statistics messages and logging",
+		.enabled = 1, .loglevel = LOGL_NOTICE,
+	},
 };
 
 /*! \brief descriptive string for each log level */
diff --git a/src/rate_ctr.c b/src/rate_ctr.c
index 8a232e86..50b3fe74 100644
--- a/src/rate_ctr.c
+++ b/src/rate_ctr.c
@@ -83,6 +83,15 @@ void rate_ctr_add(struct rate_ctr *ctr, int inc)
 	ctr->current += inc;
 }
 
+/*! \brief Return the counter difference since the last call to this function */
+int64_t rate_ctr_difference(struct rate_ctr *ctr)
+{
+	int64_t result = ctr->current - ctr->previous;
+	ctr->previous = ctr->current;
+
+	return result;
+}
+
 static void interval_expired(struct rate_ctr *ctr, enum rate_ctr_intv intv)
 {
 	/* calculate rate over last interval */
@@ -177,4 +186,36 @@ const struct rate_ctr *rate_ctr_get_by_name(const struct rate_ctr_group *ctrg, c
 	return NULL;
 }
 
+int rate_ctr_for_each_counter(struct rate_ctr_group *ctrg,
+	rate_ctr_handler_t handle_counter, void *data)
+{
+	int rc = 0;
+	int i;
+
+	for (i = 0; i < ctrg->desc->num_ctr; i++) {
+		struct rate_ctr *ctr = &ctrg->ctr[i];
+		rc = handle_counter(ctrg,
+			ctr, &ctrg->desc->ctr_desc[i], data);
+		if (rc < 0)
+			return rc;
+	}
+
+	return rc;
+}
+
+int rate_ctr_for_each_group(rate_ctr_group_handler_t handle_group, void *data)
+{
+	struct rate_ctr_group *statg;
+	int rc = 0;
+
+	llist_for_each_entry(statg, &rate_ctr_groups, list) {
+		rc = handle_group(statg, data);
+		if (rc < 0)
+			return rc;
+	}
+
+	return rc;
+}
+
+
 /*! @} */
diff --git a/src/stat_item.c b/src/stat_item.c
new file mode 100644
index 00000000..0545ea0d
--- /dev/null
+++ b/src/stat_item.c
@@ -0,0 +1,268 @@
+/* utility routines for keeping conters about events and the event rates */
+
+/* (C) 2015 by Sysmocom s.f.m.c. GmbH
+ * (C) 2009-2010 by Harald Welte <laforge@gnumonks.org>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+/*! \addtogroup osmo_stat_item
+ *  @{
+ */
+
+/*! \file stat_item.c */
+
+
+#include <stdint.h>
+#include <string.h>
+
+#include <osmocom/core/utils.h>
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/talloc.h>
+#include <osmocom/core/timer.h>
+#include <osmocom/core/stat_item.h>
+
+static LLIST_HEAD(osmo_stat_item_groups);
+static int32_t global_value_id = 0;
+
+static void *tall_stat_item_ctx;
+
+/*! \brief Allocate a new group of counters according to description
+ *  \param[in] ctx \ref talloc context
+ *  \param[in] desc Statistics item group description
+ *  \param[in] idx Index of new stat item group
+ */
+struct osmo_stat_item_group *osmo_stat_item_group_alloc(void *ctx,
+					    const struct osmo_stat_item_group_desc *desc,
+					    unsigned int idx)
+{
+	unsigned int group_size;
+	unsigned int items_size = 0;
+	unsigned int item_idx;
+	void *items;
+
+	struct osmo_stat_item_group *group;
+
+	group_size = sizeof(struct osmo_stat_item_group) +
+			desc->num_items * sizeof(struct osmo_stat_item *);
+
+	if (!ctx)
+		ctx = tall_stat_item_ctx;
+
+	group = talloc_zero_size(ctx, group_size);
+	if (!group)
+		return NULL;
+
+	group->desc = desc;
+	group->idx = idx;
+
+	/* Get combined size of all items */
+	for (item_idx = 0; item_idx < desc->num_items; item_idx++) {