commit 7e9d14ed8727e095e87513ed021adfe532f73af2
Author: Fabio M. Di Nitto <fdinitto@redhat.com>
Date:   Mon Jan 4 15:30:15 2016 +0100

    [NO PUSH] sctp
    
    Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>

diff --git a/kronosnetd/vty_cli_cmds.c b/kronosnetd/vty_cli_cmds.c
index ab979b4..34b651d 100644
--- a/kronosnetd/vty_cli_cmds.c
+++ b/kronosnetd/vty_cli_cmds.c
@@ -1002,7 +1002,7 @@ static int knet_cmd_link(struct knet_vty *vty)
 			dst = &dst_addr;
 		}
 
-		knet_link_set_config(knet_iface->cfg_ring.knet_h, vty->host_id, vty->link_id, &src_addr, dst);
+		knet_link_set_config(knet_iface->cfg_ring.knet_h, vty->host_id, vty->link_id, KNET_LINK_TCP, &src_addr, dst);
 
 		knet_link_set_timeout(knet_iface->cfg_ring.knet_h, vty->host_id, vty->link_id, 1000, 5000, 2048);
 
@@ -1751,10 +1751,11 @@ static int knet_cmd_status(struct knet_vty *vty)
 			knet_link_get_link_list(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids, &link_ids_entries);
 			for (i = 0; i < link_ids_entries; i++) {
 				int dynamic;
+				int tcp_udp;
 				struct sockaddr_storage src_addr;
 				struct sockaddr_storage dst_addr;
 
-				dynamic = knet_link_get_config(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &src_addr, &dst_addr);
+				dynamic = knet_link_get_config(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &tcp_udp, &src_addr, &dst_addr);
 				if (dynamic >= 0) {
 					knet_link_get_status(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &status);
 					if (status.enabled == 1) {
@@ -1864,10 +1865,11 @@ static int knet_cmd_print_conf(struct knet_vty *vty)
 			knet_link_get_link_list(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids, &link_ids_entries);
 			for (i = 0; i < link_ids_entries; i++) {
 				int dynamic;
+				int tcp_udp;
 				struct sockaddr_storage src_addr;
 				struct sockaddr_storage dst_addr;
 
-				dynamic = knet_link_get_config(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &src_addr, &dst_addr);
+				dynamic = knet_link_get_config(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &tcp_udp, &src_addr, &dst_addr);
 				if (dynamic >= 0) {
 					knet_link_get_status(knet_iface->cfg_ring.knet_h, host_ids[j], link_ids[i], &status);
 					if (status.enabled == 1) {
diff --git a/libknet/Makefile.am b/libknet/Makefile.am
index a8d8a01..f31237e 100644
--- a/libknet/Makefile.am
+++ b/libknet/Makefile.am
@@ -27,7 +27,8 @@ sources			= \
 			  threads_dsthandler.c \
 			  threads_heartbeat.c \
 			  threads_pmtud.c \
-			  threads_send_recv.c
+			  threads_send_recv.c \
+			  threads_tcp_links.c
 
 include_HEADERS		= libknet.h
 
@@ -49,7 +50,8 @@ noinst_HEADERS		= \
 			  threads_dsthandler.h \
 			  threads_heartbeat.h \
 			  threads_pmtud.h \
-			  threads_send_recv.h
+			  threads_send_recv.h \
+			  threads_tcp_links.h
 
 lib_LTLIBRARIES		= libknet.la
 
diff --git a/libknet/handle.c b/libknet/handle.c
index c4a1f61..8065038 100644
--- a/libknet/handle.c
+++ b/libknet/handle.c
@@ -26,6 +26,7 @@
 #include "threads_pmtud.h"
 #include "threads_dsthandler.h"
 #include "threads_send_recv.h"
+#include "threads_tcp_links.h"
 #include "logging.h"
 
 static pthread_mutex_t handle_config_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -361,6 +362,14 @@ static int _init_epolls(knet_handle_t knet_h)
 		goto exit_fail;
 	}
 
+	knet_h->tcp_links_server_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS);
+	if (knet_h->tcp_links_server_epollfd < 0) {
+		savederrno = errno;
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to create epoll for tcp links: %s",
+			strerror(savederrno));
+		goto exit_fail;
+	}
+
 	knet_h->dst_link_handler_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS);
 	if (knet_h->dst_link_handler_epollfd < 0) {
 		savederrno = errno;
@@ -383,6 +392,13 @@ static int _init_epolls(knet_handle_t knet_h)
 		goto exit_fail;
 	}
 
+	if (_fdset_cloexec(knet_h->tcp_links_server_epollfd)) {
+		savederrno = errno;
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to set CLOEXEC on link to datafd epoll fd: %s",
+			strerror(savederrno)); 
+		goto exit_fail;
+	}
+
 	if (_fdset_cloexec(knet_h->dst_link_handler_epollfd)) {
 		savederrno = errno;
 		log_err(knet_h, KNET_SUB_HANDLE, "Unable to set CLOEXEC on dst cache epoll fd: %s",
@@ -439,6 +455,7 @@ static void _close_epolls(knet_handle_t knet_h)
 	epoll_ctl(knet_h->send_to_links_epollfd, EPOLL_CTL_DEL, knet_h->hostsockfd[0], &ev);
 	epoll_ctl(knet_h->dst_link_handler_epollfd, EPOLL_CTL_DEL, knet_h->dstsockfd[0], &ev);
 	close(knet_h->send_to_links_epollfd);
+	close(knet_h->tcp_links_server_epollfd);
 	close(knet_h->recv_from_links_epollfd);
 	close(knet_h->dst_link_handler_epollfd);
 }
@@ -447,6 +464,22 @@ static int _start_threads(knet_handle_t knet_h)
 {
 	int savederrno = 0;
 
+	savederrno = pthread_create(&knet_h->tcp_links_server_handler_thread, 0,
+				    _handle_tcp_links_server_thread, (void *) knet_h);
+	if (savederrno) {
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to start tcp links server thread: %s",
+			strerror(savederrno));
+		goto exit_fail;
+	}
+
+	savederrno = pthread_create(&knet_h->tcp_links_client_handler_thread, 0,
+				    _handle_tcp_links_client_thread, (void *) knet_h);
+	if (savederrno) {
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to start tcp links client thread: %s",
+			strerror(savederrno));
+		goto exit_fail;
+	}
+
 	savederrno = pthread_create(&knet_h->pmtud_link_handler_thread, 0,
 				    _handle_pmtud_link_thread, (void *) knet_h);
 	if (savederrno) {
@@ -501,6 +534,12 @@ static void _stop_threads(knet_handle_t knet_h)
 	pthread_cond_signal(&knet_h->host_cond);
 	pthread_mutex_unlock(&knet_h->host_mutex);
 
+	pthread_cancel(knet_h->tcp_links_server_handler_thread);
+	pthread_join(knet_h->tcp_links_server_handler_thread, &retval);
+
+	pthread_cancel(knet_h->tcp_links_client_handler_thread);
+	pthread_join(knet_h->tcp_links_client_handler_thread, &retval);
+
 	pthread_cancel(knet_h->heartbt_thread);
 	pthread_join(knet_h->heartbt_thread, &retval);
 
diff --git a/libknet/internals.h b/libknet/internals.h
index c177e93..9be6093 100644
--- a/libknet/internals.h
+++ b/libknet/internals.h
@@ -27,6 +27,7 @@
 
 struct knet_listener {
 	int sock;
+	int tcp_udp;
 	struct sockaddr_storage address;
 	struct knet_listener *next;
 };
@@ -46,7 +47,11 @@ struct knet_link {
 	struct knet_link_status status;
 	/* internals */
 	uint8_t link_id;
+	int outsock;
 	int listener_sock;
+	int tcp_udp; /* link is tcp or udp */
+	int tcp_srv_sock; /* tcp server side sock */
+	int tcp_clt_sock; /* tcp client side sock */
 	unsigned int configured:1; /* set to 1 if src/dst have been configured */
 	unsigned int remoteconnected:1; /* link is enabled for data (peer view) */
 	unsigned int donnotremoteupdate:1;    /* define source of the update */
@@ -121,6 +126,7 @@ struct knet_handle {
 	int dstsockfd[2];
 	int send_to_links_epollfd;
 	int recv_from_links_epollfd;
+	int tcp_links_server_epollfd;
 	int dst_link_handler_epollfd;
 	int pmtud_in_progress;
 	int pmtud_fini_requested;
@@ -143,6 +149,8 @@ struct knet_handle {
 	pthread_t heartbt_thread;
 	pthread_t dst_link_handler_thread;
 	pthread_t pmtud_link_handler_thread;
+	pthread_t tcp_links_client_handler_thread;
+	pthread_t tcp_links_server_handler_thread;
 	int lock_init_done;
 	pthread_rwlock_t list_rwlock;
 	pthread_rwlock_t listener_rwlock;
diff --git a/libknet/libknet.h b/libknet/libknet.h
index 5a2c77d..b040da1 100644
--- a/libknet/libknet.h
+++ b/libknet/libknet.h
@@ -845,7 +845,11 @@ int knet_host_get_status(knet_handle_t knet_h, uint16_t host_id,
  * -1 on error and errno is set.
  */
 
+#define KNET_LINK_UDP 0
+#define KNET_LINK_TCP 1
+
 int knet_link_set_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id,
+			 int tcp_udp,
 			 struct sockaddr_storage *src_addr,
 			 struct sockaddr_storage *dst_addr);
 
@@ -871,6 +875,7 @@ int knet_link_set_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
  */
 
 int knet_link_get_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id,
+			 int *tcp_udp,
 			 struct sockaddr_storage *src_addr,
 			 struct sockaddr_storage *dst_addr);
 
@@ -1151,7 +1156,7 @@ int knet_link_get_status(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
 #define KNET_SUB_SWITCH_T    8 /* switching thread */
 #define KNET_SUB_HB_T        9 /* heartbeat thread */
 #define KNET_SUB_PMTUD_T    10 /* Path MTU Discovery thread */
-#define KNET_SUB_FILTER     11 /* (ether)filter errors */
+#define KNET_SUB_TCP_LINK_T 11 /* TCP connection handler thread */
 #define KNET_SUB_CRYPTO     12 /* crypto.c generic layer */
 #define KNET_SUB_NSSCRYPTO  13 /* nsscrypto.c */
 #define KNET_SUB_LAST       KNET_SUB_NSSCRYPTO
diff --git a/libknet/link.c b/libknet/link.c
index 24813a1..8011d9d 100644
--- a/libknet/link.c
+++ b/libknet/link.c
@@ -43,6 +43,7 @@ int _link_updown(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id,
 }
 
 int knet_link_set_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id,
+			 int tcp_udp,
 			 struct sockaddr_storage *src_addr,
 			 struct sockaddr_storage *dst_addr)
 {
@@ -65,6 +66,11 @@ int knet_link_set_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
 		return -1;
 	}
 
+	if ((tcp_udp < 0) || (tcp_udp > 1)) {
+		errno = EINVAL;
+		return -1;
+	}
+
 	savederrno = pthread_rwlock_wrlock(&knet_h->list_rwlock);
 	if (savederrno) {
 		log_err(knet_h, KNET_SUB_LINK, "Unable to get write lock: %s",
@@ -144,6 +150,7 @@ int knet_link_set_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
 
 exit_unlock:
 	if (!err) {
+		link->tcp_udp = tcp_udp;
 		link->configured = 1;
 		link->pong_count = KNET_LINK_DEFAULT_PONG_COUNT;
 	}
@@ -153,6 +160,7 @@ exit_unlock:
 }
 
 int knet_link_get_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id,
+			 int *tcp_udp,
 			 struct sockaddr_storage *src_addr,
 			 struct sockaddr_storage *dst_addr)
 {
@@ -180,6 +188,11 @@ int knet_link_get_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
 		return -1;
 	}
 
+	if (!tcp_udp) {
+		errno = EINVAL;
+		return -1;
+	}
+
 	savederrno = pthread_rwlock_rdlock(&knet_h->list_rwlock);
 	if (savederrno) {
 		log_err(knet_h, KNET_SUB_LINK, "Unable to get read lock: %s",
@@ -207,6 +220,8 @@ int knet_link_get_config(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id
 		goto exit_unlock;
 	}
 
+	*tcp_udp = link->tcp_udp;
+
 	memmove(src_addr, &link->src_addr, sizeof(struct sockaddr_storage));
 
 	if (link->dynamic == KNET_LINK_DYNIP) {
diff --git a/libknet/listener.c b/libknet/listener.c
index bf81937..0df43a8 100644
--- a/libknet/listener.c
+++ b/libknet/listener.c
@@ -15,13 +15,17 @@
 #include <sys/epoll.h>
 #include <string.h>
 #include <stdlib.h>
+#include <netinet/tcp.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/sctp.h>
 
 #include "internals.h"
 #include "common.h"
 #include "logging.h"
 #include "listener.h"
 
-#define KNET_RING_RCVBUFF 8388608
+#define KNET_RING_BUFF 8388608
 
 int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 {
@@ -44,7 +48,9 @@ int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 	while (listener) {
 		count++;
 		log_debug(knet_h, KNET_SUB_LISTENER, "checking listener: %d", count);
-		if (!memcmp(&lnk->src_addr, &listener->address, sizeof(struct sockaddr_storage))) {
+		if ((!memcmp(&lnk->src_addr, &listener->address, sizeof(struct sockaddr_storage))) &&
+		    (lnk->tcp_udp == listener->tcp_udp)) {
+			/* check for outsock! */
 			log_debug(knet_h, KNET_SUB_LISTENER, "found active listener");
 			break;
 		}
@@ -64,7 +70,13 @@ int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 		memset(listener, 0, sizeof(struct knet_listener));
 		memmove(&listener->address, &lnk->src_addr, sizeof(struct sockaddr_storage));
 
-		listener->sock = socket(listener->address.ss_family, SOCK_DGRAM, 0);
+		listener->tcp_udp = lnk->tcp_udp;
+
+		if (listener->tcp_udp) {
+			listener->sock = socket(listener->address.ss_family, SOCK_STREAM, IPPROTO_SCTP);
+		} else {
+			listener->sock = socket(listener->address.ss_family, SOCK_DGRAM, 0);
+		}
 		if (listener->sock < 0) {
 			savederrno = errno;
 			err = -1;
@@ -73,7 +85,7 @@ int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 			goto exit_unlock;
 		}
 
-		value = KNET_RING_RCVBUFF;
+		value = KNET_RING_BUFF;
 		if (setsockopt(listener->sock, SOL_SOCKET, SO_RCVBUFFORCE, &value, sizeof(value)) < 0) {
 			savederrno = errno;
 			err = -1;
@@ -82,6 +94,44 @@ int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 			goto exit_unlock;
 		}
 
+		value = KNET_RING_BUFF;
+		if (setsockopt(listener->sock, SOL_SOCKET, SO_SNDBUFFORCE, &value, sizeof(value)) < 0) {
+			savederrno = errno;
+			err = -1;
+			log_err(knet_h, KNET_SUB_LISTENER, "Unable to set listener receive buffer: %s",
+				strerror(savederrno));
+			goto exit_unlock;
+		}
+
+		if (listener->tcp_udp) {
+			value = 1;
+			if (setsockopt(listener->sock, SOL_SOCKET, SO_KEEPALIVE, &value, sizeof(value)) < 0) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to set listener keepalive: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
+
+			value = 1;
+			if (setsockopt(listener->sock, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value)) < 0) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to set listener reuseaddr: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
+
+			value = 1;
+			if (setsockopt(listener->sock, SOL_SCTP, SCTP_NODELAY, &value, sizeof(value)) < 0) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to set TCP_CORK: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
+		}
+
 		value = 1;
 		if (setsockopt(listener->sock, SOL_IP, IP_FREEBIND, &value, sizeof(value)) <0) {
 			savederrno = errno;
@@ -137,24 +187,51 @@ int _listener_add(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 			goto exit_unlock;
 		}
 
-		memset(&ev, 0, sizeof(struct epoll_event));
+		if (listener->tcp_udp) {
+			if (listen(listener->sock, 1) < 0) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to listen on listener socket: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
 
-		ev.events = EPOLLIN;
-		ev.data.fd = listener->sock;
+			memset(&ev, 0, sizeof(struct epoll_event));
 
-		if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_ADD, listener->sock, &ev)) {
-			savederrno = errno;
-			err = -1;
-			log_err(knet_h, KNET_SUB_LISTENER, "Unable to add listener to epoll pool: %s",
-				strerror(savederrno));
-			goto exit_unlock;
+			ev.events = EPOLLIN;
+			ev.data.fd = listener->sock;
+
+			if (epoll_ctl(knet_h->tcp_links_server_epollfd, EPOLL_CTL_ADD, listener->sock, &ev)) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to add listener to epoll pool: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
+		} else {
+			memset(&ev, 0, sizeof(struct epoll_event));
+
+			ev.events = EPOLLIN;
+			ev.data.fd = listener->sock;
+
+			if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_ADD, listener->sock, &ev)) {
+				savederrno = errno;
+				err = -1;
+				log_err(knet_h, KNET_SUB_LISTENER, "Unable to add listener to epoll pool: %s",
+					strerror(savederrno));
+				goto exit_unlock;
+			}
 		}
 
 		/* pushing new host to the front */
 		listener->next		= knet_h->listener_head;
 		knet_h->listener_head	= listener;
 	}
+
 	lnk->listener_sock = listener->sock;
+	if (!listener->tcp_udp) {
+		lnk->outsock = listener->sock;
+	}
 
 exit_unlock:
 	if ((err) && (listener)) {
@@ -228,7 +305,11 @@ int _listener_remove(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 		}
 	}
 
-	epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_DEL, listener->sock, &ev);
+	if (listener->tcp_udp) {
+		epoll_ctl(knet_h->tcp_links_server_epollfd, EPOLL_CTL_DEL, listener->sock, &ev);
+	} else {
+		epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_DEL, listener->sock, &ev);
+	}
 	close(listener->sock);
 	free(listener);
 
@@ -240,7 +321,7 @@ int _listener_remove(knet_handle_t knet_h, uint16_t host_id, uint8_t link_id)
 }
 
 #if 0
-void socket_debug(knet_handle_t knet_h, int sockfd)
+void socket_debug(knet_handle_t knet_h, int logsub, int sockfd)
 {
 	struct sockaddr_storage sock;
 	char host[KNET_MAX_HOST_LEN];
@@ -252,14 +333,26 @@ void socket_debug(knet_handle_t knet_h, int sockfd)
 	memset(&port, 0, KNET_MAX_PORT_LEN);
 
 	if (getsockname(sockfd, (struct sockaddr *)&sock, &socklen) < 0) {
-		log_debug(knet_h, KNET_SUB_LINK_T, "Unable to getsockname: %s", strerror(errno));
+		log_debug(knet_h, logsub, "Unable to getsockname: %s", strerror(errno));
+	} else {
+		err = getnameinfo((const struct sockaddr *)&sock, sizeof(struct sockaddr_storage),
+				(char *)&host, KNET_MAX_HOST_LEN, (char *)&port, KNET_MAX_PORT_LEN, NI_NUMERICHOST | NI_NUMERICSERV);
+		if (err) {
+			log_debug(knet_h, logsub, "Unable to getnameinfo: %d", err);
+		} else {
+			log_debug(knet_h, logsub, "Sock host: %s port: %s", host, port);
+		}
+	}
+
+	if (getpeername(sockfd, (struct sockaddr *)&sock, &socklen) < 0) {
+		log_debug(knet_h, logsub, "Unable to getpeername: %s", strerror(errno));
 	} else {
 		err = getnameinfo((const struct sockaddr *)&sock, sizeof(struct sockaddr_storage),
 				(char *)&host, KNET_MAX_HOST_LEN, (char *)&port, KNET_MAX_PORT_LEN, NI_NUMERICHOST | NI_NUMERICSERV);
 		if (err) {
-			log_debug(knet_h, KNET_SUB_LINK_T, "Unable to getnameinfo: %d", err);
+			log_debug(knet_h, logsub, "Unable to getnameinfo: %d", err);
 		} else {
-			log_debug(knet_h, KNET_SUB_LINK_T, "Sock host: %s port: %s", host, port);
+			log_debug(knet_h, logsub, "Remote host: %s port: %s", host, port);
 		}
 	}
 
diff --git a/libknet/logging.c b/libknet/logging.c
index 49a9f90..81e1ae9 100644
--- a/libknet/logging.c
+++ b/libknet/logging.c
@@ -37,7 +37,7 @@ static struct pretty_names subsystem_names[] =
 	{ "hb_t", KNET_SUB_HB_T },
 	{ "switch_t", KNET_SUB_SWITCH_T },
 	{ "pmtud_t", KNET_SUB_PMTUD_T },
-	{ "filter", KNET_SUB_FILTER },
+	{ "tcp_t", KNET_SUB_TCP_LINK_T },
 	{ "crypto", KNET_SUB_CRYPTO },
 	{ "nsscrypto", KNET_SUB_NSSCRYPTO }
 };
diff --git a/libknet/onwire.h b/libknet/onwire.h
index a391966..748e220 100644
--- a/libknet/onwire.h
+++ b/libknet/onwire.h
@@ -113,8 +113,10 @@ struct knet_header_payload_ping {
 /* taken from tracepath6 */
 #define KNET_PMTUD_SIZE_V4 65535
 #define KNET_PMTUD_SIZE_V6 KNET_PMTUD_SIZE_V4
-#define KNET_PMTUD_OVERHEAD_V4 28
-#define KNET_PMTUD_OVERHEAD_V6 48
+#define KNET_PMTUD_IPV4_OVERHEAD 20
+#define KNET_PMTUD_IPV6_OVERHEAD 40
+#define KNET_PMTUD_UDP_OVERHEAD 8
+#define KNET_PMTUD_TCP_OVERHEAD 20
 #define KNET_PMTUD_MIN_MTU_V4 576
 #define KNET_PMTUD_MIN_MTU_V6 1280
 
diff --git a/libknet/ping_test.c b/libknet/ping_test.c
index 079fd91..8f60415 100644
--- a/libknet/ping_test.c
+++ b/libknet/ping_test.c
@@ -183,10 +183,13 @@ static void argv_to_hosts(int argc, char *argv[])
 			exit(EXIT_FAILURE);
 		}
 
-		knet_link_set_config(knet_h, node_id, 0, &src_addr, &dst_addr);
+		knet_link_set_config(knet_h, node_id, 0, KNET_LINK_TCP, &src_addr, &dst_addr);
+		//knet_link_set_config(knet_h, node_id, 1, KNET_LINK_UDP, &src_addr, &dst_addr);
 		knet_link_set_timeout(knet_h, node_id, 0, 1000, 5000, 2048);
+		//knet_link_set_timeout(knet_h, node_id, 1, 1000, 5000, 2048);
 		//knet_link_set_pong_count(knet_h, node_id, 0, 10);
 		knet_link_set_enable(knet_h, node_id, 0, 1);
+		//knet_link_set_enable(knet_h, node_id, 1, 1);
 	}
 }
 
diff --git a/libknet/threads_heartbeat.c b/libknet/threads_heartbeat.c
index 9182fd5..3e27432 100644
--- a/libknet/threads_heartbeat.c
+++ b/libknet/threads_heartbeat.c
@@ -58,7 +58,7 @@ static void _handle_check_each(knet_handle_t knet_h, struct knet_host *dst_host,
 			outbuf = knet_h->pingbuf_crypt;
 		}
 
-		len = sendto(dst_link->listener_sock, outbuf, outlen,
+		len = sendto(dst_link->outsock, outbuf, outlen,
 			MSG_DONTWAIT, (struct sockaddr *) &dst_link->dst_addr,
 			sizeof(struct sockaddr_storage));
 
@@ -67,7 +67,7 @@ static void _handle_check_each(knet_handle_t knet_h, struct knet_host *dst_host,
 		if (len != outlen) {
 			log_debug(knet_h, KNET_SUB_HB_T,
 				  "Unable to send ping (sock: %d) packet (sendto): %d %s\nrecorded src ip: %s src port: %s dst ip: %s dst port: %s",
-				  dst_link->listener_sock, errno, strerror(errno),
+				  dst_link->outsock, errno, strerror(errno),
 				  dst_link->status.src_ipaddr, dst_link->status.src_port,
 				  dst_link->status.dst_ipaddr, dst_link->status.dst_port);
 		}
@@ -105,12 +105,16 @@ void *_handle_heartbt_thread(void *data)
 			continue;
 		}
 
+		/* REVIEW this check for tcp connected from dynamic connections */
 		for (dst_host = knet_h->host_head; dst_host != NULL; dst_host = dst_host->next) {
 			for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) {
 				if ((dst_host->link[link_idx].status.enabled != 1) ||
+				    ((dst_host->link[link_idx].tcp_udp) &&
+				     (!dst_host->link[link_idx].outsock)) ||
 				    ((dst_host->link[link_idx].dynamic == KNET_LINK_DYNIP) &&
 				     (dst_host->link[link_idx].status.dynconnected != 1)))
 					continue;
+				//log_debug(knet_h, KNET_SUB_HB_T, "SENDING PING TO HOST: %u LINK: %u", dst_host->host_id, link_idx);
 				_handle_check_each(knet_h, dst_host, &dst_host->link[link_idx]);
 			}
 		}
diff --git a/libknet/threads_pmtud.c b/libknet/threads_pmtud.c
index 11a5464..240dfed 100644
--- a/libknet/threads_pmtud.c
+++ b/libknet/threads_pmtud.c
@@ -44,15 +44,23 @@ static void _handle_check_pmtud(knet_handle_t knet_h, struct knet_host *dst_host
 
 	knet_h->pmtudbuf->khp_pmtud_link = dst_link->link_id;
 
+	overhead_len = 0;
+
+	if (dst_link->tcp_udp) {
+		overhead_len = KNET_PMTUD_TCP_OVERHEAD;
+	} else {
+		overhead_len = KNET_PMTUD_UDP_OVERHEAD;
+	}
+
 	switch (dst_link->dst_addr.ss_family) {
 		case AF_INET6:
 			max_mtu_len = KNET_PMTUD_SIZE_V6;
-			overhead_len = KNET_PMTUD_OVERHEAD_V6;
+			overhead_len = overhead_len + KNET_PMTUD_IPV6_OVERHEAD;
 			dst_link->last_good_mtu = KNET_PMTUD_MIN_MTU_V6;
 			break;
 		case AF_INET:
 			max_mtu_len = KNET_PMTUD_SIZE_V4;
-			overhead_len = KNET_PMTUD_OVERHEAD_V4;
+			overhead_len = overhead_len + KNET_PMTUD_IPV6_OVERHEAD;
 			dst_link->last_good_mtu = KNET_PMTUD_MIN_MTU_V4;
 			break;
 		default:
@@ -140,7 +148,7 @@ restart:
 		return;
 	}
 
-	len = sendto(dst_link->listener_sock, outbuf, data_len,
+	len = sendto(dst_link->outsock, outbuf, data_len,
 			MSG_DONTWAIT, (struct sockaddr *) &dst_link->dst_addr,
 			sizeof(struct sockaddr_storage));
 	savederrno = errno;
@@ -320,8 +328,8 @@ timer_restart:
 				log_debug(knet_h, KNET_SUB_PMTUD_T, "PMTUD completed for host: %u link: %u current link mtu: %u iface mtu: %u",
 					  dst_host->host_id, link_idx, dst_link->status.mtu,
 					  (dst_link->dst_addr.ss_family == AF_INET) ?
-						dst_link->status.mtu + KNET_PMTUD_OVERHEAD_V4 :
-						dst_link->status.mtu + KNET_PMTUD_OVERHEAD_V6);
+						dst_link->status.mtu + KNET_PMTUD_IPV4_OVERHEAD :
+						dst_link->status.mtu + KNET_PMTUD_IPV6_OVERHEAD);
 				if (dst_link->status.mtu < min_mtu) {
 					min_mtu = dst_link->status.mtu;
 				}
diff --git a/libknet/threads_send_recv.c b/libknet/threads_send_recv.c
index 8ac2c7d..a4d028b 100644
--- a/libknet/threads_send_recv.c
+++ b/libknet/threads_send_recv.c
@@ -55,7 +55,7 @@ static void _dispatch_to_links(knet_handle_t knet_h, struct knet_host *dst_host,
 			msg_idx++;
 		}
 
-		if (sendmmsg(dst_host->link[dst_host->active_links[link_idx]].listener_sock,
+		if (sendmmsg(dst_host->link[dst_host->active_links[link_idx]].outsock,
 			     msg, msg_idx, MSG_NOSIGNAL) < 0) {	
 			log_debug(knet_h, KNET_SUB_SEND_T, "Unable to send data packet to host %s (%u) link %s:%s (%u): %s",
 				  dst_host->name, dst_host->host_id,
@@ -663,7 +663,7 @@ static int pckt_defrag(knet_handle_t knet_h, struct knet_header *inbuf, ssize_t
 	return 1;
 }
 
-static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage *address, int index, ssize_t len)
+static void _parse_recv_from_links(knet_handle_t knet_h, int sockfd, struct sockaddr_storage *address, int index, ssize_t len)
 {
 	ssize_t outlen;
 	struct knet_host *src_host;
@@ -730,6 +730,15 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 			}
 			src_link->status.dynconnected = 1;
 		}
+		if (src_link->tcp_udp) {
+			//log_debug(knet_h, KNET_SUB_LINK_T, "GOT PING FROM TCP SOCK: host: %u link: %u", inbuf->kh_node, src_link->link_id);
+			/* MUST CHECK A BUNCH THINGS HERE */
+			src_link->tcp_srv_sock = sockfd;
+			if (!src_link->outsock) {
+				log_debug(knet_h, KNET_SUB_LINK_T, "NO OUTSOCK YET!");
+				return;
+			}
+		}
 	}
 
 	switch (inbuf->kh_type) {
@@ -897,7 +906,7 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 			outbuf = knet_h->recv_from_links_buf_crypt;
 		}
 
-		sendto(src_link->listener_sock, outbuf, outlen, MSG_DONTWAIT,
+		sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT,
 				(struct sockaddr *) &src_link->dst_addr,
 				sizeof(struct sockaddr_storage));
 
@@ -947,7 +956,7 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 			outbuf = knet_h->recv_from_links_buf_crypt;
 		}
 
-		sendto(src_link->listener_sock, outbuf, outlen, MSG_DONTWAIT,
+		sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT,
 				(struct sockaddr *) &src_link->dst_addr,
 				sizeof(struct sockaddr_storage));
 
@@ -966,6 +975,8 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 	}
 }
 
+#include <stdlib.h>
+
 static void _handle_recv_from_links(knet_handle_t knet_h, int sockfd, struct mmsghdr *msg)
 {
 	int i, msg_recv;
@@ -976,13 +987,19 @@ static void _handle_recv_from_links(knet_handle_t knet_h, int sockfd, struct mms
 	}
 
 	msg_recv = recvmmsg(sockfd, msg, PCKT_FRAG_MAX, MSG_DONTWAIT, NULL);
-	if (msg_recv < 0) {
-		log_err(knet_h, KNET_SUB_LINK_T, "No message received from recvmmsg: %s", strerror(errno));
+	if ((msg_recv < 0) || (msg_recv == 255)) {
+		log_err(knet_h, KNET_SUB_LINK_T, "No message received (%d) from recvmmsg: %s", msg_recv, strerror(errno));
+		sleep(2);
+		abort();
 		goto exit_unlock;
 	}
 
 	for (i = 0; i < msg_recv; i++) {
-		_parse_recv_from_links(knet_h, (struct sockaddr_storage *)&msg[i].msg_hdr.msg_name, i, msg[i].msg_len);
+		if (msg[i].msg_len <= 0) {
+			// close socket, remove from epoll and clear the link/listener?
+		} else {
+			_parse_recv_from_links(knet_h, sockfd, (struct sockaddr_storage *)&msg[i].msg_hdr.msg_name, i, msg[i].msg_len);
+		}
 	}
 
 exit_unlock:
diff --git a/libknet/threads_tcp_links.c b/libknet/threads_tcp_links.c
new file mode 100644
index 0000000..09de029
--- /dev/null
+++ b/libknet/threads_tcp_links.c
@@ -0,0 +1,233 @@
+/*
+ * Copyright (C) 2010-2015 Red Hat, Inc.  All rights reserved.
+ *
+ * Authors: Fabio M. Di Nitto <fabbione@kronosnet.org>
+ *
+ * This software licensed under GPL-2.0+, LGPL-2.0+
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <pthread.h>
+#include <sys/epoll.h>
+#include <netinet/tcp.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/sctp.h>
+
+#include "link.h"
+#include "common.h"
+#include "logging.h"
+#include "threads_common.h"
+#include "threads_tcp_links.h"
+
+#define KNET_TCP_TIMERES 200000
+#define KNET_RING_BUFF 8388608
+
+static void _handle_outgoing_tcp(knet_handle_t knet_h)
+{
+	struct knet_host *dst_host;
+	int link_idx;
+	int value;
+
+	//log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread doing nothing, timeout expired, checking outgoing");
+
+	for (dst_host = knet_h->host_head; dst_host != NULL; dst_host = dst_host->next) {
+		for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) {
+			if ((dst_host->link[link_idx].status.enabled != 1) ||
+			    (dst_host->link[link_idx].dynamic == KNET_LINK_DYNIP) ||
+			    (dst_host->link[link_idx].tcp_udp != KNET_LINK_TCP) ||
+			    (dst_host->link[link_idx].tcp_clt_sock)) {
+				continue;
+			}
+			log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: host %u link: %u does qualify for connect",
+				  dst_host->host_id, link_idx);
+
+			log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: attempting connect");
+
+			dst_host->link[link_idx].tcp_clt_sock = socket(dst_host->link[link_idx].dst_addr.ss_family, SOCK_STREAM, IPPROTO_SCTP);
+
+			if (dst_host->link[link_idx].tcp_clt_sock < 0) {
+				log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: unable to create client socket: %s",
+					  strerror(errno));
+				goto exit_err;
+			}
+
+			value = KNET_RING_BUFF;
+			if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_SOCKET, SO_RCVBUFFORCE, &value, sizeof(value)) < 0) {
+				log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set tcp sock receive buffer: %s",
+					strerror(errno));
+				goto exit_err;
+			}
+
+			value = KNET_RING_BUFF;
+			if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_SOCKET, SO_SNDBUFFORCE, &value, sizeof(value)) < 0) {
+				log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set tcp sock receive buffer: %s",
+					strerror(errno));
+				goto exit_err;
+			}
+
+			value = 1;
+			if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_SOCKET, SO_KEEPALIVE, &value, sizeof(value)) < 0) {
+				log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set tcp sock keepalive: %s",
+					strerror(errno));
+				goto exit_err;
+			}
+
+			value = 1;
+			if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_SCTP, SCTP_NODELAY, &value, sizeof(value)) < 0) {
+				log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set sctp nodelay: %s",
+					strerror(errno));
+				goto exit_err;
+			}
+
+			value = 1;
+			if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value)) < 0) {
+				log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set tcp sock reuseaddr: %s",
+					strerror(errno));
+				goto exit_err;
+			}
+
+			if (dst_host->link[link_idx].dst_addr.ss_family == AF_INET6) {
+				value = 1;
+				if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, IPPROTO_IPV6, IPV6_V6ONLY,
+					       &value, sizeof(value)) < 0) {
+					log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set tcp sock IPv6 only: %s",
+						strerror(errno));
+					goto exit_err;
+				}
+
+				value = IPV6_PMTUDISC_PROBE;
+				if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_IPV6, IPV6_MTU_DISCOVER, &value, sizeof(value)) <0) {
+					log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set PMTUDISC on tcp sock socket: %s",
+						strerror(errno));
+					goto exit_err;
+				}
+			} else {
+				value = IP_PMTUDISC_PROBE;
+				if (setsockopt(dst_host->link[link_idx].tcp_clt_sock, SOL_IP, IP_MTU_DISCOVER, &value, sizeof(value)) <0) {
+					log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to set PMTUDISC on tcp sock socket: %s",
+						strerror(errno));
+					goto exit_err;
+				}
+			}
+
+			if (_fdset_cloexec(dst_host->link[link_idx].tcp_clt_sock)) {
+				log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: unable to set cloexec on client socket: %s",
+					  strerror(errno));
+				goto exit_err;
+			}
+
+			if (connect(dst_host->link[link_idx].tcp_clt_sock,
+				    (struct sockaddr *)&dst_host->link[link_idx].dst_addr,
+				    sizeof(struct sockaddr_storage))) {
+				log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: connection failed: %s",
+					  strerror(errno));
+				goto exit_err;
+			}
+
+			dst_host->link[link_idx].outsock = dst_host->link[link_idx].tcp_clt_sock;
+
+			log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread: connection completed!");
+			continue;
+exit_err:
+			if (dst_host->link[link_idx].tcp_clt_sock > 0) {
+				close(dst_host->link[link_idx].tcp_clt_sock);
+			}
+			dst_host->link[link_idx].tcp_clt_sock = 0;
+			continue;
+		}
+	}
+}
+
+void *_handle_tcp_links_client_thread(void *data)
+{
+	knet_handle_t knet_h = (knet_handle_t) data;
+
+	/*
+	 * this one needs some serious locking work
+	 */
+	while (!knet_h->fini_in_progress) {
+		usleep(KNET_TCP_TIMERES);
+		_handle_outgoing_tcp(knet_h);
+	}
+	return NULL;
+}
+
+static void _handle_incoming_tcp(knet_handle_t knet_h, int sock)
+{
+	int newsockfd, err = 0;
+	struct sockaddr_storage newsock;
+	socklen_t newsocklen = sizeof(struct sockaddr_storage);
+	struct epoll_event ev;
+
+	log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread INCOMING on fd: %d", sock);
+
+	newsockfd = accept(sock, (struct sockaddr *)&newsock, &newsocklen);
+
+	if (newsockfd < 0) {
+		log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread INCOMING error accepting connection: %s", strerror(errno));
+		return;
+	}
+
+	err = _fdset_cloexec(newsockfd);
+	if (err) {
+		log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler thread INCOMING unable to set socket opts: %s", strerror(errno));
+		goto exit_err;
+	}
+
+//	err = pthread_rwlock_wrlock(&knet_h->listener_rwlock);
+//	if (err) {
+//		log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to get write lock: %s",
+//			strerror(err));
+//		goto exit_err;
+//	}
+
+	memset(&ev, 0, sizeof(struct epoll_event));
+
+	ev.events = EPOLLIN;
+	ev.data.fd = newsockfd;
+
+	if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_ADD, newsockfd, &ev)) {
+		log_err(knet_h, KNET_SUB_TCP_LINK_T, "Unable to add incoming tcp connection to epoll poll: %s",
+			strerror(errno));
+		err = -1;
+	}
+
+	// need to track those pending connections somewhere?
+
+//	pthread_rwlock_unlock(&knet_h->listener_rwlock);
+exit_err:
+	if (err) {
+		close(newsockfd);
+	}
+	return;
+}
+
+void *_handle_tcp_links_server_thread(void *data)
+{
+	int i, nev;
+	knet_handle_t knet_h = (knet_handle_t) data;
+	struct epoll_event events[KNET_EPOLL_MAX_EVENTS];
+
+	while (!knet_h->fini_in_progress) {
+		nev = epoll_wait(knet_h->tcp_links_server_epollfd, events, KNET_EPOLL_MAX_EVENTS, -1);
+
+		if (knet_h->fini_in_progress) {
+			break;
+		}
+
+		if (nev < 0) {
+			log_debug(knet_h, KNET_SUB_TCP_LINK_T, "TCP handler EPOLL ERROR: %s", strerror(errno));
+			continue;
+		}
+
+		for (i = 0; i < nev; i++) {
+			_handle_incoming_tcp(knet_h, events[i].data.fd);
+		}
+	}
+	return NULL;
+}
diff --git a/libknet/threads_tcp_links.h b/libknet/threads_tcp_links.h
new file mode 100644
index 0000000..b97ebac
--- /dev/null
+++ b/libknet/threads_tcp_links.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2010-2015 Red Hat, Inc.  All rights reserved.
+ *
+ * Authors: Fabio M. Di Nitto <fabbione@kronosnet.org>
+ *
+ * This software licensed under GPL-2.0+, LGPL-2.0+
+ */
+
+#ifndef __THREADS_TCP_LINKS_H__
+#define __THREADS_TCP_LINKS_H__
+
+void *_handle_tcp_links_server_thread(void *data);
+void *_handle_tcp_links_client_thread(void *data);
+
+#endif
