diff --git a/kronosnetd/etherfilter.c b/kronosnetd/etherfilter.c
index 9a5969a..de6353c 100644
--- a/kronosnetd/etherfilter.c
+++ b/kronosnetd/etherfilter.c
@@ -39,6 +39,7 @@ int ether_host_filter_fn (void *private_data,
 			  uint8_t tx_rx,
 			  uint16_t this_host_id,
 			  uint16_t src_host_id,
+			  int8_t *channel,
 			  uint16_t *dst_host_ids,
 			  size_t *dst_host_ids_entries)
 {
diff --git a/kronosnetd/etherfilter.h b/kronosnetd/etherfilter.h
index 46bfe9b..e12f20e 100644
--- a/kronosnetd/etherfilter.h
+++ b/kronosnetd/etherfilter.h
@@ -17,6 +17,7 @@ int ether_host_filter_fn (void *private_data,
 			  uint8_t tx_rx,
 			  uint16_t this_host_id,
 			  uint16_t src_host_id,
+			  int8_t *channel,
 			  uint16_t *dst_host_ids,
 			  size_t *dst_host_ids_entries);
 
diff --git a/kronosnetd/vty_cli_cmds.c b/kronosnetd/vty_cli_cmds.c
index d697e90..670a563 100644
--- a/kronosnetd/vty_cli_cmds.c
+++ b/kronosnetd/vty_cli_cmds.c
@@ -1566,6 +1566,7 @@ static int knet_cmd_interface(struct knet_vty *vty)
 	char device[IFNAMSIZ];
 	char mac[18];
 	struct knet_cfg *knet_iface = NULL;
+	int8_t channel = 0;
 
 	get_param(vty, 1, &param, &paramlen, &paramoffset);
 	param_to_str(device, IFNAMSIZ, param, paramlen);
@@ -1613,7 +1614,7 @@ tap_found:
 
 	tapfd = tap_get_fd(knet_iface->cfg_eth.tap);
 
-	knet_iface->cfg_ring.knet_h = knet_handle_new(requested_id, &tapfd, vty->logfd, vty->loglevel);
+	knet_iface->cfg_ring.knet_h = knet_handle_new(requested_id, vty->logfd, vty->loglevel);
 	if (!knet_iface->cfg_ring.knet_h) {
 		knet_vty_write(vty, "Error: Unable to create ring handle for device %s%s",
 				device, telnet_newline);
@@ -1621,6 +1622,13 @@ tap_found:
 		goto out_clean;
 	}
 
+	if (knet_handle_add_datafd(knet_iface->cfg_ring.knet_h, &tapfd, &channel) < 0) {
+		knet_vty_write(vty, "Error: Unable to add tapfd to knet_handle %s%s",
+				strerror(errno), telnet_newline);
+		err = -1;
+		goto out_clean;
+	}
+
 	knet_handle_enable_filter(knet_iface->cfg_ring.knet_h, NULL, ether_host_filter_fn);
 
 	if (knet_handle_enable_pmtud_notify(knet_iface->cfg_ring.knet_h,
diff --git a/libknet/handle.c b/libknet/handle.c
index 82472e5..f177a08 100644
--- a/libknet/handle.c
+++ b/libknet/handle.c
@@ -178,30 +178,10 @@ static void _close_socketpair(knet_handle_t knet_h, int (*sock)[2])
 	}
 }
 
-static int _init_socks(knet_handle_t knet_h, int *datafd)
+static int _init_socks(knet_handle_t knet_h)
 {
 	int savederrno = 0;
 
-	knet_h->is_socket = 1;
-	if (*datafd == 0) {
-		if (_init_socketpair(knet_h, &knet_h->sockpair)) {
-			savederrno = errno;
-			log_err(knet_h, KNET_SUB_HANDLE, "Unable to initialize data sockpair: %s",
-				strerror(savederrno));
-			goto exit_fail;
-		}
-		knet_h->sockfd = knet_h->sockpair[0];
-		*datafd = knet_h->sockpair[1];
-	} else {
-		int sockopt;
-		socklen_t sockoptlen = sizeof(sockopt);
-
-		knet_h->sockfd = *datafd;
-		if (getsockopt(knet_h->sockfd, SOL_SOCKET, SO_TYPE, &sockopt, &sockoptlen) < 0) {
-			knet_h->is_socket = 0;
-		}
-	}
-
 	if (_init_socketpair(knet_h, &knet_h->hostsockfd)) {
 		savederrno = errno;
 		log_err(knet_h, KNET_SUB_HANDLE, "Unable to initialize internal hostsockpar: %s",
@@ -225,7 +205,6 @@ exit_fail:
 
 static void _close_socks(knet_handle_t knet_h)
 {
-	_close_socketpair(knet_h, &knet_h->sockpair);
 	_close_socketpair(knet_h, &knet_h->dstsockfd);
 	_close_socketpair(knet_h, &knet_h->hostsockfd);
 }
@@ -362,7 +341,11 @@ static int _init_epolls(knet_handle_t knet_h)
 	struct epoll_event ev;
 	int savederrno = 0;
 
-	knet_h->send_to_links_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS);
+	/*
+	 * even if the kernel does dynamic allocation with epoll_ctl
+	 * we need to reserve one extra for host to host communication
+	 */
+	knet_h->send_to_links_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS + 1);
 	if (knet_h->send_to_links_epollfd < 0) {
 		savederrno = errno;
 		log_err(knet_h, KNET_SUB_HANDLE, "Unable to create epoll datafd to link fd: %s",
@@ -399,7 +382,7 @@ static int _init_epolls(knet_handle_t knet_h)
 			strerror(savederrno)); 
 		goto exit_fail;
 	}
-		
+
 	if (_fdset_cloexec(knet_h->dst_link_handler_epollfd)) {
 		savederrno = errno;
 		log_err(knet_h, KNET_SUB_HANDLE, "Unable to set CLOEXEC on dst cache epoll fd: %s",
@@ -409,18 +392,6 @@ static int _init_epolls(knet_handle_t knet_h)
 
 	memset(&ev, 0, sizeof(struct epoll_event));
 	ev.events = EPOLLIN;
-	ev.data.fd = knet_h->sockfd;
-
-	if (epoll_ctl(knet_h->send_to_links_epollfd,
-		      EPOLL_CTL_ADD, knet_h->sockfd, &ev)) {
-		savederrno = errno;
-		log_err(knet_h, KNET_SUB_HANDLE, "Unable to add datafd to link fd to epoll pool: %s",
-			strerror(savederrno));
-		goto exit_fail;
-	}
-
-	memset(&ev, 0, sizeof(struct epoll_event));
-	ev.events = EPOLLIN;
 	ev.data.fd = knet_h->hostsockfd[0];
 
 	if (epoll_ctl(knet_h->send_to_links_epollfd,
@@ -452,8 +423,19 @@ exit_fail:
 static void _close_epolls(knet_handle_t knet_h)
 {
 	struct epoll_event ev;
+	int i;
+
+	memset(&ev, 0, sizeof(struct epoll_event));
+
+	for (i = 0; i < KNET_DATAFD_MAX; i++) {
+		if (knet_h->sockfd[i].in_use) {
+			epoll_ctl(knet_h->send_to_links_epollfd, EPOLL_CTL_DEL, knet_h->sockfd[i].sockfd[knet_h->sockfd[i].is_created], &ev);
+			if  (knet_h->sockfd[i].sockfd[knet_h->sockfd[i].is_created]) {
+				 _close_socketpair(knet_h, &knet_h->sockfd[i].sockfd);
+			}
+		}
+	}
 
-	epoll_ctl(knet_h->send_to_links_epollfd, EPOLL_CTL_DEL, knet_h->sockfd, &ev);
 	epoll_ctl(knet_h->send_to_links_epollfd, EPOLL_CTL_DEL, knet_h->hostsockfd[0], &ev);
 	epoll_ctl(knet_h->dst_link_handler_epollfd, EPOLL_CTL_DEL, knet_h->dstsockfd[0], &ev);
 	close(knet_h->send_to_links_epollfd);
@@ -544,7 +526,6 @@ static void _stop_threads(knet_handle_t knet_h)
 }
 
 knet_handle_t knet_handle_new(uint16_t host_id,
-			      int      *datafd,
 			      int      log_fd,
 			      uint8_t  default_log_level)
 {
@@ -555,16 +536,6 @@ knet_handle_t knet_handle_new(uint16_t host_id,
 	 * validate incoming request
 	 */
 
-	if (datafd == NULL) {
-		errno = EINVAL;
-		return NULL;
-	}
-
-	if (*datafd < 0) {
-		errno = EINVAL;
-		return NULL;
-	}
-
 	if ((log_fd > 0) && (default_log_level > KNET_LOG_DEBUG)) {
 		errno = EINVAL;
 		return NULL;
@@ -609,7 +580,7 @@ knet_handle_t knet_handle_new(uint16_t host_id,
 	 * init sockets
 	 */
 
-	if (_init_socks(knet_h, datafd)) {
+	if (_init_socks(knet_h)) {
 		savederrno = errno;
 		goto exit_fail;
 	}
@@ -720,6 +691,223 @@ exit_nolock:
 	return 0;
 }
 
+int knet_handle_add_datafd(knet_handle_t knet_h, int *datafd, int8_t *channel)
+{
+	int err = 0, savederrno = 0;
+	int i;
+	struct epoll_event ev;
+
+	if (!knet_h) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (datafd == NULL) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (channel == NULL) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (*channel >= KNET_DATAFD_MAX) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	savederrno = pthread_rwlock_wrlock(&knet_h->list_rwlock);
+	if (savederrno) {
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s",
+			strerror(savederrno));
+		errno = savederrno;
+		return -1;
+	}
+
+	if (*datafd > 0) {
+		for (i = 0; i < KNET_DATAFD_MAX; i++) {
+			if  ((knet_h->sockfd[i].in_use) && (knet_h->sockfd[i].sockfd[0] == *datafd)) {
+				log_err(knet_h, KNET_SUB_HANDLE, "requested datafd: %d already exist in index: %d", *datafd, i);
+				savederrno = EEXIST;
+				err = -1;
+				goto out_unlock;
+			}
+		}
+	}
+
+	/*
+	 * auto allocate a channel
+	 */
+	if (*channel < 0) {
+		for (i = 0; i < KNET_DATAFD_MAX; i++) {
+			if (!knet_h->sockfd[i].in_use) {
+				*channel = i;
+				break;
+			}
+		}
+		if (*channel < 0) {
+			savederrno = EBUSY;
+			err = -1;
+			goto out_unlock;
+		}
+	} else {
+		if (knet_h->sockfd[*channel].in_use) {
+			savederrno = EBUSY;
+			err = -1;
+			goto out_unlock;
+		}
+	}
+
+	knet_h->sockfd[*channel].is_created = 0;
+	knet_h->sockfd[*channel].is_socket = 0;
+
+	if (*datafd > 0) {
+		int sockopt;
+		socklen_t sockoptlen = sizeof(sockopt);
+
+		knet_h->sockfd[*channel].sockfd[0] = *datafd;
+		knet_h->sockfd[*channel].sockfd[1] = 0;
+
+		if (!getsockopt(knet_h->sockfd[*channel].sockfd[0], SOL_SOCKET, SO_TYPE, &sockopt, &sockoptlen)) {
+			knet_h->sockfd[*channel].is_socket = 1;
+		}
+	} else {
+		if (_init_socketpair(knet_h, &knet_h->sockfd[*channel].sockfd)) {
+			savederrno = errno;
+			err = -1;
+			goto out_unlock;
+		}
+
+		knet_h->sockfd[*channel].is_created = 1;
+		knet_h->sockfd[*channel].is_socket = 1;
+		*datafd = knet_h->sockfd[*channel].sockfd[0];
+	}
+
+	memset(&ev, 0, sizeof(struct epoll_event));
+	ev.events = EPOLLIN;
+	ev.data.fd = knet_h->sockfd[*channel].sockfd[knet_h->sockfd[*channel].is_created];
+
+	if (epoll_ctl(knet_h->send_to_links_epollfd,
+		      EPOLL_CTL_ADD, knet_h->sockfd[*channel].sockfd[knet_h->sockfd[*channel].is_created], &ev)) {
+		savederrno = errno;
+		err = -1;
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to add datafd %d to linkfd epoll pool: %s",
+			knet_h->sockfd[*channel].sockfd[knet_h->sockfd[*channel].is_created], strerror(savederrno));
+		if (knet_h->sockfd[*channel].is_created) {
+			_close_socketpair(knet_h, &knet_h->sockfd[*channel].sockfd);
+		}
+		goto out_unlock;
+	}
+
+	knet_h->sockfd[*channel].in_use = 1;
+
+out_unlock:
+	pthread_rwlock_unlock(&knet_h->list_rwlock);
+	errno = savederrno;
+	return err;
+}
+
+int knet_handle_remove_datafd(knet_handle_t knet_h, int datafd)
+{
+	int err = 0, savederrno = 0;
+	int8_t channel = -1;
+	int i;
+	struct epoll_event ev;
+
+	if (!knet_h) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (datafd <= 0) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	savederrno = pthread_rwlock_wrlock(&knet_h->list_rwlock);
+	if (savederrno) {
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s",
+			strerror(savederrno));
+		errno = savederrno;
+		return -1;
+	}
+
+	for (i = 0; i < KNET_DATAFD_MAX; i++) {
+		if  ((knet_h->sockfd[i].in_use) &&
+		     (knet_h->sockfd[i].sockfd[0] == datafd)) {
+			channel = i;
+			break;
+		}
+	}
+
+	if (channel < 0) {
+		savederrno = EINVAL;
+		err = -1;
+		goto out_unlock;
+	}
+
+	memset(&ev, 0, sizeof(struct epoll_event));
+
+	if (epoll_ctl(knet_h->send_to_links_epollfd,
+		      EPOLL_CTL_DEL, knet_h->sockfd[channel].sockfd[knet_h->sockfd[i].is_created], &ev)) {
+		savederrno = errno;
+		err = -1;
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to del datafd %d from linkfd epoll pool: %s",
+			knet_h->sockfd[channel].sockfd[knet_h->sockfd[i].is_created], strerror(savederrno));
+		goto out_unlock;
+	}
+
+	if (knet_h->sockfd[channel].is_created) {
+		_close_socketpair(knet_h, &knet_h->sockfd[channel].sockfd);
+	}
+
+	memset(&knet_h->sockfd[channel], 0, sizeof(struct knet_sock));
+
+out_unlock:
+	pthread_rwlock_unlock(&knet_h->list_rwlock);
+	errno = savederrno;
+	return err;
+}
+
+int knet_handle_enable_sock_notify(knet_handle_t knet_h,
+				   void *sock_notify_fn_private_data,
+				   void (*sock_notify_fn) (
+						void *private_data,
+						int datafd,
+						int8_t channel,
+						int error,
+						int errorno))
+{
+	int savederrno = 0;
+
+	if (!knet_h) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	savederrno = pthread_rwlock_wrlock(&knet_h->list_rwlock);
+	if (savederrno) {
+		log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s",
+			strerror(savederrno));
+		errno = savederrno;
+		return -1;
+	}
+
+	knet_h->sock_notify_fn_private_data = sock_notify_fn_private_data;
+	knet_h->sock_notify_fn = sock_notify_fn;
+	if (knet_h->sock_notify_fn) {
+		log_debug(knet_h, KNET_SUB_HANDLE, "sock_notify_fn enabled");
+	} else {
+		log_debug(knet_h, KNET_SUB_HANDLE, "sock_notify_fn disabled");
+	}
+
+	pthread_rwlock_unlock(&knet_h->list_rwlock);
+
+	return 0;
+}
+
+
 int knet_handle_enable_filter(knet_handle_t knet_h,
 			      void *dst_host_filter_fn_private_data,
 			      int (*dst_host_filter_fn) (
@@ -729,6 +917,7 @@ int knet_handle_enable_filter(knet_handle_t knet_h,
 					uint8_t tx_rx,
 					uint16_t this_host_id,
 					uint16_t src_node_id,
+					int8_t *channel,
 					uint16_t *dst_host_ids,
 					size_t *dst_host_ids_entries))
 {
@@ -953,10 +1142,9 @@ exit_unlock:
 	return err;
 }
 
-ssize_t knet_recv(knet_handle_t knet_h, char *buff, const size_t buff_len)
+ssize_t knet_recv(knet_handle_t knet_h, char *buff, const size_t buff_len, const int8_t channel)
 {
 	struct iovec iov_in;
-	int sock;
 
 	if (!knet_h) {
 		errno = EINVAL;
@@ -968,26 +1156,26 @@ ssize_t knet_recv(knet_handle_t knet_h, char *buff, const size_t buff_len)
 		return -1;
 	}
 
+	if (channel >= KNET_DATAFD_MAX) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (!knet_h->sockfd[channel].in_use) {
+		errno = EINVAL;
+		return -1;
+	}
+
 	memset(&iov_in, 0, sizeof(iov_in));
 	iov_in.iov_base = (void *)buff;
 	iov_in.iov_len = buff_len;
 
-	/*
-	 * workaround magic of the socketpair
-	 */
-	if (knet_h->sockpair[1]) {
-		sock = knet_h->sockpair[1];
-	} else {
-		sock = knet_h->sockfd;
-	}
-
-	return readv(sock, &iov_in, 1);
+	return readv(knet_h->sockfd[channel].sockfd[0], &iov_in, 1);
 }
 
-ssize_t knet_send(knet_handle_t knet_h, const char *buff, const size_t buff_len)
+ssize_t knet_send(knet_handle_t knet_h, const char *buff, const size_t buff_len, const int8_t channel)
 {
 	struct iovec iov_out[1];
-	int sock;
 
 	if ((!knet_h) ||
 	    (buff == NULL) ||
@@ -996,16 +1184,20 @@ ssize_t knet_send(knet_handle_t knet_h, const char *buff, const size_t buff_len)
 		return -1;
 	}
 
+	if (channel >= KNET_DATAFD_MAX) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (!knet_h->sockfd[channel].in_use) {
+		errno = EINVAL;
+		return -1;
+	}
+
 	memset(iov_out, 0, sizeof(iov_out));
 
 	iov_out[0].iov_base = (void *)buff;
 	iov_out[0].iov_len = buff_len;
 
-	if (knet_h->sockpair[1]) {
-		sock = knet_h->sockpair[1];
-	} else {
-		sock = knet_h->sockfd;
-	}
-
-	return writev(sock, iov_out, 1);
+	return writev(knet_h->sockfd[channel].sockfd[0], iov_out, 1);
 }
diff --git a/libknet/internals.h b/libknet/internals.h
index 68de4a4..879c4a1 100644
--- a/libknet/internals.h
+++ b/libknet/internals.h
@@ -23,6 +23,8 @@
 
 #define PCKT_FRAG_MAX UINT8_MAX
 
+#define KNET_EPOLL_MAX_EVENTS KNET_DATAFD_MAX
+
 struct knet_listener {
 	int sock;
 	struct sockaddr_storage address;
@@ -99,12 +101,18 @@ struct knet_host {
 	struct knet_host *next;
 };
 
+struct knet_sock {
+	int sockfd[2];   /* sockfd[0] will always be application facing
+			  * and sockfd[1] internal if sockpair has been created by knet */
+	int is_socket;   /* check if it's a socket for recvmmsg usage */
+	int is_created;  /* knet created this socket and has to clean up on exit/del */
+	int in_use;      /* set to 1 if it's use, 0 if free */
+};
+
 struct knet_handle {
 	uint16_t host_id;
 	unsigned int enabled:1;
-	int sockfd;
-	int sockpair[2];
-	int is_socket;
+	struct knet_sock sockfd[KNET_DATAFD_MAX];
 	int logfd;
 	uint8_t log_levels[KNET_MAX_SUBSYSTEMS];
 	int hostsockfd[2];
@@ -162,6 +170,7 @@ struct knet_handle {
 		uint8_t tx_rx,
 		uint16_t this_host_id,
 		uint16_t src_node_id,
+		int8_t *channel,
 		uint16_t *dst_host_ids,
 		size_t *dst_host_ids_entries);
 	void *pmtud_notify_fn_private_data;
@@ -176,6 +185,13 @@ struct knet_handle {
 		uint8_t reachable,
 		uint8_t remote,
 		uint8_t external);
+	void *sock_notify_fn_private_data;
+	void (*sock_notify_fn) (
+		void *private_data,
+		int datafd,
+		int8_t channel,
+		int error,
+		int errorno);
 	int fini_in_progress;
 };
 
diff --git a/libknet/khandle_test.c b/libknet/khandle_test.c
index 3b7eda1..df1fc5d 100644
--- a/libknet/khandle_test.c
+++ b/libknet/khandle_test.c
@@ -19,10 +19,10 @@
 
 int main(int argc, char *argv[])
 {
-	int sock = 0, i;
+	int i;
 	knet_handle_t knet_h;
 
-	knet_h = knet_handle_new(1, &sock, 0, 0);
+	knet_h = knet_handle_new(1, 0, 0);
 
 	for (i = 0; i < KNET_MAX_HOST; i++) {
 		printf("add host: %d\n", i);
diff --git a/libknet/libknet.h b/libknet/libknet.h
index ef3967a..11b92d9 100644
--- a/libknet/libknet.h
+++ b/libknet/libknet.h
@@ -59,26 +59,6 @@ typedef struct knet_handle *knet_handle_t;
  *            It is user responsibility to check that the value
  *            is unique, or bad might happen.
  *
- * datafd   - read/write file descriptor.
- *            knet will read data here to send to the other hosts
- *            and will write data received from the network.
- *            Each data packet can be of max size KNET_MAX_PACKET_SIZE!
- *            Applications using knet_send/knet_recv will receive
- *            proper error in case of packet size is not within boundaries.
- *            Applications using their own functions to write to the
- *            datafd should NOT write more than KNET_MAX_PACKET_SIZE
- *            or the packet will be fragmented by readv inside the
- *            delivery thread. Be aware that while you can write
- *            more than KNET_MAX_PACKET_SIZE, that can break any
- *            custom delivery filters (see below) due to data offset.
- *            Please refer to ping_test.c on how to setup socketpair.
- *            datafd can be 0, and knet_handle_new will create a properly
- *            populated socket pair the same way as ping_test, or a value
- *            higher than 0. Negative number will return error.
- *            knet_handle_new will take care to cleanup the socketpair
- *            only if it's been created by knet_handle_new, when calling
- *            knet_handle_free.
- *
  * log_fd   - write file descriptor. If set to a value > 0, it will be used
  *            to write log packets (see below) from libknet to the application.
  *            Set to 0 will disable logging from libknet.
@@ -97,7 +77,6 @@ typedef struct knet_handle *knet_handle_t;
  */
 
 knet_handle_t knet_handle_new(uint16_t host_id,
-			      int      *datafd,
 			      int      log_fd,
 			      uint8_t  default_log_level);
 
@@ -117,6 +96,117 @@ knet_handle_t knet_handle_new(uint16_t host_id,
 int knet_handle_free(knet_handle_t knet_h);
 
 /*
+ * knet_handle_add_datafd
+ *
+ * knet_h   - pointer to knet_handle_t
+ *
+ * *datafd  - read/write file descriptor.
+ *            knet will read data here to send to the other hosts
+ *            and will write data received from the network.
+ *            Each data packet can be of max size KNET_MAX_PACKET_SIZE!
+ *            Applications using knet_send/knet_recv will receive
+ *            proper error in case of packet size is not within boundaries.
+ *            Applications using their own functions to write to the
+ *            datafd should NOT write more than KNET_MAX_PACKET_SIZE.
+ *
+ *            Please refer to handle.c on how to setup socketpair.
+ *
+ *            datafd can be 0, and knet_handle_new will create a properly
+ *            populated socket pair the same way as ping_test, or a value
+ *            higher than 0. Negative number will return error.
+ *            on exit knet_handle_free will take care to cleanup the
+ *            socketpair only if they have been created by knet_handle_add_datafd.
+ *
+ *            It is possible to pass either sockets or normal fds.
+ *
+ * *channel - This value has the same effect of VLAN tagging.
+ *            A negative value will auto allocate a channel.
+ *            Setting a value between 0 and 31 will try to allocate that
+ *            specific channel (unless alredy in use).
+ *
+ *            It is possible to add up to 32 datafd but be aware that each
+ *            one of them must have a receiveing end on the other host.
+ *            Example:
+ *            hostA channel 0 will be delivered to datafd on hostB channel 0
+ *            hostA channel 1 to hostB channel 1.
+ *
+ *            Each channel must have a unique filedescriptor.
+ *
+ *            If your application could have 2 channels on one host and one
+ *            channel on another host, then you can use dst_host_filter
+ *            to manipulate channel values on TX and RX.
+ *
+ * knet_handle_add_datafd returns:
+ *
+ * 0 on success
+ *   *datafd  will be populated with a socket if the original value was 0
+ *            or if a specific fd was set, the value is untouched
+ *   *channel will be populated with a channel number if the original value
+ *            was negative or the value is untouched if a specific channel
+ *            was requested.
+ *
+ * -1 on error and errno is set.
+ *   *datafd and *channel are untouched or empty.
+ */
+
+#define KNET_DATAFD_MAX 32
+
+int knet_handle_add_datafd(knet_handle_t knet_h, int *datafd, int8_t *channel);
+
+/*
+ * knet_handle_remove_datafd
+ *
+ * knet_h   - pointer to knet_handle_t
+ *
+ * datafd   - file descriptor to remove.
+ *            NOTE that if the socket/fd was crated by knet_handle_datafd,
+ *                 the socket will be closed by libknet.
+ *
+ * knet_handle_remove_datafd returns:
+ *
+ * 0 on success
+ *
+ * -1 on error and errno is set.
+ */
+
+int knet_handle_remove_datafd(knet_handle_t knet_h, int datafd);
+
+/*
+ * knet_handle_enable_sock_notify
+ * 
+ * knet_h   - pointer to knet_handle_t
+ *
+ * sock_notify_fn_private_data
+ *            void pointer to data that can be used to identify
+ *            the callback.
+ *
+ * sock_notify_fn
+ *            is a callback function that is invoked every time
+ *            a socket in the datafd pool will report an error (-1)
+ *            or an end of read (0) (see socket.7).
+ *            This function MUST NEVER block or add substantial delays.
+ *
+ * knet_handle_enable_sock_notify returns:
+ *
+ * 0 on success
+ * -1 on error and errno is set.
+ */
+
+
+int knet_handle_enable_sock_notify(knet_handle_t knet_h,
+				   void *sock_notify_fn_private_data,
+				   void (*sock_notify_fn) (
+						void *private_data,
+						int datafd,
+						int8_t channel,
+						int error,
+						int errorno));
+
+//int knet_handle_get_replace_datafd(knet_handle_t knet_h, const int datafd, const int channel);
+//int knet_handle_get_fd_index(knet_handle_t knet_h, const int datafd, int *channel);
+//int knet_handle_get_datafd(knet_handle_t knet_h, const int channel, int *datafd);
+
+/*
  * knet_recv
  *
  * knet_h   - pointer to knet_handle_t
@@ -131,7 +221,8 @@ int knet_handle_free(knet_handle_t knet_h);
 
 ssize_t knet_recv(knet_handle_t knet_h,
 		  char *buff,
-		  const size_t buff_len);
+		  const size_t buff_len,
+		  const int8_t channel);
 
 /*
  * knet_send
@@ -148,7 +239,8 @@ ssize_t knet_recv(knet_handle_t knet_h,
 
 ssize_t knet_send(knet_handle_t knet_h,
 		  const char *buff,
-		  const size_t buff_len);
+		  const size_t buff_len,
+		  const int8_t channel);
 
 /*
  * knet_handle_enable_filter
@@ -204,6 +296,7 @@ int knet_handle_enable_filter(knet_handle_t knet_h,
 					uint8_t tx_rx,
 					uint16_t this_host_id,
 					uint16_t src_host_id,
+					int8_t *channel,
 					uint16_t *dst_host_ids,
 					size_t *dst_host_ids_entries));
 
@@ -280,7 +373,7 @@ int knet_handle_pmtud_getfreq(knet_handle_t knet_h, unsigned int *interval);
  *            and (if configured) crypto overhead,
  *            This function MUST NEVER block or add substantial delays.
  *
- * knet_handle_pmtud_notify returns:
+ * knet_handle_enable_pmtud_notify returns:
  *
  * 0 on success
  * -1 on error and errno is set.
diff --git a/libknet/onwire.h b/libknet/onwire.h
index f39fc4b..a391966 100644
--- a/libknet/onwire.h
+++ b/libknet/onwire.h
@@ -101,6 +101,7 @@ struct knet_header_payload_data {
 	uint8_t		khp_data_bcast;		/* data destination bcast/ucast */
 	uint8_t		khp_data_frag_num;	/* number of fragments of this pckt. 1 is not fragmented */
 	uint8_t		khp_data_frag_seq;	/* as above, indicates the frag sequence number */
+	int8_t		khp_data_channel;	/* transport channel data for localsock <-> knet <-> localsock mapping */
 	uint8_t		khp_data_userdata[0];	/* pointer to the real user data */
 } __attribute__((packed));
 
@@ -165,6 +166,7 @@ struct knet_header {
 #define khp_data_frag_seq kh_payload.khp_data.khp_data_frag_seq
 #define khp_data_userdata kh_payload.khp_data.khp_data_userdata
 #define khp_data_bcast    kh_payload.khp_data.khp_data_bcast
+#define khp_data_channel  kh_payload.khp_data.khp_data_channel
 
 #define khp_ping_link     kh_payload.khp_ping.khp_ping_link
 #define khp_ping_time     kh_payload.khp_ping.khp_ping_time
diff --git a/libknet/ping_test.c b/libknet/ping_test.c
index 04dddd2..689e71c 100644
--- a/libknet/ping_test.c
+++ b/libknet/ping_test.c
@@ -23,7 +23,8 @@
 
 #define KNET_RING_DEFPORT 50000
 
-static int knet_sock = 0;
+static int knet_sock[4];
+static int8_t channel[4];
 static knet_handle_t knet_h;
 static struct knet_handle_crypto_cfg knet_handle_crypto_cfg;
 static uint8_t loglevel = KNET_LOG_INFO;
@@ -256,6 +257,13 @@ static void sigint_handler(int signum)
 				printf("Unable to remove host: %s\n",strerror(errno));
 		}
 
+		for (j = 0; j < 4; j++) {
+			if (knet_handle_remove_datafd(knet_h, knet_sock[j]) < 0) {
+				printf("Unable to delete datafd!!!: %s\n",strerror(errno));
+				exit(EXIT_FAILURE);
+			}
+		}
+
 		if (knet_handle_free(knet_h)) {
 			printf("Unable to cleanup before exit: %s\n",strerror(errno));
 			exit(EXIT_FAILURE);
@@ -289,10 +297,40 @@ static void host_notify(void *private_data, uint16_t host_id, uint8_t reachable,
 	return;
 }
 
+static void recv_data(knet_handle_t khandle, int inchannel, int has_crypto)
+{
+	char recvbuff[66000];
+	ssize_t rlen = 0;
+
+	rlen = knet_recv(knet_h, recvbuff, sizeof(recvbuff), inchannel);
+
+	if (!rlen) {
+		printf("EOF\n");
+		return;
+	}
+
+	if ((rlen < 0) && ((errno = EAGAIN) || (errno = EWOULDBLOCK))) {
+		printf("NO MORE DATA TO READ: %s\n", strerror(errno));
+		return;
+	}
+
+	printf("Received data (%zu bytes): '%s' on channel: %d\n", rlen, recvbuff, inchannel);
+
+	if (has_crypto) {
+#if 0
+		printf("changing crypto key\n");
+		memset(knet_handle_crypto_cfg.private_key, has_crypto, KNET_MAX_KEY_LEN);
+		if (knet_handle_crypto(knet_h, &knet_handle_crypto_cfg)) {
+			printf("Unable to change key on the fly\n");
+			has_crypto++;
+		}
+#endif
+	}
+}
+
 int main(int argc, char *argv[])
 {
 	char out_big_buff[64000], out_big_frag[65536], hello_world[16];
-	char recvbuff[66000];
 	size_t len;
 	fd_set rfds;
 	struct timeval tv;
@@ -303,6 +341,7 @@ int main(int argc, char *argv[])
 	int logfd;
 	unsigned int link_mtu = 0, data_mtu = 0;
 	int big = 0;
+	int j;
 
 	if (argc < 3) {
 		print_usage(argv[0]);
@@ -330,7 +369,7 @@ int main(int argc, char *argv[])
 
 	set_debug(argc, argv);
 
-	if ((knet_h = knet_handle_new(1, &knet_sock, logfd, loglevel)) == NULL) {
+	if ((knet_h = knet_handle_new(1, logfd, loglevel)) == NULL) {
 		printf("Unable to create new knet_handle_t\n");
 		exit(EXIT_FAILURE);
 	}
@@ -370,12 +409,22 @@ int main(int argc, char *argv[])
 	}
 
 	argv_to_hosts(argc, argv);
-	knet_handle_setfwd(knet_h, 1);	
+	knet_handle_setfwd(knet_h, 1);
+
+	for (j = 0; j < 4; j++) {
+		knet_sock[j] = 0;
+		channel[j] = -1;
+		if (knet_handle_add_datafd(knet_h, &knet_sock[j], &channel[j]) < 0) {
+			printf("Unable to add datafd!!!\n");
+			exit(EXIT_FAILURE);
+		}
+	}
 
 	while (1) {
 		ssize_t wlen;
 		size_t i, buff_len;
 		char *buff;
+		int outchan;
 
 		knet_host_get_host_list(knet_h, host_ids, &host_ids_entries);
 		for (i = 0; i < host_ids_entries; i++) {
@@ -395,16 +444,19 @@ int main(int argc, char *argv[])
 				buff = hello_world;
 				buff_len = 13;
 				big = 1;
+				outchan = channel[0];
 				break;
 			case 1: /* big but does not require frag */
 				buff = out_big_buff;
 				buff_len = sizeof(out_big_buff);
 				big = 2;
+				outchan = channel[1];
 				break;
 			case 2: /* big and requires frag */
 				buff = out_big_frag;
 				buff_len = sizeof(out_big_frag);
 				big = 0;
+				outchan = channel[2];
 				break;
 			default:
 				printf("unknown packet size?\n");
@@ -412,7 +464,8 @@ int main(int argc, char *argv[])
 				break;
 		}
 
-		wlen = knet_send(knet_h, buff, buff_len);
+		printf("Sending '%zu' bytes on channel: %d\n", buff_len, outchan);
+		wlen = knet_send(knet_h, buff, buff_len, outchan);
 		if (wlen != buff_len) {
 			printf("Unable to send messages to socket\n");
 			exit(1);
@@ -423,7 +476,9 @@ int main(int argc, char *argv[])
 
  select_loop:
 		FD_ZERO(&rfds);
-		FD_SET(knet_sock, &rfds);
+		for (j = 0; j < 4; j++) {
+			FD_SET(knet_sock[j], &rfds);
+		}
 		FD_SET(logpipefd[0], &rfds);
 
 		len = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
@@ -434,34 +489,14 @@ int main(int argc, char *argv[])
 		if (len < 0) {
 			printf("Unable select over knet_handle_t\n");
 			exit(EXIT_FAILURE);
-		} else if (FD_ISSET(knet_sock, &rfds)) {
-			ssize_t rlen = 0;
-
-			rlen = knet_recv(knet_h, recvbuff, sizeof(recvbuff));
-
-			if (!rlen) {
-				printf("EOF\n");
-				break;
-			}
-
-			if ((rlen < 0) && ((errno = EAGAIN) || (errno = EWOULDBLOCK))) {
-				printf("NO MORE DATA TO READ: %s\n", strerror(errno));
-				break;
-			}
-
-			printf("Received data (%zu bytes): '%s'\n", rlen, recvbuff);
-
-			if (has_crypto) {
-#if 0
-				printf("changing crypto key\n");
-				memset(knet_handle_crypto_cfg.private_key, has_crypto, KNET_MAX_KEY_LEN);
-				if (knet_handle_crypto(knet_h, &knet_handle_crypto_cfg)) {
-					printf("Unable to change key on the fly\n");
-					has_crypto++;
-				}
-#endif
-			}
-
+		} else if (FD_ISSET(knet_sock[0], &rfds)) {
+			recv_data(knet_h, channel[0], has_crypto);
+		} else if (FD_ISSET(knet_sock[1], &rfds)) {
+			recv_data(knet_h, channel[1], has_crypto);
+		} else if (FD_ISSET(knet_sock[2], &rfds)) {
+			recv_data(knet_h, channel[2], has_crypto);
+		} else if (FD_ISSET(knet_sock[3], &rfds)) {
+			recv_data(knet_h, channel[3], has_crypto);
 		} else if (FD_ISSET(logpipefd[0], &rfds)) {
 			struct knet_log_msg msg;
 			size_t bytes_read = 0;
diff --git a/libknet/threads_common.h b/libknet/threads_common.h
index 198316d..2b507a1 100644
--- a/libknet/threads_common.h
+++ b/libknet/threads_common.h
@@ -19,6 +19,4 @@ do { \
 		*(diff) = end.tv_nsec - start.tv_nsec; \
 } while (0);
 
-#define KNET_EPOLL_MAX_EVENTS 8
-
 #endif
diff --git a/libknet/threads_send_recv.c b/libknet/threads_send_recv.c
index 30002ab..9c438d7 100644
--- a/libknet/threads_send_recv.c
+++ b/libknet/threads_send_recv.c
@@ -81,7 +81,7 @@ static void _dispatch_to_links(knet_handle_t knet_h, struct knet_host *dst_host,
 	return;
 }
 
-static void _parse_recv_from_sock(knet_handle_t knet_h, int buf_idx, ssize_t inlen)
+static void _parse_recv_from_sock(knet_handle_t knet_h, int buf_idx, ssize_t inlen, int8_t channel)
 {
 	ssize_t outlen, frag_len;
 	struct knet_host *dst_host;
@@ -120,6 +120,7 @@ static void _parse_recv_from_sock(knet_handle_t knet_h, int buf_idx, ssize_t inl
 						KNET_DST_HOST_FILTER_TX,
 						knet_h->host_id,
 						knet_h->host_id,
+						&channel,
 						dst_host_ids_temp,
 						&dst_host_ids_entries_temp);
 				if (bcast < 0) {
@@ -211,6 +212,7 @@ static void _parse_recv_from_sock(knet_handle_t knet_h, int buf_idx, ssize_t inl
 
 	inbuf->khp_data_bcast = bcast;
 	inbuf->khp_data_frag_num = ceil((float)inlen / temp_data_mtu);
+	inbuf->khp_data_channel = channel;
 
 	while (frag_idx < inbuf->khp_data_frag_num) {
 		/*
@@ -234,6 +236,7 @@ static void _parse_recv_from_sock(knet_handle_t knet_h, int buf_idx, ssize_t inl
 
 		knet_h->send_to_links_buf[frag_idx]->khp_data_frag_num = inbuf->khp_data_frag_num;
 		knet_h->send_to_links_buf[frag_idx]->khp_data_bcast = inbuf->khp_data_bcast;
+		knet_h->send_to_links_buf[frag_idx]->khp_data_channel = inbuf->khp_data_channel;
 
 		memmove(knet_h->send_to_links_buf[frag_idx]->khp_data_userdata,
 			inbuf->khp_data_userdata + (temp_data_mtu * frag_idx),
@@ -323,18 +326,19 @@ out_unlock:
 	}
 }
 
-static void _handle_send_to_links(knet_handle_t knet_h, int sockfd, struct mmsghdr *msg, int type)
+static void _handle_send_to_links(knet_handle_t knet_h, int sockfd, int8_t channel, struct mmsghdr *msg, int type)
 {
 	ssize_t inlen = 0;
 	struct iovec iov_in;
 	int msg_recv, i;
+	int savederrno, docallback = 0, err = 0;
 
 	if (pthread_rwlock_rdlock(&knet_h->list_rwlock) != 0) {
 		log_debug(knet_h, KNET_SUB_SEND_T, "Unable to get read lock");
 		return;
 	}
 
-	if (!knet_h->is_socket) {
+	if ((channel < KNET_DATAFD_MAX) && (!knet_h->sockfd[channel].is_socket)) {
 		memset(&iov_in, 0, sizeof(iov_in));
 		iov_in.iov_base = (void *)knet_h->recv_from_sock_buf[0]->khp_data_userdata;
 		iov_in.iov_len = KNET_MAX_PACKET_SIZE;
@@ -342,32 +346,48 @@ static void _handle_send_to_links(knet_handle_t knet_h, int sockfd, struct mmsgh
 		inlen = readv(sockfd, &iov_in, 1);
 
 		if (inlen < 0) {
-			log_err(knet_h, KNET_SUB_SEND_T, "Unrecoverable error: %s", strerror(errno));
+			err = inlen;
+			savederrno = errno;
+			docallback = 1;
+			log_debug(knet_h, KNET_SUB_SEND_T, "Unrecoverable error: %s", strerror(errno));
 			goto out_unlock;
 		}
 
 		if (inlen == 0) {
-			log_err(knet_h, KNET_SUB_SEND_T, "Unrecoverable error! Got 0 bytes from socket!");
-			/* TODO: disconnection, should never happen! */
+			err = inlen;
+			savederrno = errno;
+			docallback = 1;
+			log_debug(knet_h, KNET_SUB_SEND_T, "Unrecoverable error! Got 0 bytes from socket!");
 			goto out_unlock;
 		}
 		msg_recv = 1;
 		knet_h->recv_from_sock_buf[0]->kh_type = type;
-		_parse_recv_from_sock(knet_h, 0, inlen);
+		_parse_recv_from_sock(knet_h, 0, inlen, channel);
 	} else {
 		msg_recv = recvmmsg(sockfd, msg, PCKT_FRAG_MAX, MSG_DONTWAIT, NULL);
 		if (msg_recv < 0) {
+			err = msg_recv;
+			savederrno = errno;
+			docallback = 1;
 			log_err(knet_h, KNET_SUB_SEND_T, "No message received from recvmmsg: %s", strerror(errno));
 			goto out_unlock;
 		}
 		for (i = 0; i < msg_recv; i++) {
 			 knet_h->recv_from_sock_buf[i]->kh_type = type;
-			_parse_recv_from_sock(knet_h, i, msg[i].msg_len);
+			_parse_recv_from_sock(knet_h, i, msg[i].msg_len, channel);
 		}
 	}
 
 out_unlock:
 	pthread_rwlock_unlock(&knet_h->list_rwlock);
+	if ((docallback) &&
+	    (knet_h->sock_notify_fn)) {
+		knet_h->sock_notify_fn(knet_h->sock_notify_fn_private_data,
+				       knet_h->sockfd[channel].sockfd[0],
+				       channel,
+				       err,
+				       savederrno);
+	}
 }
 
 void *_handle_send_to_links_thread(void *data)
@@ -378,6 +398,7 @@ void *_handle_send_to_links_thread(void *data)
 	struct mmsghdr msg[PCKT_FRAG_MAX];
 	struct iovec iov_in[PCKT_FRAG_MAX];
 	int i, nev, type;
+	int8_t channel;
 
 	memset(&msg, 0, sizeof(struct mmsghdr));
 
@@ -401,15 +422,22 @@ void *_handle_send_to_links_thread(void *data)
 	}
 
 	while (!knet_h->fini_in_progress) {
-		nev = epoll_wait(knet_h->send_to_links_epollfd, events, KNET_EPOLL_MAX_EVENTS, -1);
+		nev = epoll_wait(knet_h->send_to_links_epollfd, events, KNET_EPOLL_MAX_EVENTS + 1, -1);
 
 		for (i = 0; i < nev; i++) {
-			if (events[i].data.fd == knet_h->sockfd) {
-				type = KNET_HEADER_TYPE_DATA;
-			} else {
+			if (events[i].data.fd == knet_h->hostsockfd[0]) {
 				type = KNET_HEADER_TYPE_HOST_INFO;
+				channel = -1;
+			} else {
+				type = KNET_HEADER_TYPE_DATA;
+				for (channel = 0; channel < KNET_DATAFD_MAX; channel++) {
+					if ((knet_h->sockfd[channel].in_use) &&
+					    (knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created] == events[i].data.fd)) {
+						break;
+					}
+				}
 			}
-			_handle_send_to_links(knet_h, events[i].data.fd, msg, type);
+			_handle_send_to_links(knet_h, events[i].data.fd, channel, msg, type);
 		}
 	}
 
@@ -628,6 +656,7 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 	unsigned char *outbuf = (unsigned char *)knet_h->recv_from_links_buf[index];
 	struct knet_hostinfo *knet_hostinfo;
 	struct iovec iov_out[1];
+	int8_t channel;
 
 	if (knet_h->crypto_instance) {
 		if (crypto_authenticate_and_decrypt(knet_h,
@@ -686,6 +715,7 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 	case KNET_HEADER_TYPE_HOST_INFO:
 	case KNET_HEADER_TYPE_DATA:
 		inbuf->khp_data_seq_num = ntohs(inbuf->khp_data_seq_num);
+		channel = inbuf->khp_data_channel;
 
 		if (!_seq_num_lookup(src_host, inbuf->khp_data_bcast, inbuf->khp_data_seq_num, 0)) {
 			if (src_host->link_handler_policy != KNET_LINK_POLICY_ACTIVE) {
@@ -723,6 +753,7 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 						KNET_DST_HOST_FILTER_RX,
 						knet_h->host_id,
 						inbuf->kh_node,
+						&channel,
 						dst_host_ids,
 						&dst_host_ids_entries);
 				if (bcast < 0) {
@@ -752,11 +783,18 @@ static void _parse_recv_from_links(knet_handle_t knet_h, struct sockaddr_storage
 		}
 
 		if (inbuf->kh_type == KNET_HEADER_TYPE_DATA) {
+			if (!knet_h->sockfd[channel].in_use) {
+				log_debug(knet_h, KNET_SUB_LINK_T,
+					  "recieved packet fo channel %d but there is no local sock connected",
+					  channel);
+				return;
+			}
+
 			memset(iov_out, 0, sizeof(iov_out));
 			iov_out[0].iov_base = (void *) inbuf->khp_data_userdata;
 			iov_out[0].iov_len = len - KNET_HEADER_DATA_SIZE;
 
-			if (writev(knet_h->sockfd, iov_out, 1) == iov_out[0].iov_len) {
+			if (writev(knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created], iov_out, 1) == iov_out[0].iov_len) {
 				_seq_num_set(src_host, bcast, inbuf->khp_data_seq_num, 0);
 			} else {
 				log_debug(knet_h, KNET_SUB_LINK_T, "Packet has not been delivered");
