Update virtio-loopback-adapter Beta version:

- Add exclusive eventfd for notifications
- Update eventfd handler for vhost-user call

Signed-off-by: Timos Ampelikiotis <t.ampelikiotis@virtualopensystems.com>

6 files changed, 96 insertions, 38 deletions

diff --git a/Makefile b/Makefile
index 218d7a6..3d18b6a 100644
--- a/Makefile
+++ b/Makefile
@@ -22,13 +22,13 @@
 #CFLAGS := -Wall -Wextra -Wno-unused-variable -Wno-unused-function
 CFLAGS := -Wno-unused-variable -Wno-unused-function
 CFLAGS =
-CC =
+CC ?=
 
 ifeq ($(ARCH), arm64)
  # arm64
- CC = aarch64-linux-gnu-gcc
+ CC ?= aarch64-linux-gnu-gcc
 else
- CC = gcc
+ CC ?= gcc
 endif
 
 ifeq ($(VHOST_USER_RNG), 1)
diff --git a/vhost_loopback.c b/vhost_loopback.c
index a8b78b6..0dc5c52 100644
--- a/vhost_loopback.c
+++ b/vhost_loopback.c
@@ -260,7 +260,7 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
 
     /* The next line has to be disable for rng */
     /* Clear and discard previous events if any. */
-    //event_notifier_test_and_clear(virtio_queue_get_host_notifier(vvq));
+    event_notifier_test_and_clear(virtio_queue_get_host_notifier(vvq));
 
     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
     r = vhost_user_set_vring_kick(&file);
diff --git a/vhost_user_loopback.c b/vhost_user_loopback.c
index 8ea366f..91e2efb 100644
--- a/vhost_user_loopback.c
+++ b/vhost_user_loopback.c
@@ -465,7 +465,15 @@ void vhost_user_share_fd(void)
      * msg.flags &= ~VHOST_USER_NEED_REPLY_MASK;
      */
 
-    (void)vu_message_write(client_sock, &msg);
+    if (vu_message_write(client_sock, &msg) < 0) {
+        DBG("vhost_user_share_fd -> write failed\n");
+        exit(1);
+    }
+
+    if (msg.flags & VHOST_USER_NEED_REPLY_MASK) {
+        process_message_reply(&msg);
+    }
+
 }
 
 int vhost_set_vring_file(VhostUserRequest request,
diff --git a/vhost_user_loopback.h b/vhost_user_loopback.h
index 45b6206..82d0c6c 100644
--- a/vhost_user_loopback.h
+++ b/vhost_user_loopback.h
@@ -244,7 +244,6 @@ extern struct vhost_user *vudev;
 /* Based on qemu/hw/virtio/vhost-user.c */
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 #define VHOST_LOG_PAGE 4096
-#define VIRTQUEUE_MAX_SIZE 1024
 #define VHOST_MEMORY_BASELINE_NREGIONS 8
 
 /* The version of the protocol we support */
diff --git a/virtio_loopback.c b/virtio_loopback.c
index b9fd353..174e263 100644
--- a/virtio_loopback.c
+++ b/virtio_loopback.c
@@ -73,6 +73,7 @@
 
 int s; /* To be deleted */
 int efd; /* Eventfd file descriptor */
+int efd_notify; /* Eventfd file descriptor */
 uint64_t eftd_ctr;
 fd_set rfds;
 int fd;
@@ -1076,50 +1077,52 @@ int virtio_set_features(VirtIODevice *vdev, uint64_t val)
 /* TODO: MMIO notifiers -- This might not be needed anymore  */
 static void virtio_queue_guest_notifier_read(EventNotifier *n)
 {
+    VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
+    if (event_notifier_test_and_clear(n)) {
+        virtio_irq(vq);
+    }
 }
 
-int vhost_user_loopback_eventfd = 0;
 
 int eventfd_count = 0;
 
-void *loopback_event_select(void *wfd)
+
+
+
+void *loopback_event_select(void *_e)
 {
     int retval;
     uint64_t eftd_ctr;
     fd_set rfds;
     int s;
+    EventNotifier *e = (EventNotifier *)_e;
+    int rfd = e->rfd;
+    VirtQueue *vq = container_of(e, VirtQueue, guest_notifier);
 
     DBG("\nWaiting event from vhost-user-device\n");
-    fflush(stdout);
 
     FD_ZERO(&rfds);
-    FD_SET(*(int *)wfd, &rfds);
+    FD_SET(rfd, &rfds);
 
     while (1) {
 
-        retval = select(vhost_user_loopback_eventfd + 1,
-                        &rfds, NULL, NULL, NULL);
+        retval = select(rfd + 1, &rfds, NULL, NULL, NULL);
 
         if (retval == -1) {
-            DBG("\nselect() error. Exiting...");
-            exit(EXIT_FAILURE);
-        } else if (retval > 0) {
+            DBG("select() error. Exiting...\n");
+            exit(1);
+        }
+        if (retval > 0) {
 
-            s = read(*(int *)wfd, &eftd_ctr, sizeof(uint64_t));
-            if (s != sizeof(uint64_t)) {
-                DBG("\neventfd read error. Exiting...");
-                exit(1);
-            } else {
-                DBG("\n\nEvent has come from the vhost-user-device "
-                    "(eventfd: %d) -> event_count: %d\n\n",
-                                        *(int *)wfd, eventfd_count);
+            DBG("\n\nEvent has come from the vhost-user-device "
+                "(eventfd: %d) -> event_count: %d\n\n",
+                                    rfd, eventfd_count);
 
-                eventfd_count++;
-                virtio_irq(global_vdev->vq);
+            eventfd_count++;
+            if (event_notifier_test_and_clear(e)) {
+                virtio_irq(vq);
             }
 
-        } else if (retval == 0) {
-            DBG("\nselect() says that no data was available");
         }
     }
 }
@@ -1131,17 +1134,16 @@ void event_notifier_set_handler(EventNotifier *e,
     int ret;
     pthread_t thread_id;
 
-    vhost_user_loopback_eventfd = e->wfd;
-
-    if (vhost_user_loopback_eventfd > 0) {
+    if (e->wfd > 0) {
         ret = pthread_create(&thread_id, NULL, loopback_event_select,
-                             (void *)(&(e->wfd)));
+                             (void *)e);
         if (ret != 0) {
             exit(1);
         }
     }
 }
 
+
 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                 bool with_irqfd)
 {
@@ -1370,6 +1372,7 @@ void virtio_queue_notify(VirtIODevice *vdev, int n)
     }
 
     if (vq->host_notifier_enabled) {
+        DBG("vq->host_notifier_enabled\n");
         event_notifier_set(&vq->host_notifier);
     } else if (vq->handle_output) {
         DBG("vq->handle_output\n");
@@ -1913,11 +1916,39 @@ void adapter_read_write_cb(void)
 
 }
 
+void *notify_select(void *data)
+{
+    int retval;
+    fd_set rfds;
+    int efd = *(int *)data;
+
+    DBG("\nWaiting for loopback notify events\n");
+
+    FD_ZERO(&rfds);
+    FD_SET(efd, &rfds);
+
+    while (1) {
+
+        retval = select(efd + 1, &rfds, NULL, NULL, NULL);
+
+        if (retval > 0) {
+            s = read(efd, &eftd_ctr, sizeof(uint64_t));
+            if (s != sizeof(uint64_t)) {
+                DBG("Eventfd read error\n");
+                exit(1);
+            } else {
+                //rcu_read_lock();
+                virtio_queue_notify(global_vdev, 0);
+                //rcu_read_unlock();
+            }
+        }
+    }
+}
 
 void *driver_event_select(void *data)
 {
     int retval;
-    (void) data;
+    int efd = *(int *)data;
 
     DBG("\nWaiting for loopback read/write events\n");
 
@@ -1945,7 +1976,6 @@ void *driver_event_select(void *data)
             DBG("\nselect() says that no data was available");
         }
     }
-
 }
 
 void create_rng_struct(void)
@@ -2055,11 +2085,10 @@ int virtio_loopback_start(void)
 {
     efd_data_t info;
     pthread_t thread_id;
+    pthread_t thread_id_notify;
     int ret = -1;
     int flags;
 
-    (void)info;
-
     fd = open("/dev/loopback", O_RDWR);
     if (fd < 0) {
         perror("Open call failed");
@@ -2074,8 +2103,15 @@ int virtio_loopback_start(void)
         exit(EXIT_FAILURE);
     }
 
+    efd_notify = eventfd(0, 0);
+    if (efd_notify == -1) {
+        DBG("\nUnable to create eventfd! Exiting...\n");
+        exit(EXIT_FAILURE);
+    }
+
     info.pid = getpid();
-    info.efd = efd;
+    info.efd[0] = efd;
+    info.efd[1] = efd_notify;
 
     /*
      * Send the appropriate information to the driver
@@ -2092,7 +2128,13 @@ int virtio_loopback_start(void)
     }
 
     /* Wait the eventfd */
-    ret = pthread_create(&thread_id, NULL, driver_event_select, NULL);
+    ret = pthread_create(&thread_id, NULL, driver_event_select, (void *)&efd);
+    if (ret != 0) {
+        exit(1);
+    }
+
+    /* Wait the eventfd */
+    ret = pthread_create(&thread_id_notify, NULL, notify_select, (void *)&efd_notify);
     if (ret != 0) {
         exit(1);
     }
@@ -2105,6 +2147,11 @@ int virtio_loopback_start(void)
         exit(1);
     }
 
+    ret = pthread_join(thread_id_notify, NULL);
+    if (ret != 0) {
+        exit(1);
+    }
+
     DBG("\nClosing eventfd. Exiting...\n");
     close(efd);
 
diff --git a/virtio_loopback.h b/virtio_loopback.h
index 1bd2d79..1fdb7db 100644
--- a/virtio_loopback.h
+++ b/virtio_loopback.h
@@ -255,6 +255,10 @@ typedef struct VirtIOMMIOProxy {
 /******************/
 
 
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+
 extern uint64_t vring_phys_addrs[2];
 extern uint32_t vring_phys_addrs_idx;
 
@@ -398,7 +402,7 @@ typedef struct VirtIODevice {
 } VirtIODevice;
 
 typedef struct efd_data {
-    int efd;
+    int efd[2];
     int pid;
 } efd_data_t;