evmgr: Isolate the event loop from jobs

The event loop is renamed evmgr for "event manager"
with the intention (1) still use evloop in jobs (2)
to provide an abstract event manager/handler/loop
abstraction.

Change-Id: Ib1955f661f98df80e1c7be99e9fe26a1e06d78f6
Signed-off-by: José Bollo <jose.bollo@iot.bzh>

4 files changed, 281 insertions, 152 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 3c9763d3..2985db4b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -63,6 +63,7 @@ SET(AFB_LIB_SOURCES
 	afb-ws.c
 	afb-wsj1.c
 	afb-xreq.c
+	evmgr.c
 	fdev.c
 	fdev-epoll.c
 	fdev-systemd.c
diff --git a/src/evmgr.c b/src/evmgr.c
new file mode 100644
index 00000000..2dc35e42
--- /dev/null
+++ b/src/evmgr.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (C) 2016-2019 "IoT.bzh"
+ * Author José Bollo <jose.bollo@iot.bzh>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <signal.h>
+#include <string.h>
+#include <time.h>
+#include <sys/syscall.h>
+#include <pthread.h>
+#include <errno.h>
+#include <assert.h>
+#include <sys/eventfd.h>
+
+#include <systemd/sd-event.h>
+
+#include "evmgr.h"
+#include "verbose.h"
+#include "systemd.h"
+
+/** Description of handled event loops */
+struct evmgr
+{
+	unsigned state;        /**< encoded state */
+	int efd;               /**< event notification */
+	void *holder;          /**< holder of the evmgr */
+	struct sd_event *sdev; /**< the systemd event loop */
+};
+
+#define EVLOOP_STATE_WAIT           1U
+#define EVLOOP_STATE_RUN            2U
+
+/**
+ * Run the event loop is set.
+ */
+void evmgr_run(struct evmgr *evmgr)
+{
+	int rc;
+	struct sd_event *se;
+
+	evmgr->state = EVLOOP_STATE_WAIT|EVLOOP_STATE_RUN;
+	se = evmgr->sdev;
+	rc = sd_event_prepare(se);
+	if (rc < 0) {
+		errno = -rc;
+		CRITICAL("sd_event_prepare returned an error (state: %d): %m", sd_event_get_state(se));
+		abort();
+	} else {
+		if (rc == 0) {
+			rc = sd_event_wait(se, (uint64_t)(int64_t)-1);
+			if (rc < 0) {
+				errno = -rc;
+				ERROR("sd_event_wait returned an error (state: %d): %m", sd_event_get_state(se));
+			}
+		}
+		evmgr->state = EVLOOP_STATE_RUN;
+		if (rc > 0) {
+			rc = sd_event_dispatch(se);
+			if (rc < 0) {
+				errno = -rc;
+				ERROR("sd_event_dispatch returned an error (state: %d): %m", sd_event_get_state(se));
+			}
+		}
+	}
+	evmgr->state = 0;
+}
+
+void evmgr_job_run(int signum, struct evmgr *evmgr)
+{
+	if (signum)
+		evmgr->state = 0;
+	else
+		evmgr_run(evmgr);
+}
+
+int evmgr_can_run(struct evmgr *evmgr)
+{
+	return !evmgr->state;
+}
+
+/**
+ * Internal callback for evmgr management.
+ * The effect of this function is hidden: it exits
+ * the waiting poll if any.
+ */
+static void evmgr_on_efd_event(struct evmgr *evmgr)
+{
+	uint64_t x;
+	read(evmgr->efd, &x, sizeof x);
+}
+
+/**
+ * wakeup the event loop if needed by sending
+ * an event.
+ */
+void evmgr_wakeup(struct evmgr *evmgr)
+{
+	uint64_t x;
+
+	if (evmgr->state & EVLOOP_STATE_WAIT) {
+		x = 1;
+		write(evmgr->efd, &x, sizeof x);
+	}
+}
+
+/**
+ */
+void *evmgr_holder(struct evmgr *evmgr)
+{
+	return evmgr->holder;
+}
+
+/**
+ */
+int evmgr_release_if(struct evmgr *evmgr, void *holder)
+{
+	if (evmgr->holder != holder)
+		return 0;
+	evmgr->holder = 0;
+	return 1;
+}
+
+/**
+ */
+int evmgr_try_hold(struct evmgr *evmgr, void *holder)
+{
+	if (!evmgr->holder)
+		evmgr->holder = holder;
+	return evmgr->holder == holder;
+}
+
+/******************************************************************************/
+/******************************************************************************/
+/******  SYSTEM D                                                        ******/
+/******************************************************************************/
+/******************************************************************************/
+
+/**
+ * Internal callback for evmgr management.
+ * The effect of this function is hidden: it exits
+ * the waiting poll if any. Then it wakes up a thread
+ * awaiting the evmgr using signal.
+ */
+static int on_evmgr_efd(sd_event_source *s, int fd, uint32_t revents, void *userdata)
+{
+	struct evmgr *evmgr = userdata;
+	evmgr_on_efd_event(evmgr);
+	return 1;
+}
+
+/**
+ * Gets a sd_event item for the current thread.
+ * @return a sd_event or NULL in case of error
+ */
+int evmgr_create(struct evmgr **result)
+{
+	int rc;
+	struct evmgr *evmgr;
+
+	/* creates the evmgr on need */
+	evmgr = malloc(sizeof *evmgr);
+	if (!evmgr) {
+		ERROR("out of memory");
+		rc = -ENOMEM;
+		goto error;
+	}
+
+	/* creates the eventfd for waking up polls */
+	evmgr->efd = eventfd(0, EFD_CLOEXEC|EFD_SEMAPHORE);
+	if (evmgr->efd < 0) {
+		ERROR("can't make eventfd for events");
+		rc = -errno;
+		goto error1;
+	}
+	/* create the systemd event loop */
+	evmgr->sdev = systemd_get_event_loop();
+	if (!evmgr->sdev) {
+		ERROR("can't make new event loop");
+		goto error2;
+	}
+	/* put the eventfd in the event loop */
+	rc = sd_event_add_io(evmgr->sdev, NULL, evmgr->efd, EPOLLIN, on_evmgr_efd, evmgr);
+	if (rc < 0) {
+		ERROR("can't register eventfd");
+		goto error2;
+	}
+
+	/* start the creation */
+	evmgr->state = 0;
+	evmgr->holder = 0;
+	*result = evmgr;
+	return 0;
+
+
+error2:
+	close(evmgr->efd);
+error1:
+	free(evmgr);
+error:
+	*result = 0;
+	return rc;
+}
+
diff --git a/src/evmgr.h b/src/evmgr.h
new file mode 100644
index 00000000..c78ea33f
--- /dev/null
+++ b/src/evmgr.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2016-2019 "IoT.bzh"
+ * Author José Bollo <jose.bollo@iot.bzh>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+
+struct evmgr;
+
+extern void evmgr_run(struct evmgr *evmgr);
+extern void evmgr_job_run(int signum, struct evmgr *evmgr);
+extern int evmgr_can_run(struct evmgr *evmgr);
+extern void evmgr_wakeup(struct evmgr *evmgr);
+extern int evmgr_release_if(struct evmgr *evmgr, void *holder);
+extern int evmgr_try_hold(struct evmgr *evmgr, void *holder);
+extern void *evmgr_holder(struct evmgr *evmgr);
+extern int evmgr_create(struct evmgr **result);
diff --git a/src/jobs.c b/src/jobs.c
index 217c5391..83ca2ed4 100644
--- a/src/jobs.c
+++ b/src/jobs.c
@@ -32,6 +32,7 @@
 #include <systemd/sd-event.h>
 
 #include "jobs.h"
+#include "evmgr.h"
 #include "sig-monitor.h"
 #include "verbose.h"
 #include "systemd.h"
@@ -56,18 +57,6 @@ struct job
 	unsigned dropped: 1; /**< is removed ? */
 };
 
-/** Description of handled event loops */
-struct evloop
-{
-	unsigned state;        /**< encoded state */
-	int efd;               /**< event notification */
-	struct sd_event *sdev; /**< the systemd event loop */
-	struct thread *holder; /**< holder of the evloop */
-};
-
-#define EVLOOP_STATE_WAIT           1U
-#define EVLOOP_STATE_RUN            2U
-
 /** Description of threads */
 struct thread
 {
@@ -115,7 +104,7 @@ static struct job *first_job;
 static struct job *free_jobs;
 
 /* event loop */
-static struct evloop evloop;
+static struct evmgr *evmgr;
 
 /**
  * Create a new job with the given parameters
@@ -254,70 +243,13 @@ static void job_cancel(int signum, void *arg)
 }
 
 /**
- * Monitored normal callback for events.
- * This function is called by the monitor
- * to run the event loop when the safe environment
- * is set.
- * @param signum 0 on normal flow or the number
- *               of the signal that interrupted the normal
- *               flow
- * @param arg     the events to run
- */
-static void evloop_run(int signum, void *arg)
-{
-	int rc;
-	struct sd_event *se;
-
-	if (!signum) {
-		se = evloop.sdev;
-		rc = sd_event_prepare(se);
-		if (rc < 0) {
-			errno = -rc;
-			CRITICAL("sd_event_prepare returned an error (state: %d): %m", sd_event_get_state(se));
-			abort();
-		} else {
-			if (rc == 0) {
-				rc = sd_event_wait(se, (uint64_t)(int64_t)-1);
-				if (rc < 0) {
-					errno = -rc;
-					ERROR("sd_event_wait returned an error (state: %d): %m", sd_event_get_state(se));
-				}
-			}
-			evloop.state = EVLOOP_STATE_RUN;
-			if (rc > 0) {
-				rc = sd_event_dispatch(se);
-				if (rc < 0) {
-					errno = -rc;
-					ERROR("sd_event_dispatch returned an error (state: %d): %m", sd_event_get_state(se));
-				}
-			}
-		}
-	}
-}
-
-/**
- * Internal callback for evloop management.
- * The effect of this function is hidden: it exits
- * the waiting poll if any.
- */
-static void evloop_on_efd_event()
-{
-	uint64_t x;
-	read(evloop.efd, &x, sizeof x);
-}
-
-/**
  * wakeup the event loop if needed by sending
  * an event.
  */
 static void evloop_wakeup()
 {
-	uint64_t x;
-
-	if (evloop.state & EVLOOP_STATE_WAIT) {
-		x = 1;
-		write(evloop.efd, &x, sizeof x);
-	}
+	if (evmgr)
+		evmgr_wakeup(evmgr);
 }
 
 /**
@@ -327,11 +259,13 @@ static void evloop_release()
 {
 	struct thread *nh, *ct = current_thread;
 
-	if (ct && evloop.holder == ct) {
+	if (ct && evmgr && evmgr_release_if(evmgr, ct)) {
 		nh = ct->nholder;
-		evloop.holder = nh;
-		if (nh)
+		ct->nholder = 0;
+		if (nh) {
+			evmgr_try_hold(evmgr, nh);
 			pthread_cond_signal(nh->cwhold);
+		}
 	}
 }
 
@@ -340,17 +274,7 @@ static void evloop_release()
  */
 static int evloop_get()
 {
-	struct thread *ct = current_thread;
-
-	if (evloop.holder)
-		return evloop.holder == ct;
-
-	if (!evloop.sdev)
-		return 0;
-
-	ct->nholder = NULL;
-	evloop.holder = ct;
-	return 1;
+	return evmgr && evmgr_try_hold(evmgr, current_thread);
 }
 
 /**
@@ -358,7 +282,7 @@ static int evloop_get()
  */
 static void evloop_acquire()
 {
-	struct thread **pwait, *ct;
+	struct thread *pwait, *ct;
 	pthread_cond_t cond;
 
 	/* try to get the evloop */
@@ -370,10 +294,10 @@ static void evloop_acquire()
 		pthread_cond_init(&cond, NULL);
 
 		/* queue current thread in holder list */
-		pwait = &evloop.holder;
-		while (*pwait)
-			pwait = &(*pwait)->nholder;
-		*pwait = ct;
+		pwait = evmgr_holder(evmgr);
+		while (pwait->nholder)
+			pwait = pwait->nholder;
+		pwait->nholder = ct;
 
 		/* wake up the evloop */
 		evloop_wakeup();
@@ -395,6 +319,7 @@ static void thread_enter(volatile struct thread *me)
 	me->tid = pthread_self();
 	me->stop = 0;
 	me->waits = 0;
+	me->nholder = 0;
 	me->upper = current_thread;
 	me->next = threads;
 	threads = (struct thread*)me;
@@ -454,17 +379,15 @@ static void thread_run_internal(volatile struct thread *me)
 			job_release(job);
 		/* no job, check event loop wait */
 		} else if (evloop_get()) {
-			if (evloop.state != 0) {
+			if (!evmgr_can_run(evmgr)) {
 				/* busy ? */
 				CRITICAL("Can't enter dispatch while in dispatch!");
 				abort();
 			}
 			/* run the events */
-			evloop.state = EVLOOP_STATE_RUN|EVLOOP_STATE_WAIT;
 			pthread_mutex_unlock(&mutex);
-			sig_monitor(0, evloop_run, NULL);
+			sig_monitor(0, (void(*)(int,void*))evmgr_job_run, evmgr);
 			pthread_mutex_lock(&mutex);
-			evloop.state = 0;
 		} else {
 			/* no job and no event loop */
 			running--;
@@ -754,60 +677,6 @@ int jobs_call(
 }
 
 /**
- * Internal callback for evloop management.
- * The effect of this function is hidden: it exits
- * the waiting poll if any. Then it wakes up a thread
- * awaiting the evloop using signal.
- */
-static int on_evloop_efd(sd_event_source *s, int fd, uint32_t revents, void *userdata)
-{
-	evloop_on_efd_event();
-	return 1;
-}
-
-/**
- * Gets a sd_event item for the current thread.
- * @return a sd_event or NULL in case of error
- */
-static struct sd_event *get_sd_event_locked()
-{
-	int rc;
-
-	/* creates the evloop on need */
-	if (!evloop.sdev) {
-		/* start the creation */
-		evloop.state = 0;
-		/* creates the eventfd for waking up polls */
-		evloop.efd = eventfd(0, EFD_CLOEXEC|EFD_SEMAPHORE);
-		if (evloop.efd < 0) {
-			ERROR("can't make eventfd for events");
-			goto error1;
-		}
-		/* create the systemd event loop */
-		evloop.sdev = systemd_get_event_loop();
-		if (!evloop.sdev) {
-			ERROR("can't make event loop");
-			goto error2;
-		}
-		/* put the eventfd in the event loop */
-		rc = sd_event_add_io(evloop.sdev, NULL, evloop.efd, EPOLLIN, on_evloop_efd, NULL);
-		if (rc < 0) {
-			ERROR("can't register eventfd");
-			evloop.sdev = NULL;
-error2:
-			close(evloop.efd);
-error1:
-			return NULL;
-		}
-	}
-
-	/* acquire the event loop */
-	evloop_acquire();
-
-	return evloop.sdev;
-}
-
-/**
  * Ensure that the current running thread can control the event loop.
  */
 void jobs_acquire_event_manager()
@@ -820,9 +689,18 @@ void jobs_acquire_event_manager()
 		current_thread = &lt;
 	}
 
-	/* process */
+	/* lock */
 	pthread_mutex_lock(&mutex);
-	get_sd_event_locked();
+
+	/* creates the evloop on need */
+	if (!evmgr)
+		evmgr_create(&evmgr);
+
+	/* acquire the event loop under lock */
+	if (evmgr)
+		evloop_acquire();
+
+	/* unlock */
 	pthread_mutex_unlock(&mutex);
 
 	/* release the faked thread environment if needed */
@@ -835,7 +713,7 @@ void jobs_acquire_event_manager()
 		 * A workaround to achieve that goal is for the caller to
 		 * require the event loop a second time after having modified it.
 		 */
-		NOTICE("Requiring sd_event loop out of binder callbacks is hazardous!");
+		NOTICE("Requiring event manager/loop from outside of binder's callback is hazardous!");
 		if (verbose_wants(Log_Level_Info))
 			sig_monitor_dumpstack();
 		evloop_release();