Merge pull request #27 from montjoie/multi_slave_2018_x

Multi slave 2018 x

10 files changed, 424 insertions, 32 deletions

diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile
index a65fd9a..137a59a 100644
--- a/lava-master/Dockerfile
+++ b/lava-master/Dockerfile
@@ -1,5 +1,7 @@
 FROM bitnami/minideb:stretch
 
+RUN echo "deb http://ftp.fr.debian.org/debian/ stretch-backports main" >> /etc/apt/sources.list
+
 RUN apt-get update
 
 # Add services helper utilities to start and stop LAVA
@@ -46,10 +48,10 @@ RUN if [ "$(uname -m)" = "x86_64" -o "$(uname -m)" = "x86" ] ;then apt-get -y in
 
 # Install latest
 COPY scripts/build-lava /
-RUN bash /build-lava lava-dispatcher 2017.11 2017.11.post1-1
-RUN service postgresql start && \
- bash /build-lava lava-server 2017.11 2017.11.post1-1 && \
- /stop.sh
+RUN bash /build-lava lava-dispatcher 2018.4 2018.4-1
+RUN /etc/init.d/postgresql start && \
+ bash /build-lava lava-server 2018.4 2018.4-1 && \
+ /etc/init.d/postgresql stop
 
 RUN a2enmod proxy \
  && a2enmod proxy_http \
@@ -62,20 +64,35 @@ RUN git clone https://github.com/BayLibre/lava-healthchecks.git
 RUN cp lava-healthchecks/health-checks/* /etc/lava-server/dispatcher-config/health-checks/
 COPY health-checks/* /etc/lava-server/dispatcher-config/health-checks/
 
+# Add services helper utilities to start and stop LAVA
+COPY scripts/stop.sh .
+COPY scripts/start.sh .
+COPY scripts/lava-logs /etc/init.d/
+RUN chmod 755 /etc/init.d/lava-logs
+COPY scripts/lava-master /etc/init.d/
+RUN chmod 755 /etc/init.d/lava-master
+COPY scripts/lava-slave /etc/init.d/
+RUN chmod 755 /etc/init.d/lava-slave
+COPY scripts/lava-server-gunicorn /etc/init.d/
+RUN chmod 755 /etc/init.d/lava-server-gunicorn
+
 COPY devices/ /root/devices/
 COPY device-types/ /root/device-types/
 COPY users/ /root/lava-users/
 COPY tokens/ /root/lava-callback-tokens/
-COPY slaves/* /etc/lava-server/dispatcher.d/
 COPY scripts/setup.sh /
 RUN /start.sh && /setup.sh && /stop.sh
 
 #uncomment if you want to use squid
+# warning the address used must be network accessible by all slave
 #RUN sed -i 's,^.*http_proxy:.*,  http_proxy: http://squid:3128,' /etc/lava-server/env.yaml
 
 #comment this if you do HTTPS (For reenabling CSRF cookie)
 COPY settings.conf /etc/lava-server/
 
+COPY device-types-patch/ /root/device-types-patch/
+RUN cd /etc/lava-server/dispatcher-config/device-types/ && for patch in $(ls /root/device-types-patch/*patch) ; do sed -i 's,lava_scheduler_app/tests/device-types/,,' $patch && patch < $patch ; done
+
 EXPOSE 69/udp 80 3079 5555 5556
 
 CMD /start.sh && while [ true ];do sleep 365d; done
diff --git a/lava-master/device-types-patch/.empty b/lava-master/device-types-patch/.empty
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lava-master/device-types-patch/.empty
diff --git a/lava-master/scripts/lava-logs b/lava-master/scripts/lava-logs
new file mode 100644
index 0000000..69b2f82
--- /dev/null
+++ b/lava-master/scripts/lava-logs
@@ -0,0 +1,97 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          lava-logs
+# Required-Start:    $time
+# Required-Stop:     $time
+# X-Stop-After:      sendsigs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: lava-logs
+# Description:       lava-logs
+### END INIT INFO
+
+#
+# Author: Corentin LABBE <clabbe@baylibre.com>
+#
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="lava-logs"
+NAME=lava-logs
+LOGLEVEL=DEBUG
+
+DAEMON=/usr/bin/lava-server
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+[ -r /etc/lava-server/$NAME ] && . /etc/lava-server/$NAME
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+do_start()
+{
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   other if daemon could not be started or a failure occured
+	start-stop-daemon --start --quiet --background --pidfile $PIDFILE --exec $DAEMON -- manage lava-logs --level $LOGLEVEL $SOCKET $MASTER_SOCKET $IPV6 $ENCRYPT $MASTER_CERT $SLAVES_CERTS
+}
+
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   other if daemon could not be stopped or a failure occurred
+	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --exec $DAEMON
+}
+
+case "$1" in
+  start)
+	log_daemon_msg "Starting $DESC" "$NAME"
+	do_start
+	case "$?" in
+		0)
+		   log_end_msg 0 ;;
+		1) log_progress_msg "already started"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  stop)
+	log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	case "$?" in
+		0) log_end_msg 0 ;;
+		1) log_progress_msg "already stopped"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  restart|force-reload)
+	$0 stop
+	$0 start
+	;;
+  try-restart)
+	$0 status >/dev/null 2>&1 && $0 restart
+	;;
+  status)
+	status_of_proc -p $PIDFILE $DAEMON $RSYSLOGD && exit 0 || exit $?
+	;;
+  *)
+	echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload|try-restart|status}" >&2
+	exit 3
+	;;
+esac
+
+:
+
diff --git a/lava-master/scripts/lava-master b/lava-master/scripts/lava-master
new file mode 100644
index 0000000..3bd75aa
--- /dev/null
+++ b/lava-master/scripts/lava-master
@@ -0,0 +1,97 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          lava-master
+# Required-Start:    $time
+# Required-Stop:     $time
+# X-Stop-After:      sendsigs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: lava-master
+# Description:       lava-master
+### END INIT INFO
+
+#
+# Author: Corentin LABBE <clabbe@baylibre.com>
+#
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="lava-master"
+NAME=lava-master
+LOGLEVEL=DEBUG
+
+DAEMON=/usr/bin/lava-server
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+[ -r /etc/lava-server/$NAME ] && . /etc/lava-server/$NAME
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+do_start()
+{
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   other if daemon could not be started or a failure occured
+	start-stop-daemon --start --quiet --background --pidfile $PIDFILE --exec $DAEMON -- manage lava-master --level $LOGLEVEL $SOCKET $MASTER_SOCKET $IPV6 $ENCRYPT $MASTER_CERT $SLAVES_CERTS
+}
+
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   other if daemon could not be stopped or a failure occurred
+	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --exec $DAEMON
+}
+
+case "$1" in
+  start)
+	log_daemon_msg "Starting $DESC" "$NAME"
+	do_start
+	case "$?" in
+		0)
+		   log_end_msg 0 ;;
+		1) log_progress_msg "already started"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  stop)
+	log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	case "$?" in
+		0) log_end_msg 0 ;;
+		1) log_progress_msg "already stopped"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  restart|force-reload)
+	$0 stop
+	$0 start
+	;;
+  try-restart)
+	$0 status >/dev/null 2>&1 && $0 restart
+	;;
+  status)
+	status_of_proc -p $PIDFILE $DAEMON $RSYSLOGD && exit 0 || exit $?
+	;;
+  *)
+	echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload|try-restart|status}" >&2
+	exit 3
+	;;
+esac
+
+:
+
diff --git a/lava-master/scripts/lava-server-gunicorn b/lava-master/scripts/lava-server-gunicorn
new file mode 100644
index 0000000..217ad57
--- /dev/null
+++ b/lava-master/scripts/lava-server-gunicorn
@@ -0,0 +1,99 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          lava-server-gunicorn
+# Required-Start:    $time
+# Required-Stop:     $time
+# X-Stop-After:      sendsigs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: lava-server-gunicorn
+# Description:       lava-server-gunicorn
+### END INIT INFO
+
+#
+# Author: Corentin LABBE <clabbe@baylibre.com>
+#
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="lava-server-gunicorn"
+NAME=lava-server-gunicorn
+LOGLEVEL=DEBUG
+WORKERS=4
+LOGFILE=/var/log/lava-server/gunicorn.log
+RELOAD=
+DAEMON=/usr/bin/gunicorn3
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+[ -r /etc/lava-server/$NAME ] && . /etc/lava-server/$NAME
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+do_start()
+{
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   other if daemon could not be started or a failure occured
+	start-stop-daemon --start --quiet --background --pidfile $PIDFILE --exec $DAEMON -- lava_server.wsgi --log-level $LOGLEVEL --log-file $LOGFILE -u lavaserver -g lavaserver --workers $WORKERS $RELOAD
+}
+
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   other if daemon could not be stopped or a failure occurred
+	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --exec $DAEMON
+}
+
+case "$1" in
+  start)
+	log_daemon_msg "Starting $DESC" "$NAME"
+	do_start
+	case "$?" in
+		0)
+		   log_end_msg 0 ;;
+		1) log_progress_msg "already started"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  stop)
+	log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	case "$?" in
+		0) log_end_msg 0 ;;
+		1) log_progress_msg "already stopped"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  restart|force-reload)
+	$0 stop
+	$0 start
+	;;
+  try-restart)
+	$0 status >/dev/null 2>&1 && $0 restart
+	;;
+  status)
+	status_of_proc -p $PIDFILE $DAEMON $RSYSLOGD && exit 0 || exit $?
+	;;
+  *)
+	echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload|try-restart|status}" >&2
+	exit 3
+	;;
+esac
+
+:
+
diff --git a/lava-master/scripts/lava-slave b/lava-master/scripts/lava-slave
new file mode 100644
index 0000000..3ba00f7
--- /dev/null
+++ b/lava-master/scripts/lava-slave
@@ -0,0 +1,98 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          lava-slave
+# Required-Start:    $time
+# Required-Stop:     $time
+# X-Stop-After:      sendsigs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: lava-slave
+# Description:       lava-slave
+### END INIT INFO
+
+#
+# Author: Corentin LABBE <clabbe@baylibre.com>
+#
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="lava-slave"
+NAME=lava-slave
+
+DAEMON=/usr/bin/lava-slave
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+MASTER_URL=tcp://localhost:5556
+LOGGER_URL=tcp://localhost:5555
+LOGLEVEL=DEBUG
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+[ -r /etc/lava-dispatcher/$NAME ] && . /etc/lava-dispatcher/$NAME
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+do_start()
+{
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   other if daemon could not be started or a failure occured
+	start-stop-daemon --start --quiet --background --pidfile $PIDFILE --exec $DAEMON -- --level $LOGLEVEL --master $MASTER_URL --socket-addr $LOGGER_URL $IPV6 $ENCRYPT $MASTER_CERT $SLAVE_CERT $HOSTNAME
+}
+
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   other if daemon could not be stopped or a failure occurred
+	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --exec $DAEMON
+}
+
+case "$1" in
+  start)
+	log_daemon_msg "Starting $DESC" "$NAME"
+	do_start
+	case "$?" in
+		0)
+		   log_end_msg 0 ;;
+		1) log_progress_msg "already started"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  stop)
+	log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	case "$?" in
+		0) log_end_msg 0 ;;
+		1) log_progress_msg "already stopped"
+		   log_end_msg 0 ;;
+		*) log_end_msg 1 ;;
+	esac
+
+	;;
+  restart|force-reload)
+	$0 stop
+	$0 start
+	;;
+  try-restart)
+	$0 status >/dev/null 2>&1 && $0 restart
+	;;
+  status)
+	status_of_proc -p $PIDFILE $DAEMON $RSYSLOGD && exit 0 || exit $?
+	;;
+  *)
+	echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload|try-restart|status}" >&2
+	exit 3
+	;;
+esac
+
+:
+
diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh
index 5b62c45..3ab2abd 100755
--- a/lava-master/scripts/setup.sh
+++ b/lava-master/scripts/setup.sh
@@ -45,7 +45,7 @@ if [ -e /root/lava-callback-tokens ];then
 			exit 1
 		fi
 		echo "Adding $USER ($DESCRIPTION) DEBUG($TOKEN)"
-		lava-server manage tokens add --user $USER --secret $TOKEN --description $DESCRIPTION || exit 1
+		lava-server manage tokens add --user $USER --secret $TOKEN --description "$DESCRIPTION" || exit 1
 	done
 fi
 
diff --git a/lava-master/scripts/start.sh b/lava-master/scripts/start.sh
index f4cc4eb..0e223b5 100755
--- a/lava-master/scripts/start.sh
+++ b/lava-master/scripts/start.sh
@@ -24,14 +24,14 @@ start () {
 #remove lava-pid files incase the image is stored without first stopping the services
 rm -f /var/run/lava-*.pid 2> /dev/null
 
-start postgresql
-start apache2
-start lava-server
-start lava-master
-start lava-coordinator
-start lava-slave
-start lava-server-gunicorn
-start tftpd-hpa
+/etc/init.d/postgresql start
+start apache2 || exit $?
+start lava-logs || exit $?
+start lava-master || exit $?
+start lava-coordinator || exit $?
+start lava-slave || exit $?
+start lava-server-gunicorn || exit $?
+start tftpd-hpa || exit $?
 
 postgres-ready
 service apache2 reload #added after the website not running a few times on boot
diff --git a/lava-master/scripts/stop.sh b/lava-master/scripts/stop.sh
index ccf30fb..35d503f 100755
--- a/lava-master/scripts/stop.sh
+++ b/lava-master/scripts/stop.sh
@@ -2,9 +2,9 @@
 
 service lava-master stop
 service lava-slave stop
-service lava-server stop
+service lava-logs stop
 service lava-coordinator stop
 service lava-server-gunicorn stop
 service apache2 stop
-service postgresql stop
+/etc/init.d/postgresql stop
 service tftpd-hpa stop
diff --git a/lava-master/settings.conf b/lava-master/settings.conf
deleted file mode 100644
index c809e2c..0000000
--- a/lava-master/settings.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-    "DEBUG": false,
-    "STATICFILES_DIRS": [
-        ["lava-server", "/usr/share/pyshared/lava_server/htdocs/"]
-    ],
-    "MEDIA_ROOT": "/var/lib/lava-server/default/media",
-    "ARCHIVE_ROOT": "/var/lib/lava-server/default/archive",
-    "STATIC_ROOT": "/usr/share/lava-server/static",
-    "STATIC_URL": "/static/",
-    "MOUNT_POINT": "/",
-    "HTTPS_XML_RPC": false,
-    "LOGIN_URL": "/accounts/login/",
-    "LOGIN_REDIRECT_URL": "/",
-	"CSRF_COOKIE_SECURE": false,
-	"SESSION_COOKIE_SECURE": false
-}