16 files changed, 576 insertions, 337 deletions

diff --git a/README.md b/README.md
index 0529b54..5580f06 100644
--- a/README.md
+++ b/README.md
@@ -26,17 +26,11 @@ The following packages are necessary on the host machine:
 Example to use lava-docker with only one QEMU device:
 
 * Checkout the lava-docker repository
-* You will obtain the following boards.yaml
-```
-lab-slave-0:
-  boardlist:
-    qemu-01:
-      type: qemu
-```
 * Generate configuration files for LAVA, udev, serial ports, etc. from boards.yaml via
 ```
 ./lavalab-gen.py
 ```
+* Go to output/local directory
 * Build docker images via
 ```
 docker-compose build
@@ -65,9 +59,9 @@ You could check in https://github.com/Linaro/lava-server/tree/release/lava_sched
 
 Example:
 For a beagleboneblack, the device-type is beaglebone-black (Even if official DT name is am335x-boneblack)
-So you have now:
+So you need to add in the boards section:
 ```
-    beagleboneblack-01:
+    - name: beagleboneblack-01
       type: beaglebone-black
 ```
 
@@ -87,7 +81,7 @@ The serial is AK04TU1X
 ```
 So you have now:
 ```
-    beagleboneblack-01:
+    - name: beagleboneblack-01
       type: beaglebone-black
       uart:
         idvendor: 0x0403
@@ -105,7 +99,7 @@ udevadm info -a -n /dev/ttyUSB1 |grep devpath | head -n1
 ```
 So you have now:
 ```
-    beagleboneblack-01:
+    - name: beagleboneblack-01
       type: beaglebone-black
       uart:
         idvendor: 0x0403
@@ -130,7 +124,7 @@ If the beagleboneblack is wired to port 3 and the ACME board have IP 192.168.66.
 #### Example:
 beagleboneblack, with FTDI (serial 1234567), connected to port 5 of an ACME
 ```
-    beagleboneblack-01:
+    - name: beagleboneblack-01
       type: beaglebone-black
       pdu_generic:
         hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 5
@@ -142,14 +136,11 @@ beagleboneblack, with FTDI (serial 1234567), connected to port 5 of an ACME
 	serial: 1234567
 ```
 
-## Known limitations
-The current lava-docker provides support for generating only one LAVA slave.
-
 ## Architecture
-The setup is composed of a host which runs the following docker images and DUT to be tested.<br/>
+The basic setup is composed of a host which runs the following docker images and DUT to be tested.<br/>
 * lava-master: run lava-server along with the web interface
 * lava-slave: run lava-dispatcher, the compoment which sends jobs to DUTs
-* squid: an HTTP proxy for caching downloaded contents (kernel/dtb/rootfs)
+* squid: an HTTP proxy for caching downloaded contents (kernel/dtb/rootfs) (Work in progress)
 
 The host and DUTs must share a common LAN.<br/>
 The host IP on this LAN must be set as dispatcher_ip in boards.yaml.<br/>
@@ -159,6 +150,12 @@ So, on the LAN shared with DUTs, a running DHCPD is necessary. (See DHCPD below)
 
 ![lava-docker diagram](doc/lava-docker.png)
 
+## Multi-host architectures
+Lava-docker support multi-host architecture, Master and slaves could be on different host.
+
+Lava-docker support multiples slaves, but with a maximum of one slave per host.
+This is due to that slave need TFTP port accessible from outside.
+
 ### Power supply
 You need to have a PDU for powering your DUT.
 Managing PDUs is done via pdu_generic
@@ -214,25 +211,50 @@ You can use the lavalab-gen.sh helper script which will do all the above actions
 ### boards.yaml
 This file describe how the DUTs are connected and powered.
 ```
-lab-slave-XX:		The name of the slave (where XX is a number)
-  dispatcher_ip: the IP where the slave could be contacted. In lava-docker it is the host IP since docker proxify TFTP from host to the slave.
-  host_has_cpuflag_kvm: Does the host running lab-slave-XX have KVM
-  boardlist:
-    devicename:	Each board must be named by their device-type as "device-type-XX" (where XX is a number)
-      type: the LAVA device-type of this device
-      uboot_ipaddr: (optional) a static IP to set in uboot
-      uboot_macaddr: (Optional) the MAC address to set in uboot
+masters:
+ - name:  lava-master	name of the master
+    host: name		name of the host running lava-master (default to "local")
+    users:
+    - name: LAVA username
+      token: The token of this user
+      password: Password the this user (generated if not provided)
+      superuser: yes/no (default no)
+      staff: yes/no (default no)
+    tokens:
+    - username: The LAVA user owning the token below. (This user should be created via users:)
+      token: The token for this callback
+      description: The description of this token. This string could be used with LAVA-CI.
+slaves:
+  - name: lab-slave-XX		The name of the slave (where XX is a number)
+    host: name			name of the host running lava-slave-XX (default to "local")
+    dispatcher_ip: 		the IP where the slave could be contacted. In lava-docker it is the host IP since docker proxify TFTP from host to the slave.
+    remote_master: 		the name of the master to connect to
+    remote_address: 		the FQDN or IP address of the master (if different from remote_master)
+    remote_rpc_port: 		the port used by the LAVA RPC2 (default 80)
+    remote_user: 		the user used for connecting to the master
+    remote_proto:		http(default) or https
+    extra_actions:		An optional list of action to do at end of the docker build
+    - "apt-get install package"
+
+boards:
+  - name: devicename	Each board must be named by their device-type as "device-type-XX" (where XX is a number)
+    type: the LAVA device-type of this device
+    kvm: (For qemu only) Does the qemu could use KVM (default: no)
+    uboot_ipaddr:	(optional) a static IP to set in uboot
+    uboot_macaddr:	(Optional) the MAC address to set in uboot
+    custom_option:	(optional) All following strings will be directly append to devicefile
+    - "set x=1"
 # One of uart or connection_command must be choosen
-      uart:
-	idvendor: The VID of the UART (Formated as 0xXXXX)
-	idproduct: the PID of the UART (Formated as 0xXXXX)
-        serial: The serial number in case of FTDI uart
-        devpath: the UDEV devpath to this uart for UART without serial number
-      connection_command: A command to be ran for getting a serial console
-      pdu_generic:
-        hard_reset_command: commandline to reset the board
-	power_off_command: commandline to power off the board
-	power_on_command: commandline to power on the board
+    uart:
+      idvendor: The VID of the UART (Formated as 0xXXXX)
+      idproduct: the PID of the UART (Formated as 0xXXXX)
+      serial: The serial number in case of FTDI uart
+      devpath: the UDEV devpath to this uart for UART without serial number
+    connection_command: A command to be ran for getting a serial console
+    pdu_generic:
+      hard_reset_command: commandline to reset the board
+      power_off_command: commandline to power off the board
+      power_on_command: commandline to power on the board
 ```
 Notes on UART:
 * Only one of devpath/serial is necessary.
@@ -251,23 +273,6 @@ Note on connection_command: connection_command is for people which want to use o
 
 Examples: see [boards.yaml.example](boards.yaml.example)
 
-### tokens.yaml
-The tokens format has two sections, one for LAVA users, the other for callback tokens
-```
-lava_server_users:
-  - name: LAVA username
-    token: The token of this user
-    password: Password the this user (generated if not provided)
-    superuser: yes/no (default no)
-    staff: yes/no (default no)
-callback_tokens:
-  - filename: The filename for storing the informations below, the name should be unique along other callback tokens
-    username: The LAVA user owning the token below. (This user should be created via lava_server_users:)
-    token: The token for this callback
-    description: The description of this token. This string could be used with LAVA-CI.
-```
-Example: see [tokens.yaml](tokens.yaml)
-
 ### Generate
 ```
 lavalab-gen.py
@@ -275,13 +280,12 @@ lavalab-gen.py
 
 this script will generate all necessary files in the following locations:
 ```
-conmux/		All files needed by conmux
-tokens/		This is where the callback tokens will be generated
-users/		This is where the users will be generated
-devices/	All LAVA devices files
-slaves/		Contain the dispatcher_ip to give to slave node
-udev-rules for host
-docker-compose.yml	Generated from docker-compose.template
+output/host/lava-master/tokens/			This is where the callback tokens will be generated
+output/host/lava-master/users/			This is where the users will be generated
+output/host/lab-slave-XX/conmux/		All files needed by conmux
+output/host/lab-slave-XX/devices/		All LAVA devices files
+output/host/udev/99-lavaworker-udev.rules 	udev rules for host
+output/host/docker-compose.yml			Generated from docker-compose.template
 ```
 
 All thoses file (except for udev-rules) will be handled by docker.
@@ -306,10 +310,10 @@ For running all images, simply run:
 docker-compose up -d
 ```
 
-## Proxy cache
+## Proxy cache (Work in progress)
 A squid docker is provided for caching all LAVA downloads (image, dtb, rootfs, etc...)<br/>
 You have to uncomment a line in lava-master/Dockerfile to enable it.<br/>
-Note that the squid proxy is always built and run.
+For the moment, it is unsupported and unbuilded.
 
 ## Security
 Note that this container provides defaults which are unsecure. If you plan on deploying this in a production enviroment please consider the following items:
diff --git a/boards.yaml b/boards.yaml
index 88f7e6d..df6548c 100644
--- a/boards.yaml
+++ b/boards.yaml
@@ -1,4 +1,22 @@
-lab-slave-0:
-  boardlist:
-    qemu-01:
-      type: qemu
+masters:
+  - name: master1
+    host: local
+    users:
+    - name: admin
+      token: longrandomtokenadmin
+      password: admin
+      superuser: yes
+      staff: yes
+    tokens:
+    - username: admin
+      token: dfjdfkfkdjfkdsjfsl
+      description: no description
+slaves:
+  - name: lab-slave-0
+    host: local
+    remote_master: master1
+    remote_user: admin
+
+boards:
+  - name: qemu-01
+    type: qemu
diff --git a/boards.yaml.example b/boards.yaml.example
index 5737a48..ee4469f 100644
--- a/boards.yaml.example
+++ b/boards.yaml.example
@@ -1,66 +1,47 @@
-lab-slave-0:
-  dispatcher_ip: 192.168.66.1
-  boardlist:
-    qemu-01:
-      type: qemu
-    bcm2837-rpi-3-b-01:
-      type: bcm2837-rpi-3-b
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 1
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 1
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 1
-      uart:
-        idvendor: 0x0403
-        idproduct: 0x6001
-        serial: FT9QQZTA
-    am335x-boneblack-01:
-      type: beaglebone-black
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 2
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 2
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 2
-      uart:
-        idvendor: 0x0403
-        idproduct: 0x6001
-        serial: FT9QR1A9
-    meson-gxl-s905x-libretech-cc-01:
-      type: meson-gxl-s905x-libretech-cc
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 3
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 3
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 3
-      uart:
-        idvendor: 0x067b
-        idproduct: 0x2303
-        devpath: 1.1.4
-      uboot_macaddr: "00:FA:E0:DE:AD:78"
-    dragonboard-410c-01:
-      type: dragonboard-410c
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 4
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 4
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 4
-      uart:
-        idvendor: 0x403
-        idproduct: 0x6001
-        serial: FT9R7VDB
-    r8a7796-m3ulcb-01:
-      type: r8a7796-m3ulcb
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 5
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 5
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 5
-      uart:
-        idvendor: 0x0403
-        idproduct: 0x6001
-        serial: AK04WW0Q
-    imx6q-sabrelite-01:
-      type: imx6q-sabrelite
-      pdu_generic:
-        hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 6
-        power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 6
-        power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 6
-      uart:
-        idvendor: 0x0403
-        idproduct: 0x6015
-        serial: DAZ0KEUH
+masters:
+  - name: master1
+    host: local
+    users:
+    - name: admin
+      token: longrandomtokenadmin
+      password: admin
+      superuser: yes
+      staff: yes
+slaves:
+  - name: lab-slave-0
+    host: local
+    remote_master: master1
+    remote_user: admin
+    dispatcher_ip: 192.168.66.1
+
+boards:
+  - name: qemu-02
+    type: qemu
+    slave: lab-slave-0
+    kvm: True
+  - name: meson-gxl-s905x-libretech-cc-01
+    type: meson-gxl-s905x-libretech-cc
+    slave: lab-slave-0
+    pdu_generic:
+      hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 2
+      power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 2
+      power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 2
+    uart:
+      idvendor: 0x0403
+      idproduct: 0x6001
+      serial: FT9QR2TZ
+  - name: meson-gxbb-nanopi-k2-01
+    type: meson-gxbb-nanopi-k2
+    slave: lab-slave-0
+    custom_option:
+    - 'set bootloader_prompt = "nanopi-k2#"'
+    - "set interrupt_prompt = 'nanopi'"
+    uboot_ipaddr: 192.168.66.201
+    pdu_generic:
+      hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 3
+      power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 3
+      power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 3
+    uart:
+      idvendor: 0x0403
+      idproduct: 0x6001
+      serial: FT9ZOR0I
diff --git a/deploy.sh b/deploy.sh
new file mode 100755
index 0000000..f7be5be
--- /dev/null
+++ b/deploy.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+#check for root
+BEROOT=""
+if [ $(id -u) -ne 0 ];then
+	BEROOT="sudo "
+fi
+$BEROOT rm /etc/udev/rules.d/*lava*rules
+$BEROOT cp udev/*lava*rules /etc/udev/rules.d/
+$BEROOT udevadm control --reload-rules || exit $?
+$BEROOT udevadm trigger || exit $?
+
+docker-compose build || exit 1
+docker-compose up -d || exit 1
diff --git a/docker-compose.template b/docker-compose.template
deleted file mode 100644
index ced5762..0000000
--- a/docker-compose.template
+++ /dev/null
@@ -1,45 +0,0 @@
-version: '2.0'
-services:
-  lava-master:
-    hostname: lava-master
-    restart: always
-    build:
-      context: lava-master
-    ports:
-      - "10080:80"
-      - "5555:5555"
-      - "5556:5556"
-    volumes:
-# boot and /lib/modules are for libguestfs (TODO set them read_only with docker-compose 3.0)
-      - "/boot:/boot"
-      - "/lib/modules:/lib/modules"
-  lab-slave-0:
-    hostname: lab-slave-0
-#conmux does not support dns_search
-    dns_search: ""
-    restart: always
-    build:
-      context: lava-slave
-    environment:
-      LAVA_MASTER: "lava-master"
-    ports:
-      - "69:69/udp"
-      - "80:80"
-      - "61950-62000:61950-62000"
-    volumes:
-# boot and /lib/modules are for libguestfs (TODO set them read_only with docker-compose 3.0)
-      - "/boot:/boot"
-      - "/lib/modules:/lib/modules"
-    links:
-      - "lava-master"
-  squid:
-    hostname: squid
-    restart: always
-    build:
-      context: squid
-    volumes:
-      - squid-cache:/var/spool/squid
-    ports:
-      - "3128:3128"
-volumes:
-  squid-cache:
diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile
index a65fd9a..b475c8c 100644
--- a/lava-master/Dockerfile
+++ b/lava-master/Dockerfile
@@ -66,11 +66,11 @@ COPY devices/ /root/devices/
 COPY device-types/ /root/device-types/
 COPY users/ /root/lava-users/
 COPY tokens/ /root/lava-callback-tokens/
-COPY slaves/* /etc/lava-server/dispatcher.d/
 COPY scripts/setup.sh /
 RUN /start.sh && /setup.sh && /stop.sh
 
 #uncomment if you want to use squid
+# warning the address used must be network accessible by all slave
 #RUN sed -i 's,^.*http_proxy:.*,  http_proxy: http://squid:3128,' /etc/lava-server/env.yaml
 
 #comment this if you do HTTPS (For reenabling CSRF cookie)
diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh
index 5b62c45..3ab2abd 100755
--- a/lava-master/scripts/setup.sh
+++ b/lava-master/scripts/setup.sh
@@ -45,7 +45,7 @@ if [ -e /root/lava-callback-tokens ];then
 			exit 1
 		fi
 		echo "Adding $USER ($DESCRIPTION) DEBUG($TOKEN)"
-		lava-server manage tokens add --user $USER --secret $TOKEN --description $DESCRIPTION || exit 1
+		lava-server manage tokens add --user $USER --secret $TOKEN --description "$DESCRIPTION" || exit 1
 	done
 fi
 
diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile
index b5e6385..be2dbaa 100644
--- a/lava-slave/Dockerfile
+++ b/lava-slave/Dockerfile
@@ -41,7 +41,7 @@ COPY configs/lava-slave /etc/lava-dispatcher/lava-slave
 COPY configs/tftpd-hpa /etc/default/tftpd-hpa
 
 COPY scripts/cu-loop /usr/local/bin/
-COPY conmux/* /etc/conmux/
+COPY conmux/ /etc/conmux/
 
 # Caution to not use any port between the Linux dynamic port range: 32768-60999
 RUN sed -i 's,XNBD_PORT_RANGE_MIN.*,XNBD_PORT_RANGE_MIN=61950,' /usr/lib/python2.7/dist-packages/lava_dispatcher/utils/constants.py
@@ -57,6 +57,24 @@ RUN rm /etc/apt/sources.list.d/testing.list
 COPY scripts/stop.sh .
 COPY scripts/start.sh .
 
+# lava-cli dependencies
+RUN apt-get -y install python3-setuptools python3-dev python3-zmq
+RUN git clone https://git.linaro.org/lava/lavacli.git /root/lavacli && cd /root/lavacli && git checkout v0.6 && python3 setup.py install
+
+COPY phyhostname /root/
+COPY scripts/setup.sh .
+
+COPY scripts/setdispatcherip.py /usr/local/bin/
+RUN chmod 755 /usr/local/bin/setdispatcherip.py
+
+COPY scripts/retire.sh /usr/local/bin/
+RUN chmod 755 /usr/local/bin/retire.sh
+
+COPY devices/ /root/devices/
+
+COPY scripts/extra_actions /root/
+RUN if [ -x /root/extra_actions ] ; then /root/extra_actions ; fi
+
 EXPOSE 69/udp 80
 
 CMD /start.sh
diff --git a/lava-slave/scripts/extra_actions b/lava-slave/scripts/extra_actions
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lava-slave/scripts/extra_actions
diff --git a/lava-slave/scripts/retire.sh b/lava-slave/scripts/retire.sh
new file mode 100755
index 0000000..def6c8e
--- /dev/null
+++ b/lava-slave/scripts/retire.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+LAVA_MASTER_URI=$1
+
+if [ -z "$LAVA_MASTER_URI" ];then
+	echo "retire.sh: remove an offline worker"
+	echo "Usage: $0 LAVA_MASTER_URI"
+	echo "ERROR: Missing LAVA_MASTER_URI"
+	exit 11
+fi
+
+LAVACLIOPTS="--uri $LAVA_MASTER_URI"
+
+retire_worker() {
+	worker=$1
+	lavacli $LAVACLIOPTS workers list |grep -q $worker
+	if [ $? -eq 0 ];then
+		echo "Removing $worker"
+		lavacli $LAVACLIOPTS workers update $worker || exit $?
+	else
+		echo "SKIP: worker $worker does not exists"
+		return 0
+	fi
+	lavacli $LAVACLIOPTS devices list -a | grep '^\*' | cut -d' ' -f2 |
+	while read devicename
+	do
+		lavacli $LAVACLIOPTS devices show $devicename |grep -q "^worker.*$worker$"
+		if [ $? -eq 0 ];then
+			echo "Retire $devicename"
+			lavacli $LAVACLIOPTS devices update --health RETIRED --worker $worker $devicename || exit $?
+		fi
+	done
+	return 0
+}
+
+if [ -z "$2" ];then
+	for ww in $(ls devices/)
+	do
+		retire_worker $ww
+	done
+else
+	retire_worker $2
+fi
diff --git a/lava-slave/scripts/setdispatcherip.py b/lava-slave/scripts/setdispatcherip.py
new file mode 100644
index 0000000..a058cba
--- /dev/null
+++ b/lava-slave/scripts/setdispatcherip.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import xmlrpclib
+import sys
+
+if len(sys.argv) < 4:
+    print("ERROR: Usage: %s URI workername dispatcherIP" % sys.argv[0])
+    sys.exit(1)
+
+server = xmlrpclib.ServerProxy("%s" % sys.argv[1])
+server.scheduler.workers.set_config("%s" % sys.argv[2], "dispatcher_ip: %s" % sys.argv[3])
diff --git a/lava-slave/scripts/setup.sh b/lava-slave/scripts/setup.sh
new file mode 100755
index 0000000..bf91c7a
--- /dev/null
+++ b/lava-slave/scripts/setup.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+if [ ! -e "/root/devices/$(hostname)" ];then
+	echo "Static slave for $LAVA_MASTER"
+	exit 0
+fi
+
+if [ -z "$LAVA_MASTER_URI" ];then
+	echo "ERROR: Missing LAVA_MASTER_URI"
+	exit 11
+fi
+
+echo "Dynamic slave for $LAVA_MASTER ($LAVA_MASTER_URI)"
+cd /root/lavacli
+LAVACLIOPTS="--uri $LAVA_MASTER_URI"
+
+# do a sort of ping for letting master to be up
+TIMEOUT=30
+while [ $TIMEOUT -ge 1 ];
+do
+	lavacli $LAVACLIOPTS device-types list 2>/dev/null >/dev/null
+	if [ $? -eq 0 ];then
+		TIMEOUT=0
+	else
+		echo "Wait for master...."
+		sleep 2	
+	fi
+	TIMEOUT=$(($TIMEOUT-1))
+done
+
+# This directory is used for storing device-types already added
+mkdir -p /root/.lavadocker/
+if [ -e /root/device-types ];then
+	for i in $(ls /root/device-types/*jinja2)
+	do
+		devicetype=$(basename $i |sed 's,.jinja2,,')
+		echo "Adding custom $devicetype"
+		lavacli $LAVACLIOPTS device-types list || exit $?
+		touch /root/.lavadocker/devicetype-$devicetype
+	done
+fi
+
+for worker in $(ls /root/devices/)
+do
+	lavacli $LAVACLIOPTS workers list |grep -q $worker
+	if [ $? -eq 0 ];then
+		echo "Remains of $worker, cleaning it"
+		/usr/local/bin/retire.sh $LAVA_MASTER_URI $worker
+		#lavacli $LAVACLIOPTS workers update $worker || exit $?
+	else
+		echo "Adding worker $worker"
+		lavacli $LAVACLIOPTS workers add --description "LAVA dispatcher on $(cat /root/phyhostname)" $worker || exit $?
+	fi
+	if [ ! -z "$LAVA_DISPATCHER_IP" ];then
+		echo "Add dispatcher_ip $LAVA_DISPATCHER_IP to $worker"
+		/usr/local/bin/setdispatcherip.py $LAVA_MASTER_URI $worker $LAVA_DISPATCHER_IP || exit $?
+	fi
+	for device in $(ls /root/devices/$worker/)
+	do
+		devicename=$(echo $device | sed 's,.jinja2,,')
+		devicetype=$(grep -h extends /root/devices/$worker/$device| grep -o '[a-zA-Z0-9_-]*.jinja2' | sed 's,.jinja2,,')
+		if [ -e /root/.lavadocker/devicetype-$devicetype ];then
+			echo "Skip devicetype $devicetype"
+		else
+			echo "Add devicetype $devicetype"
+			lavacli $LAVACLIOPTS device-types list | grep -q "$devicetype[[:space:]]"
+			if [ $? -eq 0 ];then
+				echo "Skip devicetype $devicetype"
+			else
+				lavacli $LAVACLIOPTS device-types add $devicetype || exit $?
+			fi
+			touch /root/.lavadocker/devicetype-$devicetype
+		fi
+		echo "Add device $devicename on $worker"
+		lavacli $LAVACLIOPTS devices list -a | grep -q $devicename
+		if [ $? -eq 0 ];then
+			echo "$devicename already present"
+			#verify if present on another worker
+			#TODO
+			lavacli $LAVACLIOPTS devices show $devicename |grep ^worker |grep -q $worker
+			if [ $? -ne 0 ];then
+				echo "ERROR: $devicename already present on another worker"
+				exit 1
+			fi
+			lavacli $LAVACLIOPTS devices update --worker $worker --health UNKNOWN $devicename || exit $?
+			# always reset the device dict in case of update of it
+			lavacli $LAVACLIOPTS devices dict set $devicename /root/devices/$worker/$device || exit $?
+		else
+			lavacli $LAVACLIOPTS devices add --type $devicetype --worker $worker $devicename || exit $?
+			lavacli $LAVACLIOPTS devices dict set $devicename /root/devices/$worker/$device || exit $?
+		fi
+	done
+done
diff --git a/lava-slave/scripts/start.sh b/lava-slave/scripts/start.sh
index 92e76bb..35e58a4 100755
--- a/lava-slave/scripts/start.sh
+++ b/lava-slave/scripts/start.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+
+/setup.sh || exit $?
+
 # Set LAVA Master IP
 if [[ -n "$LAVA_MASTER" ]]; then
 	sed -i -e "s/{LAVA_MASTER}/$LAVA_MASTER/g" /etc/lava-dispatcher/lava-slave
diff --git a/lavalab-gen.py b/lavalab-gen.py
index 037b5ff..7645d08 100755
--- a/lavalab-gen.py
+++ b/lavalab-gen.py
@@ -47,130 +47,52 @@ def main():
     fp = open(boards_yaml, "r")
     workers = yaml.load(fp)
     fp.close()
-    tdc = open("docker-compose.template", "r")
-    dockcomp = yaml.load(tdc)
-    tdc.close()
-
-    # The slaves directory must exists
-    if not os.path.isdir("lava-master/slaves/"):
-        os.mkdir("lava-master/slaves/")
-        fp = open("lava-master/slaves/.empty", "w")
-        fp.close()
-    if not os.path.isdir("lava-slave/conmux/"):
-        os.mkdir("lava-slave/conmux/")
-        fp = open("lava-slave/conmux/.empty", "w")
-        fp.close()
 
-    for worker_name in workers:
-        udev_line =""
-        worker = workers[worker_name]
-        use_kvm = False
-        if "host_has_cpuflag_kvm" in worker:
-            use_kvm = worker["host_has_cpuflag_kvm"]
-        if use_kvm:
-            if "devices" in dockcomp["services"][worker_name]:
-                dc_devices = dockcomp["services"][worker_name]["devices"]
-            else:
-                dockcomp["services"][worker_name]["devices"] = []
-                dc_devices = dockcomp["services"][worker_name]["devices"]
-            dc_devices.append("/dev/kvm:/dev/kvm")
-        for board_name in worker["boardlist"]:
-            b = worker["boardlist"][board_name]
-            if b.get("disabled", None):
-                continue
-
-            devicetype = b["type"]
-            device_line = template_device.substitute(devicetype=devicetype)
-            if "pdu_generic" in b:
-                hard_reset_command = b["pdu_generic"]["hard_reset_command"]
-                power_off_command = b["pdu_generic"]["power_off_command"]
-                power_on_command = b["pdu_generic"]["power_on_command"]
-                device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
-            if "uart" in b:
-                uart = b["uart"]
-                baud = b["uart"].get("baud", baud_default)
-                idvendor = b["uart"]["idvendor"]
-                idproduct = b["uart"]["idproduct"]
-                if type(idproduct) == str:
-                    print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name,idproduct))
-                    sys.exit(1)
-                if type(idvendor) == str:
-                    print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name,idvendor))
-                    sys.exit(1)
-                line = template_conmux.substitute(board=board_name, baud=baud)
-                if "serial" in uart:
-                    serial = b["uart"]["serial"]
-                    udev_line += template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
-                else:
-                    devpath = b["uart"]["devpath"]
-                    udev_line += template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
-                if "devices" in dockcomp["services"][worker_name]:
-                    dc_devices = dockcomp["services"][worker_name]["devices"]
-                else:
-                    dockcomp["services"][worker_name]["devices"] = []
-                    dc_devices = dockcomp["services"][worker_name]["devices"]
-                dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
-                fp = open("lava-slave/conmux/%s.cf" % board_name, "w")
-                fp.write(line)
-                fp.close()
-                device_line += template_device_conmux.substitute(board=board_name)
-            elif "connection_command" in b:
-                connection_command = b["connection_command"]
-                device_line += template_device_connection_command.substitute(connection_command=connection_command)
-            if "uboot_ipaddr" in b:
-                device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % b["uboot_ipaddr"]
-            if "uboot_macaddr" in b:
-                device_line += '{% set uboot_set_mac = true %}'
-                device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % b["uboot_macaddr"]
-            if "fastboot_serial_number" in b:
-                fserial = b["fastboot_serial_number"]
-                device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
-            if "custom_option" in b:
-                for coption in b["custom_option"]:
-                    device_line += "{%% %s %%}" % coption
+    os.mkdir("output")
 
-            # board specific hacks
-            if devicetype == "qemu" and not use_kvm:
-                device_line += "{% set no_kvm = True %}\n"
-            if not os.path.isdir("lava-master/devices/"):
-                os.mkdir("lava-master/devices/")
-            device_path = "lava-master/devices/%s" % worker_name
-            if not os.path.isdir(device_path):
-                os.mkdir(device_path)
-            board_device_file = "%s/%s.jinja2" % (device_path, board_name)
-            fp = open(board_device_file, "w")
-            fp.write(device_line)
-            fp.close()
-        if not os.path.isdir("udev"):
-            os.mkdir("udev")
-        fp = open("udev/99-lavaworker-udev-%s.rules" % worker_name, "w")
-        fp.write(udev_line)
-        fp.close()
-        if "dispatcher_ip" in worker:
-            fp = open("lava-master/slaves/%s.yaml" % worker_name, "w")
-            fp.write("dispatcher_ip: %s" % worker["dispatcher_ip"])
-            fp.close()
+    if "masters" not in workers:
+        print("Missing masters entry in boards.yaml")
+        sys.exit(1)
+    masters = workers["masters"]
+    for master in masters:
+        name = master["name"]
+        print("Handle %s\n" % name)
+        if not "host" in master:
+            host = "local"
+        else:
+            host = master["host"]
+        workerdir = "output/%s/%s" % (host, name)
+        os.mkdir("output/%s" % host)
+        shutil.copy("deploy.sh", "output/%s/" % host)
+        dockcomp = {}
+        dockcomp["version"] = "2.0"
+        dockcomp["services"] = {}
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        dockcomp["services"][name] = {}
+        dockcomp["services"][name]["hostname"] = name
+        dockcomp["services"][name]["ports"] = [ "10080:80", "5555:5555", "5556:5556", "5500:5500" ]
+        dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+        dockcomp["services"][name]["build"] = {}
+        dockcomp["services"][name]["build"]["context"] = name
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
 
-    #now proceed with tokens
-    fp = open(tokens_yaml, "r")
-    tokens = yaml.load(fp)
-    fp.close()
-    if not os.path.isdir("lava-master/users/"):
-        os.mkdir("lava-master/users/")
-    if not os.path.isdir("lava-master/tokens/"):
-        os.mkdir("lava-master/tokens/")
-    for section_name in tokens:
-        section = tokens[section_name]
-        if section_name == "lava_server_users":
-            for user in section:
+        shutil.copytree("lava-master", workerdir)
+        os.mkdir("%s/devices" % workerdir)
+        # handle users / tokens
+        userdir = "%s/users" % workerdir
+        os.mkdir(userdir)
+        worker = master
+        if "users" in worker:
+            for user in worker["users"]:
                 username = user["name"]
-                ftok = open("lava-master/users/%s" % username, "w")
+                ftok = open("%s/%s" % (userdir, username), "w")
                 token = user["token"]
                 ftok.write("TOKEN=" + token + "\n")
                 if "password" in user:
                     password = user["password"]
                     ftok.write("PASSWORD=" + password + "\n")
-                # libyaml convert yes/no to true/false...
+                    # libyaml convert yes/no to true/false...
                 if "staff" in user:
                     value = user["staff"]
                     if value is True:
@@ -180,19 +102,230 @@ def main():
                     if value is True:
                         ftok.write("SUPERUSER=1\n")
                 ftok.close()
-        if section_name == "callback_tokens":
-            for token in section:
-                filename = token["filename"]
-                ftok = open("lava-master/tokens/%s" % filename, "w")
+        tokendir = "%s/tokens" % workerdir
+        os.mkdir(tokendir)
+        if "tokens" in worker:
+            filename_num = {}
+            print("Found tokens")
+            for token in worker["tokens"]:
                 username = token["username"]
+                description = token["description"]
+                if username in filename_num:
+                    number = filename_num[username]
+                    filename_num[username] = filename_num[username] + 1
+                else:
+                    filename_num[username] = 1
+                    number = 0
+                filename = "%s-%d" % (username, number)
+                print("\tAdd token for %s in %s" % (username, filename))
+                ftok = open("%s/%s" % (tokendir, filename), "w")
                 ftok.write("USER=" + username + "\n")
                 vtoken = token["token"]
                 ftok.write("TOKEN=" + vtoken + "\n")
-                description = token["description"]
-                ftok.write("DESCRIPTION=" + description)
+                ftok.write("DESCRIPTION=\"%s\"" % description)
                 ftok.close()
-    with open('docker-compose.yml', 'w') as f:
-        yaml.dump(dockcomp, f)
+
+    default_slave = "lab-slave-0"
+    if "slaves" not in workers:
+        print("Missing slaves entry in boards.yaml")
+        sys.exit(1)
+    slaves = workers["slaves"]
+    for slave in slaves:
+        name = slave["name"]
+        if len(slaves) == 1:
+            default_slave = name
+        print("Handle %s" % name)
+        if not "host" in slave:
+            host = "local"
+        else:
+            host = slave["host"]
+        if slave.get("default_slave") and slave["default_slave"]:
+             default_slave = name
+        workerdir = "output/%s/%s" % (host, name)
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        if not os.path.isdir("output/%s" % host):
+            os.mkdir("output/%s" % host)
+            shutil.copy("deploy.sh", "output/%s/" % host)
+            dockcomp = {}
+            dockcomp["version"] = "2.0"
+            dockcomp["services"] = {}
+        else:
+            #master exists
+            fp = open(dockcomposeymlpath, "r")
+            dockcomp = yaml.load(fp)
+            fp.close()
+        dockcomp["services"][name] = {}
+        dockcomp["services"][name]["hostname"] = name
+        dockcomp["services"][name]["dns_search"] = ""
+        dockcomp["services"][name]["ports"] = [ "69:69/udp", "80:80", "61950-62000:61950-62000" ]
+        dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+        dockcomp["services"][name]["environment"] = {}
+        dockcomp["services"][name]["build"] = {}
+        dockcomp["services"][name]["build"]["context"] = name
+        # insert here remote
+
+        shutil.copytree("lava-slave", workerdir)
+        fp = open("%s/phyhostname" % workerdir, "w")
+        fp.write(host)
+        fp.close()
+        conmuxpath = "%s/conmux" % workerdir
+        if not os.path.isdir(conmuxpath):
+            os.mkdir(conmuxpath)
+
+        worker = slave
+        worker_name = name
+        #NOTE remote_master is on slave
+        if not "remote_master" in worker:
+            remote_master = "lava-master"
+        else:
+            remote_master = worker["remote_master"]
+        if not "remote_address" in worker:
+            remote_address = remote_master
+        else:
+            remote_address = worker["remote_address"]
+        if not "remote_rpc_port" in worker:
+            remote_rpc_port = "80"
+        else:
+            remote_rpc_port = worker["remote_rpc_port"]
+        dockcomp["services"][worker_name]["environment"]["LAVA_MASTER"] = remote_address
+        remote_user = worker["remote_user"]
+        # find master
+        remote_token = "BAD"
+        for fm in workers["masters"]:
+            if fm["name"] == remote_master:
+                for fuser in fm["users"]:
+                    if fuser["name"] == remote_user:
+                        remote_token = fuser["token"]
+        if remote_token is "BAD":
+            print("Cannot find %s on %s" % (remote_user, remote_master))
+            sys.exit(1)
+        if not "remote_proto" in worker:
+            remote_proto = "http"
+        else:
+            remote_proto = worker["remote_proto"]
+        remote_uri = "%s://%s:%s@%s:%s/RPC2" % (remote_proto, remote_user, remote_token, remote_address, remote_rpc_port)
+        dockcomp["services"][worker_name]["environment"]["LAVA_MASTER_URI"] = remote_uri
+
+        if "dispatcher_ip" in worker:
+            dockcomp["services"][worker_name]["environment"]["LAVA_DISPATCHER_IP"] = worker["dispatcher_ip"]
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
+        if "extra_actions" in worker:
+            fp = open("%s/scripts/extra_actions" % workerdir, "w")
+            for eaction in worker["extra_actions"]:
+                fp.write(eaction)
+                fp.write("\n")
+            fp.close()
+            os.chmod("%s/scripts/extra_actions" % workerdir, 0o755)
+
+    if "boards" not in workers:
+        print("Missing boards")
+        sys.exit(1)
+    boards = workers["boards"]
+    for board in boards:
+        board_name = board["name"]
+        if "slave" in board:
+            slave_name = board["slave"]
+        else:
+            slave_name = default_slave
+        print("\tFound %s on %s" % (board_name, slave_name))
+        found_slave = False
+        for fs in workers["slaves"]:
+            if fs["name"] == slave_name:
+                slave = fs
+                found_slave = True
+        if not found_slave:
+            print("Cannot find slave %s" % slave_name)
+            sys.exit(1)
+        if not "host" in slave:
+            host = "local"
+        else:
+            host = slave["host"]
+        workerdir = "output/%s/%s" % (host, slave_name)
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        fp = open(dockcomposeymlpath, "r")
+        dockcomp = yaml.load(fp)
+        fp.close()
+        device_path = "%s/devices/" % workerdir
+        devices_path = "%s/devices/%s" % (workerdir, slave_name)
+        devicetype = board["type"]
+        device_line = template_device.substitute(devicetype=devicetype)
+        if "pdu_generic" in board:
+            hard_reset_command = board["pdu_generic"]["hard_reset_command"]
+            power_off_command = board["pdu_generic"]["power_off_command"]
+            power_on_command = board["pdu_generic"]["power_on_command"]
+            device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
+        use_kvm = False
+        if "kvm" in board:
+            use_kvm = board["kvm"]
+        if use_kvm:
+            if "devices" in dockcomp["services"][worker_name]:
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            else:
+                dockcomp["services"][worker_name]["devices"] = []
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            dc_devices.append("/dev/kvm:/dev/kvm")
+            # board specific hacks
+        if devicetype == "qemu" and not use_kvm:
+            device_line += "{% set no_kvm = True %}\n"
+        if "uart" in board:
+            uart = board["uart"]
+            baud = board["uart"].get("baud", baud_default)
+            idvendor = board["uart"]["idvendor"]
+            idproduct = board["uart"]["idproduct"]
+            if type(idproduct) == str:
+                print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name, idproduct))
+                sys.exit(1)
+            if type(idvendor) == str:
+                print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name, idvendor))
+                sys.exit(1)
+            line = template_conmux.substitute(board=board_name, baud=baud)
+            if "serial" in uart:
+                serial = board["uart"]["serial"]
+                udev_line = template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+            else:
+                devpath = board["uart"]["devpath"]
+                udev_line = template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+            if not os.path.isdir("output/%s/udev" % host):
+                os.mkdir("output/%s/udev" % host)
+            fp = open("output/%s/udev/99-lavaworker-udev.rules" % host, "a")
+            fp.write(udev_line)
+            fp.close()
+            if "devices" in dockcomp["services"][worker_name]:
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            else:
+                dockcomp["services"][worker_name]["devices"] = []
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
+            fp = open("%s/conmux/%s.cf" % (workerdir, board_name), "w")
+            fp.write(line)
+            fp.close()
+            device_line += template_device_conmux.substitute(board=board_name)
+        elif "connection_command" in board:
+            connection_command = board["connection_command"]
+            device_line += template_device_connection_command.substitute(connection_command=connection_command)
+        if "uboot_ipaddr" in board:
+            device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % board["uboot_ipaddr"]
+        if "uboot_macaddr" in board:
+            device_line += '{% set uboot_set_mac = true %}'
+            device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % board["uboot_macaddr"]
+        if "fastboot_serial_number" in board:
+            fserial = board["fastboot_serial_number"]
+            device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
+        if "custom_option" in board:
+            for coption in board["custom_option"]:
+                device_line += "{%% %s %%}" % coption
+        if not os.path.isdir(device_path):
+            os.mkdir(device_path)
+        if not os.path.isdir(devices_path):
+            os.mkdir(devices_path)
+        board_device_file = "%s/%s.jinja2" % (devices_path, board_name)
+        fp = open(board_device_file, "w")
+        fp.write(device_line)
+        fp.close()
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
+
 
 if __name__ == "__main__":
     shutil.copy("common/build-lava", "lava-slave/scripts/build-lava")
diff --git a/lavalab-gen.sh b/lavalab-gen.sh
index 60c5d94..08ab2ad 100755
--- a/lavalab-gen.sh
+++ b/lavalab-gen.sh
@@ -1,30 +1,11 @@
 #!/bin/sh
 
-rm -rv lava-master/devices/
-rm -rv lava-master/slaves/
-rm -rv lava-slave/conmux/
-rm -rv lava-master/tokens/
-rm -rv lava-master/users/
-rm udev/*lavalab*rules
 rm lava-master/scripts/build-lava
 rm lava-slave/scripts/build-lava
-rm docker-compose.yml
+rm -r output
 
 if [ "$1" = "mrproper" ];then
 	exit 0
 fi
 
 ./lavalab-gen.py || exit 1
-
-#check for root
-BEROOT=""
-if [ $(id -u) -ne 0 ];then
-	BEROOT="sudo "
-fi
-$BEROOT rm /etc/udev/rules.d/*lava*rules
-$BEROOT cp udev/*lava*rules /etc/udev/rules.d/
-$BEROOT udevadm control --reload-rules || exit $?
-$BEROOT udevadm trigger || exit $?
-
-docker-compose build || exit 1
-docker-compose up -d || exit 1
diff --git a/tokens.yaml b/tokens.yaml
deleted file mode 100644
index d3a8760..0000000
--- a/tokens.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-lava_server_users:
-  - name: admin
-    token: longrandomtokenadmin
-    password: admin
-    superuser: yes
-    staff: yes
-  - name: example
-    token: longrandomtoken
-    password: examplepassword
-
-callback_tokens:
-  - filename: example
-    username: example
-    token: longrandomtokencallback
-    description: example-callback