diff options
-rw-r--r-- | README.md | 126 | ||||
-rw-r--r-- | boards.yaml | 26 | ||||
-rw-r--r-- | boards.yaml.example | 113 | ||||
-rwxr-xr-x | deploy.sh | 14 | ||||
-rw-r--r-- | docker-compose.template | 45 | ||||
-rw-r--r-- | lava-master/Dockerfile | 2 | ||||
-rwxr-xr-x | lava-master/scripts/setup.sh | 2 | ||||
-rw-r--r-- | lava-slave/Dockerfile | 20 | ||||
-rw-r--r-- | lava-slave/scripts/extra_actions | 0 | ||||
-rwxr-xr-x | lava-slave/scripts/retire.sh | 43 | ||||
-rw-r--r-- | lava-slave/scripts/setdispatcherip.py | 11 | ||||
-rwxr-xr-x | lava-slave/scripts/setup.sh | 93 | ||||
-rwxr-xr-x | lava-slave/scripts/start.sh | 3 | ||||
-rwxr-xr-x | lavalab-gen.py | 379 | ||||
-rwxr-xr-x | lavalab-gen.sh | 21 | ||||
-rw-r--r-- | tokens.yaml | 15 |
16 files changed, 576 insertions, 337 deletions
@@ -26,17 +26,11 @@ The following packages are necessary on the host machine: Example to use lava-docker with only one QEMU device: * Checkout the lava-docker repository -* You will obtain the following boards.yaml -``` -lab-slave-0: - boardlist: - qemu-01: - type: qemu -``` * Generate configuration files for LAVA, udev, serial ports, etc. from boards.yaml via ``` ./lavalab-gen.py ``` +* Go to output/local directory * Build docker images via ``` docker-compose build @@ -65,9 +59,9 @@ You could check in https://github.com/Linaro/lava-server/tree/release/lava_sched Example: For a beagleboneblack, the device-type is beaglebone-black (Even if official DT name is am335x-boneblack) -So you have now: +So you need to add in the boards section: ``` - beagleboneblack-01: + - name: beagleboneblack-01 type: beaglebone-black ``` @@ -87,7 +81,7 @@ The serial is AK04TU1X ``` So you have now: ``` - beagleboneblack-01: + - name: beagleboneblack-01 type: beaglebone-black uart: idvendor: 0x0403 @@ -105,7 +99,7 @@ udevadm info -a -n /dev/ttyUSB1 |grep devpath | head -n1 ``` So you have now: ``` - beagleboneblack-01: + - name: beagleboneblack-01 type: beaglebone-black uart: idvendor: 0x0403 @@ -130,7 +124,7 @@ If the beagleboneblack is wired to port 3 and the ACME board have IP 192.168.66. #### Example: beagleboneblack, with FTDI (serial 1234567), connected to port 5 of an ACME ``` - beagleboneblack-01: + - name: beagleboneblack-01 type: beaglebone-black pdu_generic: hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 5 @@ -142,14 +136,11 @@ beagleboneblack, with FTDI (serial 1234567), connected to port 5 of an ACME serial: 1234567 ``` -## Known limitations -The current lava-docker provides support for generating only one LAVA slave. - ## Architecture -The setup is composed of a host which runs the following docker images and DUT to be tested.<br/> +The basic setup is composed of a host which runs the following docker images and DUT to be tested.<br/> * lava-master: run lava-server along with the web interface * lava-slave: run lava-dispatcher, the compoment which sends jobs to DUTs -* squid: an HTTP proxy for caching downloaded contents (kernel/dtb/rootfs) +* squid: an HTTP proxy for caching downloaded contents (kernel/dtb/rootfs) (Work in progress) The host and DUTs must share a common LAN.<br/> The host IP on this LAN must be set as dispatcher_ip in boards.yaml.<br/> @@ -159,6 +150,12 @@ So, on the LAN shared with DUTs, a running DHCPD is necessary. (See DHCPD below) ![lava-docker diagram](doc/lava-docker.png) +## Multi-host architectures +Lava-docker support multi-host architecture, Master and slaves could be on different host. + +Lava-docker support multiples slaves, but with a maximum of one slave per host. +This is due to that slave need TFTP port accessible from outside. + ### Power supply You need to have a PDU for powering your DUT. Managing PDUs is done via pdu_generic @@ -214,25 +211,50 @@ You can use the lavalab-gen.sh helper script which will do all the above actions ### boards.yaml This file describe how the DUTs are connected and powered. ``` -lab-slave-XX: The name of the slave (where XX is a number) - dispatcher_ip: the IP where the slave could be contacted. In lava-docker it is the host IP since docker proxify TFTP from host to the slave. - host_has_cpuflag_kvm: Does the host running lab-slave-XX have KVM - boardlist: - devicename: Each board must be named by their device-type as "device-type-XX" (where XX is a number) - type: the LAVA device-type of this device - uboot_ipaddr: (optional) a static IP to set in uboot - uboot_macaddr: (Optional) the MAC address to set in uboot +masters: + - name: lava-master name of the master + host: name name of the host running lava-master (default to "local") + users: + - name: LAVA username + token: The token of this user + password: Password the this user (generated if not provided) + superuser: yes/no (default no) + staff: yes/no (default no) + tokens: + - username: The LAVA user owning the token below. (This user should be created via users:) + token: The token for this callback + description: The description of this token. This string could be used with LAVA-CI. +slaves: + - name: lab-slave-XX The name of the slave (where XX is a number) + host: name name of the host running lava-slave-XX (default to "local") + dispatcher_ip: the IP where the slave could be contacted. In lava-docker it is the host IP since docker proxify TFTP from host to the slave. + remote_master: the name of the master to connect to + remote_address: the FQDN or IP address of the master (if different from remote_master) + remote_rpc_port: the port used by the LAVA RPC2 (default 80) + remote_user: the user used for connecting to the master + remote_proto: http(default) or https + extra_actions: An optional list of action to do at end of the docker build + - "apt-get install package" + +boards: + - name: devicename Each board must be named by their device-type as "device-type-XX" (where XX is a number) + type: the LAVA device-type of this device + kvm: (For qemu only) Does the qemu could use KVM (default: no) + uboot_ipaddr: (optional) a static IP to set in uboot + uboot_macaddr: (Optional) the MAC address to set in uboot + custom_option: (optional) All following strings will be directly append to devicefile + - "set x=1" # One of uart or connection_command must be choosen - uart: - idvendor: The VID of the UART (Formated as 0xXXXX) - idproduct: the PID of the UART (Formated as 0xXXXX) - serial: The serial number in case of FTDI uart - devpath: the UDEV devpath to this uart for UART without serial number - connection_command: A command to be ran for getting a serial console - pdu_generic: - hard_reset_command: commandline to reset the board - power_off_command: commandline to power off the board - power_on_command: commandline to power on the board + uart: + idvendor: The VID of the UART (Formated as 0xXXXX) + idproduct: the PID of the UART (Formated as 0xXXXX) + serial: The serial number in case of FTDI uart + devpath: the UDEV devpath to this uart for UART without serial number + connection_command: A command to be ran for getting a serial console + pdu_generic: + hard_reset_command: commandline to reset the board + power_off_command: commandline to power off the board + power_on_command: commandline to power on the board ``` Notes on UART: * Only one of devpath/serial is necessary. @@ -251,23 +273,6 @@ Note on connection_command: connection_command is for people which want to use o Examples: see [boards.yaml.example](boards.yaml.example) -### tokens.yaml -The tokens format has two sections, one for LAVA users, the other for callback tokens -``` -lava_server_users: - - name: LAVA username - token: The token of this user - password: Password the this user (generated if not provided) - superuser: yes/no (default no) - staff: yes/no (default no) -callback_tokens: - - filename: The filename for storing the informations below, the name should be unique along other callback tokens - username: The LAVA user owning the token below. (This user should be created via lava_server_users:) - token: The token for this callback - description: The description of this token. This string could be used with LAVA-CI. -``` -Example: see [tokens.yaml](tokens.yaml) - ### Generate ``` lavalab-gen.py @@ -275,13 +280,12 @@ lavalab-gen.py this script will generate all necessary files in the following locations: ``` -conmux/ All files needed by conmux -tokens/ This is where the callback tokens will be generated -users/ This is where the users will be generated -devices/ All LAVA devices files -slaves/ Contain the dispatcher_ip to give to slave node -udev-rules for host -docker-compose.yml Generated from docker-compose.template +output/host/lava-master/tokens/ This is where the callback tokens will be generated +output/host/lava-master/users/ This is where the users will be generated +output/host/lab-slave-XX/conmux/ All files needed by conmux +output/host/lab-slave-XX/devices/ All LAVA devices files +output/host/udev/99-lavaworker-udev.rules udev rules for host +output/host/docker-compose.yml Generated from docker-compose.template ``` All thoses file (except for udev-rules) will be handled by docker. @@ -306,10 +310,10 @@ For running all images, simply run: docker-compose up -d ``` -## Proxy cache +## Proxy cache (Work in progress) A squid docker is provided for caching all LAVA downloads (image, dtb, rootfs, etc...)<br/> You have to uncomment a line in lava-master/Dockerfile to enable it.<br/> -Note that the squid proxy is always built and run. +For the moment, it is unsupported and unbuilded. ## Security Note that this container provides defaults which are unsecure. If you plan on deploying this in a production enviroment please consider the following items: diff --git a/boards.yaml b/boards.yaml index 88f7e6d..df6548c 100644 --- a/boards.yaml +++ b/boards.yaml @@ -1,4 +1,22 @@ -lab-slave-0: - boardlist: - qemu-01: - type: qemu +masters: + - name: master1 + host: local + users: + - name: admin + token: longrandomtokenadmin + password: admin + superuser: yes + staff: yes + tokens: + - username: admin + token: dfjdfkfkdjfkdsjfsl + description: no description +slaves: + - name: lab-slave-0 + host: local + remote_master: master1 + remote_user: admin + +boards: + - name: qemu-01 + type: qemu diff --git a/boards.yaml.example b/boards.yaml.example index 5737a48..ee4469f 100644 --- a/boards.yaml.example +++ b/boards.yaml.example @@ -1,66 +1,47 @@ -lab-slave-0: - dispatcher_ip: 192.168.66.1 - boardlist: - qemu-01: - type: qemu - bcm2837-rpi-3-b-01: - type: bcm2837-rpi-3-b - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 1 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 1 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 1 - uart: - idvendor: 0x0403 - idproduct: 0x6001 - serial: FT9QQZTA - am335x-boneblack-01: - type: beaglebone-black - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 2 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 2 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 2 - uart: - idvendor: 0x0403 - idproduct: 0x6001 - serial: FT9QR1A9 - meson-gxl-s905x-libretech-cc-01: - type: meson-gxl-s905x-libretech-cc - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 3 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 3 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 3 - uart: - idvendor: 0x067b - idproduct: 0x2303 - devpath: 1.1.4 - uboot_macaddr: "00:FA:E0:DE:AD:78" - dragonboard-410c-01: - type: dragonboard-410c - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 4 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 4 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 4 - uart: - idvendor: 0x403 - idproduct: 0x6001 - serial: FT9R7VDB - r8a7796-m3ulcb-01: - type: r8a7796-m3ulcb - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 5 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 5 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 5 - uart: - idvendor: 0x0403 - idproduct: 0x6001 - serial: AK04WW0Q - imx6q-sabrelite-01: - type: imx6q-sabrelite - pdu_generic: - hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 6 - power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 6 - power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 6 - uart: - idvendor: 0x0403 - idproduct: 0x6015 - serial: DAZ0KEUH +masters: + - name: master1 + host: local + users: + - name: admin + token: longrandomtokenadmin + password: admin + superuser: yes + staff: yes +slaves: + - name: lab-slave-0 + host: local + remote_master: master1 + remote_user: admin + dispatcher_ip: 192.168.66.1 + +boards: + - name: qemu-02 + type: qemu + slave: lab-slave-0 + kvm: True + - name: meson-gxl-s905x-libretech-cc-01 + type: meson-gxl-s905x-libretech-cc + slave: lab-slave-0 + pdu_generic: + hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 2 + power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 2 + power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 2 + uart: + idvendor: 0x0403 + idproduct: 0x6001 + serial: FT9QR2TZ + - name: meson-gxbb-nanopi-k2-01 + type: meson-gxbb-nanopi-k2 + slave: lab-slave-0 + custom_option: + - 'set bootloader_prompt = "nanopi-k2#"' + - "set interrupt_prompt = 'nanopi'" + uboot_ipaddr: 192.168.66.201 + pdu_generic: + hard_reset_command: /usr/local/bin/acme-cli -s 192.168.66.2 reset 3 + power_off_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_off 3 + power_on_command: /usr/local/bin/acme-cli -s 192.168.66.2 switch_on 3 + uart: + idvendor: 0x0403 + idproduct: 0x6001 + serial: FT9ZOR0I diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..f7be5be --- /dev/null +++ b/deploy.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +#check for root +BEROOT="" +if [ $(id -u) -ne 0 ];then + BEROOT="sudo " +fi +$BEROOT rm /etc/udev/rules.d/*lava*rules +$BEROOT cp udev/*lava*rules /etc/udev/rules.d/ +$BEROOT udevadm control --reload-rules || exit $? +$BEROOT udevadm trigger || exit $? + +docker-compose build || exit 1 +docker-compose up -d || exit 1 diff --git a/docker-compose.template b/docker-compose.template deleted file mode 100644 index ced5762..0000000 --- a/docker-compose.template +++ /dev/null @@ -1,45 +0,0 @@ -version: '2.0' -services: - lava-master: - hostname: lava-master - restart: always - build: - context: lava-master - ports: - - "10080:80" - - "5555:5555" - - "5556:5556" - volumes: -# boot and /lib/modules are for libguestfs (TODO set them read_only with docker-compose 3.0) - - "/boot:/boot" - - "/lib/modules:/lib/modules" - lab-slave-0: - hostname: lab-slave-0 -#conmux does not support dns_search - dns_search: "" - restart: always - build: - context: lava-slave - environment: - LAVA_MASTER: "lava-master" - ports: - - "69:69/udp" - - "80:80" - - "61950-62000:61950-62000" - volumes: -# boot and /lib/modules are for libguestfs (TODO set them read_only with docker-compose 3.0) - - "/boot:/boot" - - "/lib/modules:/lib/modules" - links: - - "lava-master" - squid: - hostname: squid - restart: always - build: - context: squid - volumes: - - squid-cache:/var/spool/squid - ports: - - "3128:3128" -volumes: - squid-cache: diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile index a65fd9a..b475c8c 100644 --- a/lava-master/Dockerfile +++ b/lava-master/Dockerfile @@ -66,11 +66,11 @@ COPY devices/ /root/devices/ COPY device-types/ /root/device-types/ COPY users/ /root/lava-users/ COPY tokens/ /root/lava-callback-tokens/ -COPY slaves/* /etc/lava-server/dispatcher.d/ COPY scripts/setup.sh / RUN /start.sh && /setup.sh && /stop.sh #uncomment if you want to use squid +# warning the address used must be network accessible by all slave #RUN sed -i 's,^.*http_proxy:.*, http_proxy: http://squid:3128,' /etc/lava-server/env.yaml #comment this if you do HTTPS (For reenabling CSRF cookie) diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh index 5b62c45..3ab2abd 100755 --- a/lava-master/scripts/setup.sh +++ b/lava-master/scripts/setup.sh @@ -45,7 +45,7 @@ if [ -e /root/lava-callback-tokens ];then exit 1 fi echo "Adding $USER ($DESCRIPTION) DEBUG($TOKEN)" - lava-server manage tokens add --user $USER --secret $TOKEN --description $DESCRIPTION || exit 1 + lava-server manage tokens add --user $USER --secret $TOKEN --description "$DESCRIPTION" || exit 1 done fi diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index b5e6385..be2dbaa 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -41,7 +41,7 @@ COPY configs/lava-slave /etc/lava-dispatcher/lava-slave COPY configs/tftpd-hpa /etc/default/tftpd-hpa COPY scripts/cu-loop /usr/local/bin/ -COPY conmux/* /etc/conmux/ +COPY conmux/ /etc/conmux/ # Caution to not use any port between the Linux dynamic port range: 32768-60999 RUN sed -i 's,XNBD_PORT_RANGE_MIN.*,XNBD_PORT_RANGE_MIN=61950,' /usr/lib/python2.7/dist-packages/lava_dispatcher/utils/constants.py @@ -57,6 +57,24 @@ RUN rm /etc/apt/sources.list.d/testing.list COPY scripts/stop.sh . COPY scripts/start.sh . +# lava-cli dependencies +RUN apt-get -y install python3-setuptools python3-dev python3-zmq +RUN git clone https://git.linaro.org/lava/lavacli.git /root/lavacli && cd /root/lavacli && git checkout v0.6 && python3 setup.py install + +COPY phyhostname /root/ +COPY scripts/setup.sh . + +COPY scripts/setdispatcherip.py /usr/local/bin/ +RUN chmod 755 /usr/local/bin/setdispatcherip.py + +COPY scripts/retire.sh /usr/local/bin/ +RUN chmod 755 /usr/local/bin/retire.sh + +COPY devices/ /root/devices/ + +COPY scripts/extra_actions /root/ +RUN if [ -x /root/extra_actions ] ; then /root/extra_actions ; fi + EXPOSE 69/udp 80 CMD /start.sh diff --git a/lava-slave/scripts/extra_actions b/lava-slave/scripts/extra_actions new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lava-slave/scripts/extra_actions diff --git a/lava-slave/scripts/retire.sh b/lava-slave/scripts/retire.sh new file mode 100755 index 0000000..def6c8e --- /dev/null +++ b/lava-slave/scripts/retire.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +LAVA_MASTER_URI=$1 + +if [ -z "$LAVA_MASTER_URI" ];then + echo "retire.sh: remove an offline worker" + echo "Usage: $0 LAVA_MASTER_URI" + echo "ERROR: Missing LAVA_MASTER_URI" + exit 11 +fi + +LAVACLIOPTS="--uri $LAVA_MASTER_URI" + +retire_worker() { + worker=$1 + lavacli $LAVACLIOPTS workers list |grep -q $worker + if [ $? -eq 0 ];then + echo "Removing $worker" + lavacli $LAVACLIOPTS workers update $worker || exit $? + else + echo "SKIP: worker $worker does not exists" + return 0 + fi + lavacli $LAVACLIOPTS devices list -a | grep '^\*' | cut -d' ' -f2 | + while read devicename + do + lavacli $LAVACLIOPTS devices show $devicename |grep -q "^worker.*$worker$" + if [ $? -eq 0 ];then + echo "Retire $devicename" + lavacli $LAVACLIOPTS devices update --health RETIRED --worker $worker $devicename || exit $? + fi + done + return 0 +} + +if [ -z "$2" ];then + for ww in $(ls devices/) + do + retire_worker $ww + done +else + retire_worker $2 +fi diff --git a/lava-slave/scripts/setdispatcherip.py b/lava-slave/scripts/setdispatcherip.py new file mode 100644 index 0000000..a058cba --- /dev/null +++ b/lava-slave/scripts/setdispatcherip.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import xmlrpclib +import sys + +if len(sys.argv) < 4: + print("ERROR: Usage: %s URI workername dispatcherIP" % sys.argv[0]) + sys.exit(1) + +server = xmlrpclib.ServerProxy("%s" % sys.argv[1]) +server.scheduler.workers.set_config("%s" % sys.argv[2], "dispatcher_ip: %s" % sys.argv[3]) diff --git a/lava-slave/scripts/setup.sh b/lava-slave/scripts/setup.sh new file mode 100755 index 0000000..bf91c7a --- /dev/null +++ b/lava-slave/scripts/setup.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +if [ ! -e "/root/devices/$(hostname)" ];then + echo "Static slave for $LAVA_MASTER" + exit 0 +fi + +if [ -z "$LAVA_MASTER_URI" ];then + echo "ERROR: Missing LAVA_MASTER_URI" + exit 11 +fi + +echo "Dynamic slave for $LAVA_MASTER ($LAVA_MASTER_URI)" +cd /root/lavacli +LAVACLIOPTS="--uri $LAVA_MASTER_URI" + +# do a sort of ping for letting master to be up +TIMEOUT=30 +while [ $TIMEOUT -ge 1 ]; +do + lavacli $LAVACLIOPTS device-types list 2>/dev/null >/dev/null + if [ $? -eq 0 ];then + TIMEOUT=0 + else + echo "Wait for master...." + sleep 2 + fi + TIMEOUT=$(($TIMEOUT-1)) +done + +# This directory is used for storing device-types already added +mkdir -p /root/.lavadocker/ +if [ -e /root/device-types ];then + for i in $(ls /root/device-types/*jinja2) + do + devicetype=$(basename $i |sed 's,.jinja2,,') + echo "Adding custom $devicetype" + lavacli $LAVACLIOPTS device-types list || exit $? + touch /root/.lavadocker/devicetype-$devicetype + done +fi + +for worker in $(ls /root/devices/) +do + lavacli $LAVACLIOPTS workers list |grep -q $worker + if [ $? -eq 0 ];then + echo "Remains of $worker, cleaning it" + /usr/local/bin/retire.sh $LAVA_MASTER_URI $worker + #lavacli $LAVACLIOPTS workers update $worker || exit $? + else + echo "Adding worker $worker" + lavacli $LAVACLIOPTS workers add --description "LAVA dispatcher on $(cat /root/phyhostname)" $worker || exit $? + fi + if [ ! -z "$LAVA_DISPATCHER_IP" ];then + echo "Add dispatcher_ip $LAVA_DISPATCHER_IP to $worker" + /usr/local/bin/setdispatcherip.py $LAVA_MASTER_URI $worker $LAVA_DISPATCHER_IP || exit $? + fi + for device in $(ls /root/devices/$worker/) + do + devicename=$(echo $device | sed 's,.jinja2,,') + devicetype=$(grep -h extends /root/devices/$worker/$device| grep -o '[a-zA-Z0-9_-]*.jinja2' | sed 's,.jinja2,,') + if [ -e /root/.lavadocker/devicetype-$devicetype ];then + echo "Skip devicetype $devicetype" + else + echo "Add devicetype $devicetype" + lavacli $LAVACLIOPTS device-types list | grep -q "$devicetype[[:space:]]" + if [ $? -eq 0 ];then + echo "Skip devicetype $devicetype" + else + lavacli $LAVACLIOPTS device-types add $devicetype || exit $? + fi + touch /root/.lavadocker/devicetype-$devicetype + fi + echo "Add device $devicename on $worker" + lavacli $LAVACLIOPTS devices list -a | grep -q $devicename + if [ $? -eq 0 ];then + echo "$devicename already present" + #verify if present on another worker + #TODO + lavacli $LAVACLIOPTS devices show $devicename |grep ^worker |grep -q $worker + if [ $? -ne 0 ];then + echo "ERROR: $devicename already present on another worker" + exit 1 + fi + lavacli $LAVACLIOPTS devices update --worker $worker --health UNKNOWN $devicename || exit $? + # always reset the device dict in case of update of it + lavacli $LAVACLIOPTS devices dict set $devicename /root/devices/$worker/$device || exit $? + else + lavacli $LAVACLIOPTS devices add --type $devicetype --worker $worker $devicename || exit $? + lavacli $LAVACLIOPTS devices dict set $devicename /root/devices/$worker/$device || exit $? + fi + done +done diff --git a/lava-slave/scripts/start.sh b/lava-slave/scripts/start.sh index 92e76bb..35e58a4 100755 --- a/lava-slave/scripts/start.sh +++ b/lava-slave/scripts/start.sh @@ -1,4 +1,7 @@ #!/bin/bash + +/setup.sh || exit $? + # Set LAVA Master IP if [[ -n "$LAVA_MASTER" ]]; then sed -i -e "s/{LAVA_MASTER}/$LAVA_MASTER/g" /etc/lava-dispatcher/lava-slave diff --git a/lavalab-gen.py b/lavalab-gen.py index 037b5ff..7645d08 100755 --- a/lavalab-gen.py +++ b/lavalab-gen.py @@ -47,130 +47,52 @@ def main(): fp = open(boards_yaml, "r") workers = yaml.load(fp) fp.close() - tdc = open("docker-compose.template", "r") - dockcomp = yaml.load(tdc) - tdc.close() - - # The slaves directory must exists - if not os.path.isdir("lava-master/slaves/"): - os.mkdir("lava-master/slaves/") - fp = open("lava-master/slaves/.empty", "w") - fp.close() - if not os.path.isdir("lava-slave/conmux/"): - os.mkdir("lava-slave/conmux/") - fp = open("lava-slave/conmux/.empty", "w") - fp.close() - for worker_name in workers: - udev_line ="" - worker = workers[worker_name] - use_kvm = False - if "host_has_cpuflag_kvm" in worker: - use_kvm = worker["host_has_cpuflag_kvm"] - if use_kvm: - if "devices" in dockcomp["services"][worker_name]: - dc_devices = dockcomp["services"][worker_name]["devices"] - else: - dockcomp["services"][worker_name]["devices"] = [] - dc_devices = dockcomp["services"][worker_name]["devices"] - dc_devices.append("/dev/kvm:/dev/kvm") - for board_name in worker["boardlist"]: - b = worker["boardlist"][board_name] - if b.get("disabled", None): - continue - - devicetype = b["type"] - device_line = template_device.substitute(devicetype=devicetype) - if "pdu_generic" in b: - hard_reset_command = b["pdu_generic"]["hard_reset_command"] - power_off_command = b["pdu_generic"]["power_off_command"] - power_on_command = b["pdu_generic"]["power_on_command"] - device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command) - if "uart" in b: - uart = b["uart"] - baud = b["uart"].get("baud", baud_default) - idvendor = b["uart"]["idvendor"] - idproduct = b["uart"]["idproduct"] - if type(idproduct) == str: - print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name,idproduct)) - sys.exit(1) - if type(idvendor) == str: - print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name,idvendor)) - sys.exit(1) - line = template_conmux.substitute(board=board_name, baud=baud) - if "serial" in uart: - serial = b["uart"]["serial"] - udev_line += template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) - else: - devpath = b["uart"]["devpath"] - udev_line += template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) - if "devices" in dockcomp["services"][worker_name]: - dc_devices = dockcomp["services"][worker_name]["devices"] - else: - dockcomp["services"][worker_name]["devices"] = [] - dc_devices = dockcomp["services"][worker_name]["devices"] - dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name)) - fp = open("lava-slave/conmux/%s.cf" % board_name, "w") - fp.write(line) - fp.close() - device_line += template_device_conmux.substitute(board=board_name) - elif "connection_command" in b: - connection_command = b["connection_command"] - device_line += template_device_connection_command.substitute(connection_command=connection_command) - if "uboot_ipaddr" in b: - device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % b["uboot_ipaddr"] - if "uboot_macaddr" in b: - device_line += '{% set uboot_set_mac = true %}' - device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % b["uboot_macaddr"] - if "fastboot_serial_number" in b: - fserial = b["fastboot_serial_number"] - device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial - if "custom_option" in b: - for coption in b["custom_option"]: - device_line += "{%% %s %%}" % coption + os.mkdir("output") - # board specific hacks - if devicetype == "qemu" and not use_kvm: - device_line += "{% set no_kvm = True %}\n" - if not os.path.isdir("lava-master/devices/"): - os.mkdir("lava-master/devices/") - device_path = "lava-master/devices/%s" % worker_name - if not os.path.isdir(device_path): - os.mkdir(device_path) - board_device_file = "%s/%s.jinja2" % (device_path, board_name) - fp = open(board_device_file, "w") - fp.write(device_line) - fp.close() - if not os.path.isdir("udev"): - os.mkdir("udev") - fp = open("udev/99-lavaworker-udev-%s.rules" % worker_name, "w") - fp.write(udev_line) - fp.close() - if "dispatcher_ip" in worker: - fp = open("lava-master/slaves/%s.yaml" % worker_name, "w") - fp.write("dispatcher_ip: %s" % worker["dispatcher_ip"]) - fp.close() + if "masters" not in workers: + print("Missing masters entry in boards.yaml") + sys.exit(1) + masters = workers["masters"] + for master in masters: + name = master["name"] + print("Handle %s\n" % name) + if not "host" in master: + host = "local" + else: + host = master["host"] + workerdir = "output/%s/%s" % (host, name) + os.mkdir("output/%s" % host) + shutil.copy("deploy.sh", "output/%s/" % host) + dockcomp = {} + dockcomp["version"] = "2.0" + dockcomp["services"] = {} + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + dockcomp["services"][name] = {} + dockcomp["services"][name]["hostname"] = name + dockcomp["services"][name]["ports"] = [ "10080:80", "5555:5555", "5556:5556", "5500:5500" ] + dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ] + dockcomp["services"][name]["build"] = {} + dockcomp["services"][name]["build"]["context"] = name + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) - #now proceed with tokens - fp = open(tokens_yaml, "r") - tokens = yaml.load(fp) - fp.close() - if not os.path.isdir("lava-master/users/"): - os.mkdir("lava-master/users/") - if not os.path.isdir("lava-master/tokens/"): - os.mkdir("lava-master/tokens/") - for section_name in tokens: - section = tokens[section_name] - if section_name == "lava_server_users": - for user in section: + shutil.copytree("lava-master", workerdir) + os.mkdir("%s/devices" % workerdir) + # handle users / tokens + userdir = "%s/users" % workerdir + os.mkdir(userdir) + worker = master + if "users" in worker: + for user in worker["users"]: username = user["name"] - ftok = open("lava-master/users/%s" % username, "w") + ftok = open("%s/%s" % (userdir, username), "w") token = user["token"] ftok.write("TOKEN=" + token + "\n") if "password" in user: password = user["password"] ftok.write("PASSWORD=" + password + "\n") - # libyaml convert yes/no to true/false... + # libyaml convert yes/no to true/false... if "staff" in user: value = user["staff"] if value is True: @@ -180,19 +102,230 @@ def main(): if value is True: ftok.write("SUPERUSER=1\n") ftok.close() - if section_name == "callback_tokens": - for token in section: - filename = token["filename"] - ftok = open("lava-master/tokens/%s" % filename, "w") + tokendir = "%s/tokens" % workerdir + os.mkdir(tokendir) + if "tokens" in worker: + filename_num = {} + print("Found tokens") + for token in worker["tokens"]: username = token["username"] + description = token["description"] + if username in filename_num: + number = filename_num[username] + filename_num[username] = filename_num[username] + 1 + else: + filename_num[username] = 1 + number = 0 + filename = "%s-%d" % (username, number) + print("\tAdd token for %s in %s" % (username, filename)) + ftok = open("%s/%s" % (tokendir, filename), "w") ftok.write("USER=" + username + "\n") vtoken = token["token"] ftok.write("TOKEN=" + vtoken + "\n") - description = token["description"] - ftok.write("DESCRIPTION=" + description) + ftok.write("DESCRIPTION=\"%s\"" % description) ftok.close() - with open('docker-compose.yml', 'w') as f: - yaml.dump(dockcomp, f) + + default_slave = "lab-slave-0" + if "slaves" not in workers: + print("Missing slaves entry in boards.yaml") + sys.exit(1) + slaves = workers["slaves"] + for slave in slaves: + name = slave["name"] + if len(slaves) == 1: + default_slave = name + print("Handle %s" % name) + if not "host" in slave: + host = "local" + else: + host = slave["host"] + if slave.get("default_slave") and slave["default_slave"]: + default_slave = name + workerdir = "output/%s/%s" % (host, name) + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + if not os.path.isdir("output/%s" % host): + os.mkdir("output/%s" % host) + shutil.copy("deploy.sh", "output/%s/" % host) + dockcomp = {} + dockcomp["version"] = "2.0" + dockcomp["services"] = {} + else: + #master exists + fp = open(dockcomposeymlpath, "r") + dockcomp = yaml.load(fp) + fp.close() + dockcomp["services"][name] = {} + dockcomp["services"][name]["hostname"] = name + dockcomp["services"][name]["dns_search"] = "" + dockcomp["services"][name]["ports"] = [ "69:69/udp", "80:80", "61950-62000:61950-62000" ] + dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ] + dockcomp["services"][name]["environment"] = {} + dockcomp["services"][name]["build"] = {} + dockcomp["services"][name]["build"]["context"] = name + # insert here remote + + shutil.copytree("lava-slave", workerdir) + fp = open("%s/phyhostname" % workerdir, "w") + fp.write(host) + fp.close() + conmuxpath = "%s/conmux" % workerdir + if not os.path.isdir(conmuxpath): + os.mkdir(conmuxpath) + + worker = slave + worker_name = name + #NOTE remote_master is on slave + if not "remote_master" in worker: + remote_master = "lava-master" + else: + remote_master = worker["remote_master"] + if not "remote_address" in worker: + remote_address = remote_master + else: + remote_address = worker["remote_address"] + if not "remote_rpc_port" in worker: + remote_rpc_port = "80" + else: + remote_rpc_port = worker["remote_rpc_port"] + dockcomp["services"][worker_name]["environment"]["LAVA_MASTER"] = remote_address + remote_user = worker["remote_user"] + # find master + remote_token = "BAD" + for fm in workers["masters"]: + if fm["name"] == remote_master: + for fuser in fm["users"]: + if fuser["name"] == remote_user: + remote_token = fuser["token"] + if remote_token is "BAD": + print("Cannot find %s on %s" % (remote_user, remote_master)) + sys.exit(1) + if not "remote_proto" in worker: + remote_proto = "http" + else: + remote_proto = worker["remote_proto"] + remote_uri = "%s://%s:%s@%s:%s/RPC2" % (remote_proto, remote_user, remote_token, remote_address, remote_rpc_port) + dockcomp["services"][worker_name]["environment"]["LAVA_MASTER_URI"] = remote_uri + + if "dispatcher_ip" in worker: + dockcomp["services"][worker_name]["environment"]["LAVA_DISPATCHER_IP"] = worker["dispatcher_ip"] + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) + if "extra_actions" in worker: + fp = open("%s/scripts/extra_actions" % workerdir, "w") + for eaction in worker["extra_actions"]: + fp.write(eaction) + fp.write("\n") + fp.close() + os.chmod("%s/scripts/extra_actions" % workerdir, 0o755) + + if "boards" not in workers: + print("Missing boards") + sys.exit(1) + boards = workers["boards"] + for board in boards: + board_name = board["name"] + if "slave" in board: + slave_name = board["slave"] + else: + slave_name = default_slave + print("\tFound %s on %s" % (board_name, slave_name)) + found_slave = False + for fs in workers["slaves"]: + if fs["name"] == slave_name: + slave = fs + found_slave = True + if not found_slave: + print("Cannot find slave %s" % slave_name) + sys.exit(1) + if not "host" in slave: + host = "local" + else: + host = slave["host"] + workerdir = "output/%s/%s" % (host, slave_name) + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + fp = open(dockcomposeymlpath, "r") + dockcomp = yaml.load(fp) + fp.close() + device_path = "%s/devices/" % workerdir + devices_path = "%s/devices/%s" % (workerdir, slave_name) + devicetype = board["type"] + device_line = template_device.substitute(devicetype=devicetype) + if "pdu_generic" in board: + hard_reset_command = board["pdu_generic"]["hard_reset_command"] + power_off_command = board["pdu_generic"]["power_off_command"] + power_on_command = board["pdu_generic"]["power_on_command"] + device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command) + use_kvm = False + if "kvm" in board: + use_kvm = board["kvm"] + if use_kvm: + if "devices" in dockcomp["services"][worker_name]: + dc_devices = dockcomp["services"][worker_name]["devices"] + else: + dockcomp["services"][worker_name]["devices"] = [] + dc_devices = dockcomp["services"][worker_name]["devices"] + dc_devices.append("/dev/kvm:/dev/kvm") + # board specific hacks + if devicetype == "qemu" and not use_kvm: + device_line += "{% set no_kvm = True %}\n" + if "uart" in board: + uart = board["uart"] + baud = board["uart"].get("baud", baud_default) + idvendor = board["uart"]["idvendor"] + idproduct = board["uart"]["idproduct"] + if type(idproduct) == str: + print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name, idproduct)) + sys.exit(1) + if type(idvendor) == str: + print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name, idvendor)) + sys.exit(1) + line = template_conmux.substitute(board=board_name, baud=baud) + if "serial" in uart: + serial = board["uart"]["serial"] + udev_line = template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) + else: + devpath = board["uart"]["devpath"] + udev_line = template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) + if not os.path.isdir("output/%s/udev" % host): + os.mkdir("output/%s/udev" % host) + fp = open("output/%s/udev/99-lavaworker-udev.rules" % host, "a") + fp.write(udev_line) + fp.close() + if "devices" in dockcomp["services"][worker_name]: + dc_devices = dockcomp["services"][worker_name]["devices"] + else: + dockcomp["services"][worker_name]["devices"] = [] + dc_devices = dockcomp["services"][worker_name]["devices"] + dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name)) + fp = open("%s/conmux/%s.cf" % (workerdir, board_name), "w") + fp.write(line) + fp.close() + device_line += template_device_conmux.substitute(board=board_name) + elif "connection_command" in board: + connection_command = board["connection_command"] + device_line += template_device_connection_command.substitute(connection_command=connection_command) + if "uboot_ipaddr" in board: + device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % board["uboot_ipaddr"] + if "uboot_macaddr" in board: + device_line += '{% set uboot_set_mac = true %}' + device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % board["uboot_macaddr"] + if "fastboot_serial_number" in board: + fserial = board["fastboot_serial_number"] + device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial + if "custom_option" in board: + for coption in board["custom_option"]: + device_line += "{%% %s %%}" % coption + if not os.path.isdir(device_path): + os.mkdir(device_path) + if not os.path.isdir(devices_path): + os.mkdir(devices_path) + board_device_file = "%s/%s.jinja2" % (devices_path, board_name) + fp = open(board_device_file, "w") + fp.write(device_line) + fp.close() + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) + if __name__ == "__main__": shutil.copy("common/build-lava", "lava-slave/scripts/build-lava") diff --git a/lavalab-gen.sh b/lavalab-gen.sh index 60c5d94..08ab2ad 100755 --- a/lavalab-gen.sh +++ b/lavalab-gen.sh @@ -1,30 +1,11 @@ #!/bin/sh -rm -rv lava-master/devices/ -rm -rv lava-master/slaves/ -rm -rv lava-slave/conmux/ -rm -rv lava-master/tokens/ -rm -rv lava-master/users/ -rm udev/*lavalab*rules rm lava-master/scripts/build-lava rm lava-slave/scripts/build-lava -rm docker-compose.yml +rm -r output if [ "$1" = "mrproper" ];then exit 0 fi ./lavalab-gen.py || exit 1 - -#check for root -BEROOT="" -if [ $(id -u) -ne 0 ];then - BEROOT="sudo " -fi -$BEROOT rm /etc/udev/rules.d/*lava*rules -$BEROOT cp udev/*lava*rules /etc/udev/rules.d/ -$BEROOT udevadm control --reload-rules || exit $? -$BEROOT udevadm trigger || exit $? - -docker-compose build || exit 1 -docker-compose up -d || exit 1 diff --git a/tokens.yaml b/tokens.yaml deleted file mode 100644 index d3a8760..0000000 --- a/tokens.yaml +++ /dev/null @@ -1,15 +0,0 @@ -lava_server_users: - - name: admin - token: longrandomtokenadmin - password: admin - superuser: yes - staff: yes - - name: example - token: longrandomtoken - password: examplepassword - -callback_tokens: - - filename: example - username: example - token: longrandomtokencallback - description: example-callback |