Implement Multipleslave

This patch implement multiple slave support.
Instead of having a maximum of one master and one slave in one docker
image, it is now possible to have multiple slave accross several docker
host.

For helping this change, a new boards.yaml format is introduced (See
README.md for details)
Note that tokens.yaml is also squashed in boards.yaml

1 files changed, 256 insertions, 123 deletions

diff --git a/lavalab-gen.py b/lavalab-gen.py
index 037b5ff..7645d08 100755
--- a/lavalab-gen.py
+++ b/lavalab-gen.py
@@ -47,130 +47,52 @@ def main():
     fp = open(boards_yaml, "r")
     workers = yaml.load(fp)
     fp.close()
-    tdc = open("docker-compose.template", "r")
-    dockcomp = yaml.load(tdc)
-    tdc.close()
-
-    # The slaves directory must exists
-    if not os.path.isdir("lava-master/slaves/"):
-        os.mkdir("lava-master/slaves/")
-        fp = open("lava-master/slaves/.empty", "w")
-        fp.close()
-    if not os.path.isdir("lava-slave/conmux/"):
-        os.mkdir("lava-slave/conmux/")
-        fp = open("lava-slave/conmux/.empty", "w")
-        fp.close()
 
-    for worker_name in workers:
-        udev_line =""
-        worker = workers[worker_name]
-        use_kvm = False
-        if "host_has_cpuflag_kvm" in worker:
-            use_kvm = worker["host_has_cpuflag_kvm"]
-        if use_kvm:
-            if "devices" in dockcomp["services"][worker_name]:
-                dc_devices = dockcomp["services"][worker_name]["devices"]
-            else:
-                dockcomp["services"][worker_name]["devices"] = []
-                dc_devices = dockcomp["services"][worker_name]["devices"]
-            dc_devices.append("/dev/kvm:/dev/kvm")
-        for board_name in worker["boardlist"]:
-            b = worker["boardlist"][board_name]
-            if b.get("disabled", None):
-                continue
-
-            devicetype = b["type"]
-            device_line = template_device.substitute(devicetype=devicetype)
-            if "pdu_generic" in b:
-                hard_reset_command = b["pdu_generic"]["hard_reset_command"]
-                power_off_command = b["pdu_generic"]["power_off_command"]
-                power_on_command = b["pdu_generic"]["power_on_command"]
-                device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
-            if "uart" in b:
-                uart = b["uart"]
-                baud = b["uart"].get("baud", baud_default)
-                idvendor = b["uart"]["idvendor"]
-                idproduct = b["uart"]["idproduct"]
-                if type(idproduct) == str:
-                    print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name,idproduct))
-                    sys.exit(1)
-                if type(idvendor) == str:
-                    print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name,idvendor))
-                    sys.exit(1)
-                line = template_conmux.substitute(board=board_name, baud=baud)
-                if "serial" in uart:
-                    serial = b["uart"]["serial"]
-                    udev_line += template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
-                else:
-                    devpath = b["uart"]["devpath"]
-                    udev_line += template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
-                if "devices" in dockcomp["services"][worker_name]:
-                    dc_devices = dockcomp["services"][worker_name]["devices"]
-                else:
-                    dockcomp["services"][worker_name]["devices"] = []
-                    dc_devices = dockcomp["services"][worker_name]["devices"]
-                dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
-                fp = open("lava-slave/conmux/%s.cf" % board_name, "w")
-                fp.write(line)
-                fp.close()
-                device_line += template_device_conmux.substitute(board=board_name)
-            elif "connection_command" in b:
-                connection_command = b["connection_command"]
-                device_line += template_device_connection_command.substitute(connection_command=connection_command)
-            if "uboot_ipaddr" in b:
-                device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % b["uboot_ipaddr"]
-            if "uboot_macaddr" in b:
-                device_line += '{% set uboot_set_mac = true %}'
-                device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % b["uboot_macaddr"]
-            if "fastboot_serial_number" in b:
-                fserial = b["fastboot_serial_number"]
-                device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
-            if "custom_option" in b:
-                for coption in b["custom_option"]:
-                    device_line += "{%% %s %%}" % coption
+    os.mkdir("output")
 
-            # board specific hacks
-            if devicetype == "qemu" and not use_kvm:
-                device_line += "{% set no_kvm = True %}\n"
-            if not os.path.isdir("lava-master/devices/"):
-                os.mkdir("lava-master/devices/")
-            device_path = "lava-master/devices/%s" % worker_name
-            if not os.path.isdir(device_path):
-                os.mkdir(device_path)
-            board_device_file = "%s/%s.jinja2" % (device_path, board_name)
-            fp = open(board_device_file, "w")
-            fp.write(device_line)
-            fp.close()
-        if not os.path.isdir("udev"):
-            os.mkdir("udev")
-        fp = open("udev/99-lavaworker-udev-%s.rules" % worker_name, "w")
-        fp.write(udev_line)
-        fp.close()
-        if "dispatcher_ip" in worker:
-            fp = open("lava-master/slaves/%s.yaml" % worker_name, "w")
-            fp.write("dispatcher_ip: %s" % worker["dispatcher_ip"])
-            fp.close()
+    if "masters" not in workers:
+        print("Missing masters entry in boards.yaml")
+        sys.exit(1)
+    masters = workers["masters"]
+    for master in masters:
+        name = master["name"]
+        print("Handle %s\n" % name)
+        if not "host" in master:
+            host = "local"
+        else:
+            host = master["host"]
+        workerdir = "output/%s/%s" % (host, name)
+        os.mkdir("output/%s" % host)
+        shutil.copy("deploy.sh", "output/%s/" % host)
+        dockcomp = {}
+        dockcomp["version"] = "2.0"
+        dockcomp["services"] = {}
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        dockcomp["services"][name] = {}
+        dockcomp["services"][name]["hostname"] = name
+        dockcomp["services"][name]["ports"] = [ "10080:80", "5555:5555", "5556:5556", "5500:5500" ]
+        dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+        dockcomp["services"][name]["build"] = {}
+        dockcomp["services"][name]["build"]["context"] = name
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
 
-    #now proceed with tokens
-    fp = open(tokens_yaml, "r")
-    tokens = yaml.load(fp)
-    fp.close()
-    if not os.path.isdir("lava-master/users/"):
-        os.mkdir("lava-master/users/")
-    if not os.path.isdir("lava-master/tokens/"):
-        os.mkdir("lava-master/tokens/")
-    for section_name in tokens:
-        section = tokens[section_name]
-        if section_name == "lava_server_users":
-            for user in section:
+        shutil.copytree("lava-master", workerdir)
+        os.mkdir("%s/devices" % workerdir)
+        # handle users / tokens
+        userdir = "%s/users" % workerdir
+        os.mkdir(userdir)
+        worker = master
+        if "users" in worker:
+            for user in worker["users"]:
                 username = user["name"]
-                ftok = open("lava-master/users/%s" % username, "w")
+                ftok = open("%s/%s" % (userdir, username), "w")
                 token = user["token"]
                 ftok.write("TOKEN=" + token + "\n")
                 if "password" in user:
                     password = user["password"]
                     ftok.write("PASSWORD=" + password + "\n")
-                # libyaml convert yes/no to true/false...
+                    # libyaml convert yes/no to true/false...
                 if "staff" in user:
                     value = user["staff"]
                     if value is True:
@@ -180,19 +102,230 @@ def main():
                     if value is True:
                         ftok.write("SUPERUSER=1\n")
                 ftok.close()
-        if section_name == "callback_tokens":
-            for token in section:
-                filename = token["filename"]
-                ftok = open("lava-master/tokens/%s" % filename, "w")
+        tokendir = "%s/tokens" % workerdir
+        os.mkdir(tokendir)
+        if "tokens" in worker:
+            filename_num = {}
+            print("Found tokens")
+            for token in worker["tokens"]:
                 username = token["username"]
+                description = token["description"]
+                if username in filename_num:
+                    number = filename_num[username]
+                    filename_num[username] = filename_num[username] + 1
+                else:
+                    filename_num[username] = 1
+                    number = 0
+                filename = "%s-%d" % (username, number)
+                print("\tAdd token for %s in %s" % (username, filename))
+                ftok = open("%s/%s" % (tokendir, filename), "w")
                 ftok.write("USER=" + username + "\n")
                 vtoken = token["token"]
                 ftok.write("TOKEN=" + vtoken + "\n")
-                description = token["description"]
-                ftok.write("DESCRIPTION=" + description)
+                ftok.write("DESCRIPTION=\"%s\"" % description)
                 ftok.close()
-    with open('docker-compose.yml', 'w') as f:
-        yaml.dump(dockcomp, f)
+
+    default_slave = "lab-slave-0"
+    if "slaves" not in workers:
+        print("Missing slaves entry in boards.yaml")
+        sys.exit(1)
+    slaves = workers["slaves"]
+    for slave in slaves:
+        name = slave["name"]
+        if len(slaves) == 1:
+            default_slave = name
+        print("Handle %s" % name)
+        if not "host" in slave:
+            host = "local"
+        else:
+            host = slave["host"]
+        if slave.get("default_slave") and slave["default_slave"]:
+             default_slave = name
+        workerdir = "output/%s/%s" % (host, name)
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        if not os.path.isdir("output/%s" % host):
+            os.mkdir("output/%s" % host)
+            shutil.copy("deploy.sh", "output/%s/" % host)
+            dockcomp = {}
+            dockcomp["version"] = "2.0"
+            dockcomp["services"] = {}
+        else:
+            #master exists
+            fp = open(dockcomposeymlpath, "r")
+            dockcomp = yaml.load(fp)
+            fp.close()
+        dockcomp["services"][name] = {}
+        dockcomp["services"][name]["hostname"] = name
+        dockcomp["services"][name]["dns_search"] = ""
+        dockcomp["services"][name]["ports"] = [ "69:69/udp", "80:80", "61950-62000:61950-62000" ]
+        dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+        dockcomp["services"][name]["environment"] = {}
+        dockcomp["services"][name]["build"] = {}
+        dockcomp["services"][name]["build"]["context"] = name
+        # insert here remote
+
+        shutil.copytree("lava-slave", workerdir)
+        fp = open("%s/phyhostname" % workerdir, "w")
+        fp.write(host)
+        fp.close()
+        conmuxpath = "%s/conmux" % workerdir
+        if not os.path.isdir(conmuxpath):
+            os.mkdir(conmuxpath)
+
+        worker = slave
+        worker_name = name
+        #NOTE remote_master is on slave
+        if not "remote_master" in worker:
+            remote_master = "lava-master"
+        else:
+            remote_master = worker["remote_master"]
+        if not "remote_address" in worker:
+            remote_address = remote_master
+        else:
+            remote_address = worker["remote_address"]
+        if not "remote_rpc_port" in worker:
+            remote_rpc_port = "80"
+        else:
+            remote_rpc_port = worker["remote_rpc_port"]
+        dockcomp["services"][worker_name]["environment"]["LAVA_MASTER"] = remote_address
+        remote_user = worker["remote_user"]
+        # find master
+        remote_token = "BAD"
+        for fm in workers["masters"]:
+            if fm["name"] == remote_master:
+                for fuser in fm["users"]:
+                    if fuser["name"] == remote_user:
+                        remote_token = fuser["token"]
+        if remote_token is "BAD":
+            print("Cannot find %s on %s" % (remote_user, remote_master))
+            sys.exit(1)
+        if not "remote_proto" in worker:
+            remote_proto = "http"
+        else:
+            remote_proto = worker["remote_proto"]
+        remote_uri = "%s://%s:%s@%s:%s/RPC2" % (remote_proto, remote_user, remote_token, remote_address, remote_rpc_port)
+        dockcomp["services"][worker_name]["environment"]["LAVA_MASTER_URI"] = remote_uri
+
+        if "dispatcher_ip" in worker:
+            dockcomp["services"][worker_name]["environment"]["LAVA_DISPATCHER_IP"] = worker["dispatcher_ip"]
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
+        if "extra_actions" in worker:
+            fp = open("%s/scripts/extra_actions" % workerdir, "w")
+            for eaction in worker["extra_actions"]:
+                fp.write(eaction)
+                fp.write("\n")
+            fp.close()
+            os.chmod("%s/scripts/extra_actions" % workerdir, 0o755)
+
+    if "boards" not in workers:
+        print("Missing boards")
+        sys.exit(1)
+    boards = workers["boards"]
+    for board in boards:
+        board_name = board["name"]
+        if "slave" in board:
+            slave_name = board["slave"]
+        else:
+            slave_name = default_slave
+        print("\tFound %s on %s" % (board_name, slave_name))
+        found_slave = False
+        for fs in workers["slaves"]:
+            if fs["name"] == slave_name:
+                slave = fs
+                found_slave = True
+        if not found_slave:
+            print("Cannot find slave %s" % slave_name)
+            sys.exit(1)
+        if not "host" in slave:
+            host = "local"
+        else:
+            host = slave["host"]
+        workerdir = "output/%s/%s" % (host, slave_name)
+        dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+        fp = open(dockcomposeymlpath, "r")
+        dockcomp = yaml.load(fp)
+        fp.close()
+        device_path = "%s/devices/" % workerdir
+        devices_path = "%s/devices/%s" % (workerdir, slave_name)
+        devicetype = board["type"]
+        device_line = template_device.substitute(devicetype=devicetype)
+        if "pdu_generic" in board:
+            hard_reset_command = board["pdu_generic"]["hard_reset_command"]
+            power_off_command = board["pdu_generic"]["power_off_command"]
+            power_on_command = board["pdu_generic"]["power_on_command"]
+            device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
+        use_kvm = False
+        if "kvm" in board:
+            use_kvm = board["kvm"]
+        if use_kvm:
+            if "devices" in dockcomp["services"][worker_name]:
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            else:
+                dockcomp["services"][worker_name]["devices"] = []
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            dc_devices.append("/dev/kvm:/dev/kvm")
+            # board specific hacks
+        if devicetype == "qemu" and not use_kvm:
+            device_line += "{% set no_kvm = True %}\n"
+        if "uart" in board:
+            uart = board["uart"]
+            baud = board["uart"].get("baud", baud_default)
+            idvendor = board["uart"]["idvendor"]
+            idproduct = board["uart"]["idproduct"]
+            if type(idproduct) == str:
+                print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name, idproduct))
+                sys.exit(1)
+            if type(idvendor) == str:
+                print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name, idvendor))
+                sys.exit(1)
+            line = template_conmux.substitute(board=board_name, baud=baud)
+            if "serial" in uart:
+                serial = board["uart"]["serial"]
+                udev_line = template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+            else:
+                devpath = board["uart"]["devpath"]
+                udev_line = template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+            if not os.path.isdir("output/%s/udev" % host):
+                os.mkdir("output/%s/udev" % host)
+            fp = open("output/%s/udev/99-lavaworker-udev.rules" % host, "a")
+            fp.write(udev_line)
+            fp.close()
+            if "devices" in dockcomp["services"][worker_name]:
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            else:
+                dockcomp["services"][worker_name]["devices"] = []
+                dc_devices = dockcomp["services"][worker_name]["devices"]
+            dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
+            fp = open("%s/conmux/%s.cf" % (workerdir, board_name), "w")
+            fp.write(line)
+            fp.close()
+            device_line += template_device_conmux.substitute(board=board_name)
+        elif "connection_command" in board:
+            connection_command = board["connection_command"]
+            device_line += template_device_connection_command.substitute(connection_command=connection_command)
+        if "uboot_ipaddr" in board:
+            device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % board["uboot_ipaddr"]
+        if "uboot_macaddr" in board:
+            device_line += '{% set uboot_set_mac = true %}'
+            device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % board["uboot_macaddr"]
+        if "fastboot_serial_number" in board:
+            fserial = board["fastboot_serial_number"]
+            device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
+        if "custom_option" in board:
+            for coption in board["custom_option"]:
+                device_line += "{%% %s %%}" % coption
+        if not os.path.isdir(device_path):
+            os.mkdir(device_path)
+        if not os.path.isdir(devices_path):
+            os.mkdir(devices_path)
+        board_device_file = "%s/%s.jinja2" % (devices_path, board_name)
+        fp = open(board_device_file, "w")
+        fp.write(device_line)
+        fp.close()
+        with open(dockcomposeymlpath, 'w') as f:
+            yaml.dump(dockcomp, f)
+
 
 if __name__ == "__main__":
     shutil.copy("common/build-lava", "lava-slave/scripts/build-lava")