diff options
Diffstat (limited to 'lavalab-gen.py')
| -rwxr-xr-x | lavalab-gen.py | 410 |
1 files changed, 287 insertions, 123 deletions
diff --git a/lavalab-gen.py b/lavalab-gen.py index 037b5ff..13677c8 100755 --- a/lavalab-gen.py +++ b/lavalab-gen.py @@ -43,134 +43,87 @@ template_udev_devpath = string.Template("""# SUBSYSTEM=="tty", ATTRS{idVendor}=="${idvendor}", ATTRS{idProduct}=="${idproduct}", ATTRS{devpath}=="${devpath}", MODE="0664", OWNER="uucp", SYMLINK+="${board}" """) +template_settings_conf = string.Template(""" +{ + "DEBUG": false, + "STATICFILES_DIRS": [ + ["lava-server", "/usr/share/pyshared/lava_server/htdocs/"] + ], + "MEDIA_ROOT": "/var/lib/lava-server/default/media", + "ARCHIVE_ROOT": "/var/lib/lava-server/default/archive", + "STATIC_ROOT": "/usr/share/lava-server/static", + "STATIC_URL": "/static/", + "MOUNT_POINT": "/", + "HTTPS_XML_RPC": false, + "LOGIN_URL": "/accounts/login/", + "LOGIN_REDIRECT_URL": "/", + "CSRF_COOKIE_SECURE": $cookie_secure, + "SESSION_COOKIE_SECURE": $session_cookie_secure +} +""") + def main(): fp = open(boards_yaml, "r") workers = yaml.load(fp) fp.close() - tdc = open("docker-compose.template", "r") - dockcomp = yaml.load(tdc) - tdc.close() - - # The slaves directory must exists - if not os.path.isdir("lava-master/slaves/"): - os.mkdir("lava-master/slaves/") - fp = open("lava-master/slaves/.empty", "w") - fp.close() - if not os.path.isdir("lava-slave/conmux/"): - os.mkdir("lava-slave/conmux/") - fp = open("lava-slave/conmux/.empty", "w") - fp.close() - for worker_name in workers: - udev_line ="" - worker = workers[worker_name] - use_kvm = False - if "host_has_cpuflag_kvm" in worker: - use_kvm = worker["host_has_cpuflag_kvm"] - if use_kvm: - if "devices" in dockcomp["services"][worker_name]: - dc_devices = dockcomp["services"][worker_name]["devices"] - else: - dockcomp["services"][worker_name]["devices"] = [] - dc_devices = dockcomp["services"][worker_name]["devices"] - dc_devices.append("/dev/kvm:/dev/kvm") - for board_name in worker["boardlist"]: - b = worker["boardlist"][board_name] - if b.get("disabled", None): - continue - - devicetype = b["type"] - device_line = template_device.substitute(devicetype=devicetype) - if "pdu_generic" in b: - hard_reset_command = b["pdu_generic"]["hard_reset_command"] - power_off_command = b["pdu_generic"]["power_off_command"] - power_on_command = b["pdu_generic"]["power_on_command"] - device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command) - if "uart" in b: - uart = b["uart"] - baud = b["uart"].get("baud", baud_default) - idvendor = b["uart"]["idvendor"] - idproduct = b["uart"]["idproduct"] - if type(idproduct) == str: - print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name,idproduct)) - sys.exit(1) - if type(idvendor) == str: - print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name,idvendor)) - sys.exit(1) - line = template_conmux.substitute(board=board_name, baud=baud) - if "serial" in uart: - serial = b["uart"]["serial"] - udev_line += template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) - else: - devpath = b["uart"]["devpath"] - udev_line += template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) - if "devices" in dockcomp["services"][worker_name]: - dc_devices = dockcomp["services"][worker_name]["devices"] - else: - dockcomp["services"][worker_name]["devices"] = [] - dc_devices = dockcomp["services"][worker_name]["devices"] - dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name)) - fp = open("lava-slave/conmux/%s.cf" % board_name, "w") - fp.write(line) - fp.close() - device_line += template_device_conmux.substitute(board=board_name) - elif "connection_command" in b: - connection_command = b["connection_command"] - device_line += template_device_connection_command.substitute(connection_command=connection_command) - if "uboot_ipaddr" in b: - device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % b["uboot_ipaddr"] - if "uboot_macaddr" in b: - device_line += '{% set uboot_set_mac = true %}' - device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % b["uboot_macaddr"] - if "fastboot_serial_number" in b: - fserial = b["fastboot_serial_number"] - device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial - if "custom_option" in b: - for coption in b["custom_option"]: - device_line += "{%% %s %%}" % coption + os.mkdir("output") - # board specific hacks - if devicetype == "qemu" and not use_kvm: - device_line += "{% set no_kvm = True %}\n" - if not os.path.isdir("lava-master/devices/"): - os.mkdir("lava-master/devices/") - device_path = "lava-master/devices/%s" % worker_name - if not os.path.isdir(device_path): - os.mkdir(device_path) - board_device_file = "%s/%s.jinja2" % (device_path, board_name) - fp = open(board_device_file, "w") - fp.write(device_line) - fp.close() - if not os.path.isdir("udev"): - os.mkdir("udev") - fp = open("udev/99-lavaworker-udev-%s.rules" % worker_name, "w") - fp.write(udev_line) - fp.close() - if "dispatcher_ip" in worker: - fp = open("lava-master/slaves/%s.yaml" % worker_name, "w") - fp.write("dispatcher_ip: %s" % worker["dispatcher_ip"]) - fp.close() + if "masters" not in workers: + print("Missing masters entry in boards.yaml") + sys.exit(1) + masters = workers["masters"] + for master in masters: + name = master["name"] + print("Handle %s\n" % name) + if not "host" in master: + host = "local" + else: + host = master["host"] + workerdir = "output/%s/%s" % (host, name) + os.mkdir("output/%s" % host) + shutil.copy("deploy.sh", "output/%s/" % host) + dockcomp = {} + dockcomp["version"] = "2.0" + dockcomp["services"] = {} + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + dockcomp["services"][name] = {} + dockcomp["services"][name]["hostname"] = name + dockcomp["services"][name]["ports"] = [ "10080:80", "5555:5555", "5556:5556", "5500:5500" ] + dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ] + dockcomp["services"][name]["build"] = {} + dockcomp["services"][name]["build"]["context"] = name + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) - #now proceed with tokens - fp = open(tokens_yaml, "r") - tokens = yaml.load(fp) - fp.close() - if not os.path.isdir("lava-master/users/"): - os.mkdir("lava-master/users/") - if not os.path.isdir("lava-master/tokens/"): - os.mkdir("lava-master/tokens/") - for section_name in tokens: - section = tokens[section_name] - if section_name == "lava_server_users": - for user in section: + shutil.copytree("lava-master", workerdir) + os.mkdir("%s/devices" % workerdir) + # handle users / tokens + userdir = "%s/users" % workerdir + os.mkdir(userdir) + worker = master + webadmin_https = False + if "webadmin_https" in worker: + webadmin_https = worker["webadmin_https"] + if webadmin_https: + cookie_secure = "true" + session_cookie_secure = "true" + else: + cookie_secure = "false" + session_cookie_secure = "false" + fsettings = open("%s/settings.conf" % workerdir, 'w') + fsettings.write(template_settings_conf.substitute(cookie_secure=cookie_secure, session_cookie_secure=session_cookie_secure)) + fsettings.close() + if "users" in worker: + for user in worker["users"]: username = user["name"] - ftok = open("lava-master/users/%s" % username, "w") + ftok = open("%s/%s" % (userdir, username), "w") token = user["token"] ftok.write("TOKEN=" + token + "\n") if "password" in user: password = user["password"] ftok.write("PASSWORD=" + password + "\n") - # libyaml convert yes/no to true/false... + # libyaml convert yes/no to true/false... if "staff" in user: value = user["staff"] if value is True: @@ -180,19 +133,230 @@ def main(): if value is True: ftok.write("SUPERUSER=1\n") ftok.close() - if section_name == "callback_tokens": - for token in section: - filename = token["filename"] - ftok = open("lava-master/tokens/%s" % filename, "w") + tokendir = "%s/tokens" % workerdir + os.mkdir(tokendir) + if "tokens" in worker: + filename_num = {} + print("Found tokens") + for token in worker["tokens"]: username = token["username"] + description = token["description"] + if username in filename_num: + number = filename_num[username] + filename_num[username] = filename_num[username] + 1 + else: + filename_num[username] = 1 + number = 0 + filename = "%s-%d" % (username, number) + print("\tAdd token for %s in %s" % (username, filename)) + ftok = open("%s/%s" % (tokendir, filename), "w") ftok.write("USER=" + username + "\n") vtoken = token["token"] ftok.write("TOKEN=" + vtoken + "\n") - description = token["description"] - ftok.write("DESCRIPTION=" + description) + ftok.write("DESCRIPTION=\"%s\"" % description) ftok.close() - with open('docker-compose.yml', 'w') as f: - yaml.dump(dockcomp, f) + + default_slave = "lab-slave-0" + if "slaves" not in workers: + print("Missing slaves entry in boards.yaml") + sys.exit(1) + slaves = workers["slaves"] + for slave in slaves: + name = slave["name"] + if len(slaves) == 1: + default_slave = name + print("Handle %s" % name) + if not "host" in slave: + host = "local" + else: + host = slave["host"] + if slave.get("default_slave") and slave["default_slave"]: + default_slave = name + workerdir = "output/%s/%s" % (host, name) + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + if not os.path.isdir("output/%s" % host): + os.mkdir("output/%s" % host) + shutil.copy("deploy.sh", "output/%s/" % host) + dockcomp = {} + dockcomp["version"] = "2.0" + dockcomp["services"] = {} + else: + #master exists + fp = open(dockcomposeymlpath, "r") + dockcomp = yaml.load(fp) + fp.close() + dockcomp["services"][name] = {} + dockcomp["services"][name]["hostname"] = name + dockcomp["services"][name]["dns_search"] = "" + dockcomp["services"][name]["ports"] = [ "69:69/udp", "80:80", "61950-62000:61950-62000" ] + dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ] + dockcomp["services"][name]["environment"] = {} + dockcomp["services"][name]["build"] = {} + dockcomp["services"][name]["build"]["context"] = name + # insert here remote + + shutil.copytree("lava-slave", workerdir) + fp = open("%s/phyhostname" % workerdir, "w") + fp.write(host) + fp.close() + conmuxpath = "%s/conmux" % workerdir + if not os.path.isdir(conmuxpath): + os.mkdir(conmuxpath) + + worker = slave + worker_name = name + #NOTE remote_master is on slave + if not "remote_master" in worker: + remote_master = "lava-master" + else: + remote_master = worker["remote_master"] + if not "remote_address" in worker: + remote_address = remote_master + else: + remote_address = worker["remote_address"] + if not "remote_rpc_port" in worker: + remote_rpc_port = "80" + else: + remote_rpc_port = worker["remote_rpc_port"] + dockcomp["services"][worker_name]["environment"]["LAVA_MASTER"] = remote_address + remote_user = worker["remote_user"] + # find master + remote_token = "BAD" + for fm in workers["masters"]: + if fm["name"] == remote_master: + for fuser in fm["users"]: + if fuser["name"] == remote_user: + remote_token = fuser["token"] + if remote_token is "BAD": + print("Cannot find %s on %s" % (remote_user, remote_master)) + sys.exit(1) + if not "remote_proto" in worker: + remote_proto = "http" + else: + remote_proto = worker["remote_proto"] + remote_uri = "%s://%s:%s@%s:%s/RPC2" % (remote_proto, remote_user, remote_token, remote_address, remote_rpc_port) + dockcomp["services"][worker_name]["environment"]["LAVA_MASTER_URI"] = remote_uri + + if "dispatcher_ip" in worker: + dockcomp["services"][worker_name]["environment"]["LAVA_DISPATCHER_IP"] = worker["dispatcher_ip"] + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) + if "extra_actions" in worker: + fp = open("%s/scripts/extra_actions" % workerdir, "w") + for eaction in worker["extra_actions"]: + fp.write(eaction) + fp.write("\n") + fp.close() + os.chmod("%s/scripts/extra_actions" % workerdir, 0o755) + + if "boards" not in workers: + print("Missing boards") + sys.exit(1) + boards = workers["boards"] + for board in boards: + board_name = board["name"] + if "slave" in board: + slave_name = board["slave"] + else: + slave_name = default_slave + print("\tFound %s on %s" % (board_name, slave_name)) + found_slave = False + for fs in workers["slaves"]: + if fs["name"] == slave_name: + slave = fs + found_slave = True + if not found_slave: + print("Cannot find slave %s" % slave_name) + sys.exit(1) + if not "host" in slave: + host = "local" + else: + host = slave["host"] + workerdir = "output/%s/%s" % (host, slave_name) + dockcomposeymlpath = "output/%s/docker-compose.yml" % host + fp = open(dockcomposeymlpath, "r") + dockcomp = yaml.load(fp) + fp.close() + device_path = "%s/devices/" % workerdir + devices_path = "%s/devices/%s" % (workerdir, slave_name) + devicetype = board["type"] + device_line = template_device.substitute(devicetype=devicetype) + if "pdu_generic" in board: + hard_reset_command = board["pdu_generic"]["hard_reset_command"] + power_off_command = board["pdu_generic"]["power_off_command"] + power_on_command = board["pdu_generic"]["power_on_command"] + device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command) + use_kvm = False + if "kvm" in board: + use_kvm = board["kvm"] + if use_kvm: + if "devices" in dockcomp["services"][worker_name]: + dc_devices = dockcomp["services"][worker_name]["devices"] + else: + dockcomp["services"][worker_name]["devices"] = [] + dc_devices = dockcomp["services"][worker_name]["devices"] + dc_devices.append("/dev/kvm:/dev/kvm") + # board specific hacks + if devicetype == "qemu" and not use_kvm: + device_line += "{% set no_kvm = True %}\n" + if "uart" in board: + uart = board["uart"] + baud = board["uart"].get("baud", baud_default) + idvendor = board["uart"]["idvendor"] + idproduct = board["uart"]["idproduct"] + if type(idproduct) == str: + print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name, idproduct)) + sys.exit(1) + if type(idvendor) == str: + print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name, idvendor)) + sys.exit(1) + line = template_conmux.substitute(board=board_name, baud=baud) + if "serial" in uart: + serial = board["uart"]["serial"] + udev_line = template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) + else: + devpath = board["uart"]["devpath"] + udev_line = template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct) + if not os.path.isdir("output/%s/udev" % host): + os.mkdir("output/%s/udev" % host) + fp = open("output/%s/udev/99-lavaworker-udev.rules" % host, "a") + fp.write(udev_line) + fp.close() + if "devices" in dockcomp["services"][worker_name]: + dc_devices = dockcomp["services"][worker_name]["devices"] + else: + dockcomp["services"][worker_name]["devices"] = [] + dc_devices = dockcomp["services"][worker_name]["devices"] + dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name)) + fp = open("%s/conmux/%s.cf" % (workerdir, board_name), "w") + fp.write(line) + fp.close() + device_line += template_device_conmux.substitute(board=board_name) + elif "connection_command" in board: + connection_command = board["connection_command"] + device_line += template_device_connection_command.substitute(connection_command=connection_command) + if "uboot_ipaddr" in board: + device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % board["uboot_ipaddr"] + if "uboot_macaddr" in board: + device_line += '{% set uboot_set_mac = true %}' + device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % board["uboot_macaddr"] + if "fastboot_serial_number" in board: + fserial = board["fastboot_serial_number"] + device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial + if "custom_option" in board: + for coption in board["custom_option"]: + device_line += "{%% %s %%}" % coption + if not os.path.isdir(device_path): + os.mkdir(device_path) + if not os.path.isdir(devices_path): + os.mkdir(devices_path) + board_device_file = "%s/%s.jinja2" % (devices_path, board_name) + fp = open(board_device_file, "w") + fp.write(device_line) + fp.close() + with open(dockcomposeymlpath, 'w') as f: + yaml.dump(dockcomp, f) + if __name__ == "__main__": shutil.copy("common/build-lava", "lava-slave/scripts/build-lava") |