summaryrefslogtreecommitdiffstats
path: root/lavalab-gen.py
diff options
context:
space:
mode:
authorkhilman <khilman@users.noreply.github.com>2018-06-05 11:47:07 -0700
committerGitHub <noreply@github.com>2018-06-05 11:47:07 -0700
commitc1071d0f9b57958b22546abe8eadf670b3926b98 (patch)
tree6ac6947fc9196283ecba0c17a9033d3af3aaaa99 /lavalab-gen.py
parent2d79d03f5818c096535c9784b12df5ea99c0dbff (diff)
parenta22e4a89a06bda681ba329bf73e05a7d9196efe3 (diff)
Merge pull request #27 from montjoie/multi_slave_2018_x
Multi slave 2018 x
Diffstat (limited to 'lavalab-gen.py')
-rwxr-xr-xlavalab-gen.py410
1 files changed, 287 insertions, 123 deletions
diff --git a/lavalab-gen.py b/lavalab-gen.py
index 037b5ff..13677c8 100755
--- a/lavalab-gen.py
+++ b/lavalab-gen.py
@@ -43,134 +43,87 @@ template_udev_devpath = string.Template("""#
SUBSYSTEM=="tty", ATTRS{idVendor}=="${idvendor}", ATTRS{idProduct}=="${idproduct}", ATTRS{devpath}=="${devpath}", MODE="0664", OWNER="uucp", SYMLINK+="${board}"
""")
+template_settings_conf = string.Template("""
+{
+ "DEBUG": false,
+ "STATICFILES_DIRS": [
+ ["lava-server", "/usr/share/pyshared/lava_server/htdocs/"]
+ ],
+ "MEDIA_ROOT": "/var/lib/lava-server/default/media",
+ "ARCHIVE_ROOT": "/var/lib/lava-server/default/archive",
+ "STATIC_ROOT": "/usr/share/lava-server/static",
+ "STATIC_URL": "/static/",
+ "MOUNT_POINT": "/",
+ "HTTPS_XML_RPC": false,
+ "LOGIN_URL": "/accounts/login/",
+ "LOGIN_REDIRECT_URL": "/",
+ "CSRF_COOKIE_SECURE": $cookie_secure,
+ "SESSION_COOKIE_SECURE": $session_cookie_secure
+}
+""")
+
def main():
fp = open(boards_yaml, "r")
workers = yaml.load(fp)
fp.close()
- tdc = open("docker-compose.template", "r")
- dockcomp = yaml.load(tdc)
- tdc.close()
-
- # The slaves directory must exists
- if not os.path.isdir("lava-master/slaves/"):
- os.mkdir("lava-master/slaves/")
- fp = open("lava-master/slaves/.empty", "w")
- fp.close()
- if not os.path.isdir("lava-slave/conmux/"):
- os.mkdir("lava-slave/conmux/")
- fp = open("lava-slave/conmux/.empty", "w")
- fp.close()
- for worker_name in workers:
- udev_line =""
- worker = workers[worker_name]
- use_kvm = False
- if "host_has_cpuflag_kvm" in worker:
- use_kvm = worker["host_has_cpuflag_kvm"]
- if use_kvm:
- if "devices" in dockcomp["services"][worker_name]:
- dc_devices = dockcomp["services"][worker_name]["devices"]
- else:
- dockcomp["services"][worker_name]["devices"] = []
- dc_devices = dockcomp["services"][worker_name]["devices"]
- dc_devices.append("/dev/kvm:/dev/kvm")
- for board_name in worker["boardlist"]:
- b = worker["boardlist"][board_name]
- if b.get("disabled", None):
- continue
-
- devicetype = b["type"]
- device_line = template_device.substitute(devicetype=devicetype)
- if "pdu_generic" in b:
- hard_reset_command = b["pdu_generic"]["hard_reset_command"]
- power_off_command = b["pdu_generic"]["power_off_command"]
- power_on_command = b["pdu_generic"]["power_on_command"]
- device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
- if "uart" in b:
- uart = b["uart"]
- baud = b["uart"].get("baud", baud_default)
- idvendor = b["uart"]["idvendor"]
- idproduct = b["uart"]["idproduct"]
- if type(idproduct) == str:
- print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name,idproduct))
- sys.exit(1)
- if type(idvendor) == str:
- print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name,idvendor))
- sys.exit(1)
- line = template_conmux.substitute(board=board_name, baud=baud)
- if "serial" in uart:
- serial = b["uart"]["serial"]
- udev_line += template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
- else:
- devpath = b["uart"]["devpath"]
- udev_line += template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
- if "devices" in dockcomp["services"][worker_name]:
- dc_devices = dockcomp["services"][worker_name]["devices"]
- else:
- dockcomp["services"][worker_name]["devices"] = []
- dc_devices = dockcomp["services"][worker_name]["devices"]
- dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
- fp = open("lava-slave/conmux/%s.cf" % board_name, "w")
- fp.write(line)
- fp.close()
- device_line += template_device_conmux.substitute(board=board_name)
- elif "connection_command" in b:
- connection_command = b["connection_command"]
- device_line += template_device_connection_command.substitute(connection_command=connection_command)
- if "uboot_ipaddr" in b:
- device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % b["uboot_ipaddr"]
- if "uboot_macaddr" in b:
- device_line += '{% set uboot_set_mac = true %}'
- device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % b["uboot_macaddr"]
- if "fastboot_serial_number" in b:
- fserial = b["fastboot_serial_number"]
- device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
- if "custom_option" in b:
- for coption in b["custom_option"]:
- device_line += "{%% %s %%}" % coption
+ os.mkdir("output")
- # board specific hacks
- if devicetype == "qemu" and not use_kvm:
- device_line += "{% set no_kvm = True %}\n"
- if not os.path.isdir("lava-master/devices/"):
- os.mkdir("lava-master/devices/")
- device_path = "lava-master/devices/%s" % worker_name
- if not os.path.isdir(device_path):
- os.mkdir(device_path)
- board_device_file = "%s/%s.jinja2" % (device_path, board_name)
- fp = open(board_device_file, "w")
- fp.write(device_line)
- fp.close()
- if not os.path.isdir("udev"):
- os.mkdir("udev")
- fp = open("udev/99-lavaworker-udev-%s.rules" % worker_name, "w")
- fp.write(udev_line)
- fp.close()
- if "dispatcher_ip" in worker:
- fp = open("lava-master/slaves/%s.yaml" % worker_name, "w")
- fp.write("dispatcher_ip: %s" % worker["dispatcher_ip"])
- fp.close()
+ if "masters" not in workers:
+ print("Missing masters entry in boards.yaml")
+ sys.exit(1)
+ masters = workers["masters"]
+ for master in masters:
+ name = master["name"]
+ print("Handle %s\n" % name)
+ if not "host" in master:
+ host = "local"
+ else:
+ host = master["host"]
+ workerdir = "output/%s/%s" % (host, name)
+ os.mkdir("output/%s" % host)
+ shutil.copy("deploy.sh", "output/%s/" % host)
+ dockcomp = {}
+ dockcomp["version"] = "2.0"
+ dockcomp["services"] = {}
+ dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+ dockcomp["services"][name] = {}
+ dockcomp["services"][name]["hostname"] = name
+ dockcomp["services"][name]["ports"] = [ "10080:80", "5555:5555", "5556:5556", "5500:5500" ]
+ dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+ dockcomp["services"][name]["build"] = {}
+ dockcomp["services"][name]["build"]["context"] = name
+ with open(dockcomposeymlpath, 'w') as f:
+ yaml.dump(dockcomp, f)
- #now proceed with tokens
- fp = open(tokens_yaml, "r")
- tokens = yaml.load(fp)
- fp.close()
- if not os.path.isdir("lava-master/users/"):
- os.mkdir("lava-master/users/")
- if not os.path.isdir("lava-master/tokens/"):
- os.mkdir("lava-master/tokens/")
- for section_name in tokens:
- section = tokens[section_name]
- if section_name == "lava_server_users":
- for user in section:
+ shutil.copytree("lava-master", workerdir)
+ os.mkdir("%s/devices" % workerdir)
+ # handle users / tokens
+ userdir = "%s/users" % workerdir
+ os.mkdir(userdir)
+ worker = master
+ webadmin_https = False
+ if "webadmin_https" in worker:
+ webadmin_https = worker["webadmin_https"]
+ if webadmin_https:
+ cookie_secure = "true"
+ session_cookie_secure = "true"
+ else:
+ cookie_secure = "false"
+ session_cookie_secure = "false"
+ fsettings = open("%s/settings.conf" % workerdir, 'w')
+ fsettings.write(template_settings_conf.substitute(cookie_secure=cookie_secure, session_cookie_secure=session_cookie_secure))
+ fsettings.close()
+ if "users" in worker:
+ for user in worker["users"]:
username = user["name"]
- ftok = open("lava-master/users/%s" % username, "w")
+ ftok = open("%s/%s" % (userdir, username), "w")
token = user["token"]
ftok.write("TOKEN=" + token + "\n")
if "password" in user:
password = user["password"]
ftok.write("PASSWORD=" + password + "\n")
- # libyaml convert yes/no to true/false...
+ # libyaml convert yes/no to true/false...
if "staff" in user:
value = user["staff"]
if value is True:
@@ -180,19 +133,230 @@ def main():
if value is True:
ftok.write("SUPERUSER=1\n")
ftok.close()
- if section_name == "callback_tokens":
- for token in section:
- filename = token["filename"]
- ftok = open("lava-master/tokens/%s" % filename, "w")
+ tokendir = "%s/tokens" % workerdir
+ os.mkdir(tokendir)
+ if "tokens" in worker:
+ filename_num = {}
+ print("Found tokens")
+ for token in worker["tokens"]:
username = token["username"]
+ description = token["description"]
+ if username in filename_num:
+ number = filename_num[username]
+ filename_num[username] = filename_num[username] + 1
+ else:
+ filename_num[username] = 1
+ number = 0
+ filename = "%s-%d" % (username, number)
+ print("\tAdd token for %s in %s" % (username, filename))
+ ftok = open("%s/%s" % (tokendir, filename), "w")
ftok.write("USER=" + username + "\n")
vtoken = token["token"]
ftok.write("TOKEN=" + vtoken + "\n")
- description = token["description"]
- ftok.write("DESCRIPTION=" + description)
+ ftok.write("DESCRIPTION=\"%s\"" % description)
ftok.close()
- with open('docker-compose.yml', 'w') as f:
- yaml.dump(dockcomp, f)
+
+ default_slave = "lab-slave-0"
+ if "slaves" not in workers:
+ print("Missing slaves entry in boards.yaml")
+ sys.exit(1)
+ slaves = workers["slaves"]
+ for slave in slaves:
+ name = slave["name"]
+ if len(slaves) == 1:
+ default_slave = name
+ print("Handle %s" % name)
+ if not "host" in slave:
+ host = "local"
+ else:
+ host = slave["host"]
+ if slave.get("default_slave") and slave["default_slave"]:
+ default_slave = name
+ workerdir = "output/%s/%s" % (host, name)
+ dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+ if not os.path.isdir("output/%s" % host):
+ os.mkdir("output/%s" % host)
+ shutil.copy("deploy.sh", "output/%s/" % host)
+ dockcomp = {}
+ dockcomp["version"] = "2.0"
+ dockcomp["services"] = {}
+ else:
+ #master exists
+ fp = open(dockcomposeymlpath, "r")
+ dockcomp = yaml.load(fp)
+ fp.close()
+ dockcomp["services"][name] = {}
+ dockcomp["services"][name]["hostname"] = name
+ dockcomp["services"][name]["dns_search"] = ""
+ dockcomp["services"][name]["ports"] = [ "69:69/udp", "80:80", "61950-62000:61950-62000" ]
+ dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
+ dockcomp["services"][name]["environment"] = {}
+ dockcomp["services"][name]["build"] = {}
+ dockcomp["services"][name]["build"]["context"] = name
+ # insert here remote
+
+ shutil.copytree("lava-slave", workerdir)
+ fp = open("%s/phyhostname" % workerdir, "w")
+ fp.write(host)
+ fp.close()
+ conmuxpath = "%s/conmux" % workerdir
+ if not os.path.isdir(conmuxpath):
+ os.mkdir(conmuxpath)
+
+ worker = slave
+ worker_name = name
+ #NOTE remote_master is on slave
+ if not "remote_master" in worker:
+ remote_master = "lava-master"
+ else:
+ remote_master = worker["remote_master"]
+ if not "remote_address" in worker:
+ remote_address = remote_master
+ else:
+ remote_address = worker["remote_address"]
+ if not "remote_rpc_port" in worker:
+ remote_rpc_port = "80"
+ else:
+ remote_rpc_port = worker["remote_rpc_port"]
+ dockcomp["services"][worker_name]["environment"]["LAVA_MASTER"] = remote_address
+ remote_user = worker["remote_user"]
+ # find master
+ remote_token = "BAD"
+ for fm in workers["masters"]:
+ if fm["name"] == remote_master:
+ for fuser in fm["users"]:
+ if fuser["name"] == remote_user:
+ remote_token = fuser["token"]
+ if remote_token is "BAD":
+ print("Cannot find %s on %s" % (remote_user, remote_master))
+ sys.exit(1)
+ if not "remote_proto" in worker:
+ remote_proto = "http"
+ else:
+ remote_proto = worker["remote_proto"]
+ remote_uri = "%s://%s:%s@%s:%s/RPC2" % (remote_proto, remote_user, remote_token, remote_address, remote_rpc_port)
+ dockcomp["services"][worker_name]["environment"]["LAVA_MASTER_URI"] = remote_uri
+
+ if "dispatcher_ip" in worker:
+ dockcomp["services"][worker_name]["environment"]["LAVA_DISPATCHER_IP"] = worker["dispatcher_ip"]
+ with open(dockcomposeymlpath, 'w') as f:
+ yaml.dump(dockcomp, f)
+ if "extra_actions" in worker:
+ fp = open("%s/scripts/extra_actions" % workerdir, "w")
+ for eaction in worker["extra_actions"]:
+ fp.write(eaction)
+ fp.write("\n")
+ fp.close()
+ os.chmod("%s/scripts/extra_actions" % workerdir, 0o755)
+
+ if "boards" not in workers:
+ print("Missing boards")
+ sys.exit(1)
+ boards = workers["boards"]
+ for board in boards:
+ board_name = board["name"]
+ if "slave" in board:
+ slave_name = board["slave"]
+ else:
+ slave_name = default_slave
+ print("\tFound %s on %s" % (board_name, slave_name))
+ found_slave = False
+ for fs in workers["slaves"]:
+ if fs["name"] == slave_name:
+ slave = fs
+ found_slave = True
+ if not found_slave:
+ print("Cannot find slave %s" % slave_name)
+ sys.exit(1)
+ if not "host" in slave:
+ host = "local"
+ else:
+ host = slave["host"]
+ workerdir = "output/%s/%s" % (host, slave_name)
+ dockcomposeymlpath = "output/%s/docker-compose.yml" % host
+ fp = open(dockcomposeymlpath, "r")
+ dockcomp = yaml.load(fp)
+ fp.close()
+ device_path = "%s/devices/" % workerdir
+ devices_path = "%s/devices/%s" % (workerdir, slave_name)
+ devicetype = board["type"]
+ device_line = template_device.substitute(devicetype=devicetype)
+ if "pdu_generic" in board:
+ hard_reset_command = board["pdu_generic"]["hard_reset_command"]
+ power_off_command = board["pdu_generic"]["power_off_command"]
+ power_on_command = board["pdu_generic"]["power_on_command"]
+ device_line += template_device_pdu_generic.substitute(hard_reset_command=hard_reset_command, power_off_command=power_off_command, power_on_command=power_on_command)
+ use_kvm = False
+ if "kvm" in board:
+ use_kvm = board["kvm"]
+ if use_kvm:
+ if "devices" in dockcomp["services"][worker_name]:
+ dc_devices = dockcomp["services"][worker_name]["devices"]
+ else:
+ dockcomp["services"][worker_name]["devices"] = []
+ dc_devices = dockcomp["services"][worker_name]["devices"]
+ dc_devices.append("/dev/kvm:/dev/kvm")
+ # board specific hacks
+ if devicetype == "qemu" and not use_kvm:
+ device_line += "{% set no_kvm = True %}\n"
+ if "uart" in board:
+ uart = board["uart"]
+ baud = board["uart"].get("baud", baud_default)
+ idvendor = board["uart"]["idvendor"]
+ idproduct = board["uart"]["idproduct"]
+ if type(idproduct) == str:
+ print("Please put hexadecimal IDs for product %s (like 0x%s)" % (board_name, idproduct))
+ sys.exit(1)
+ if type(idvendor) == str:
+ print("Please put hexadecimal IDs for vendor %s (like 0x%s)" % (board_name, idvendor))
+ sys.exit(1)
+ line = template_conmux.substitute(board=board_name, baud=baud)
+ if "serial" in uart:
+ serial = board["uart"]["serial"]
+ udev_line = template_udev_serial.substitute(board=board_name, serial=serial, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+ else:
+ devpath = board["uart"]["devpath"]
+ udev_line = template_udev_devpath.substitute(board=board_name, devpath=devpath, idvendor="%04x" % idvendor, idproduct="%04x" % idproduct)
+ if not os.path.isdir("output/%s/udev" % host):
+ os.mkdir("output/%s/udev" % host)
+ fp = open("output/%s/udev/99-lavaworker-udev.rules" % host, "a")
+ fp.write(udev_line)
+ fp.close()
+ if "devices" in dockcomp["services"][worker_name]:
+ dc_devices = dockcomp["services"][worker_name]["devices"]
+ else:
+ dockcomp["services"][worker_name]["devices"] = []
+ dc_devices = dockcomp["services"][worker_name]["devices"]
+ dc_devices.append("/dev/%s:/dev/%s" % (board_name, board_name))
+ fp = open("%s/conmux/%s.cf" % (workerdir, board_name), "w")
+ fp.write(line)
+ fp.close()
+ device_line += template_device_conmux.substitute(board=board_name)
+ elif "connection_command" in board:
+ connection_command = board["connection_command"]
+ device_line += template_device_connection_command.substitute(connection_command=connection_command)
+ if "uboot_ipaddr" in board:
+ device_line += "{%% set uboot_ipaddr_cmd = 'setenv ipaddr %s' %%}\n" % board["uboot_ipaddr"]
+ if "uboot_macaddr" in board:
+ device_line += '{% set uboot_set_mac = true %}'
+ device_line += "{%% set uboot_mac_addr = '%s' %%}\n" % board["uboot_macaddr"]
+ if "fastboot_serial_number" in board:
+ fserial = board["fastboot_serial_number"]
+ device_line += "{%% set fastboot_serial_number = '%s' %%}" % fserial
+ if "custom_option" in board:
+ for coption in board["custom_option"]:
+ device_line += "{%% %s %%}" % coption
+ if not os.path.isdir(device_path):
+ os.mkdir(device_path)
+ if not os.path.isdir(devices_path):
+ os.mkdir(devices_path)
+ board_device_file = "%s/%s.jinja2" % (devices_path, board_name)
+ fp = open(board_device_file, "w")
+ fp.write(device_line)
+ fp.close()
+ with open(dockcomposeymlpath, 'w') as f:
+ yaml.dump(dockcomp, f)
+
if __name__ == "__main__":
shutil.copy("common/build-lava", "lava-slave/scripts/build-lava")