From 05146fa425d94a77ba35b43ac3b0a5c633163d4e Mon Sep 17 00:00:00 2001 From: Kevin Hilman Date: Tue, 22 Aug 2017 16:07:47 -0700 Subject: initial snapshot: LAVA job creation based on templates Change-Id: I5d545d3531f4c4190453724738076bd4eddfc4d6 Signed-off-by: Jerome Brunet Signed-off-by: Loys Ollivier Signed-off-by: Kevin Hilman --- templates/base/agl-base-defaults.jinja2 | 40 +++++++++++++++++ templates/base/agl-base.jinja2 | 36 ++++++++++++++++ templates/base/agl-simple.jinja2 | 7 +++ templates/boot/agl-prompt.jinja | 5 +++ templates/boot/generic-base-boot.jinja2 | 9 ++++ templates/boot/generic-qemu-tmpfs.jinja2 | 26 ++++++++++++ templates/boot/generic-uboot-tftp.jinja2 | 51 ++++++++++++++++++++++ templates/machines/dra7xx-evm.jinja2 | 6 +++ templates/machines/m3ulcb.jinja2 | 6 +++ templates/machines/porter.jinja2 | 5 +++ templates/machines/qemux86-64.jinja2 | 8 ++++ templates/machines/raspberrypi3.jinja2 | 4 ++ templates/tests/smoke.jinja2 | 12 ++++++ templates/tests/test1.jinja2 | 21 +++++++++ templates/tests/test2.jinja2 | 22 ++++++++++ utils/agljobtemplate.py | 73 ++++++++++++++++++++++++++++++++ utils/create-jobs.py | 68 +++++++++++++++++++++++++++++ 17 files changed, 399 insertions(+) create mode 100644 templates/base/agl-base-defaults.jinja2 create mode 100644 templates/base/agl-base.jinja2 create mode 100644 templates/base/agl-simple.jinja2 create mode 100644 templates/boot/agl-prompt.jinja create mode 100644 templates/boot/generic-base-boot.jinja2 create mode 100644 templates/boot/generic-qemu-tmpfs.jinja2 create mode 100644 templates/boot/generic-uboot-tftp.jinja2 create mode 100644 templates/machines/dra7xx-evm.jinja2 create mode 100644 templates/machines/m3ulcb.jinja2 create mode 100644 templates/machines/porter.jinja2 create mode 100644 templates/machines/qemux86-64.jinja2 create mode 100644 templates/machines/raspberrypi3.jinja2 create mode 100644 templates/tests/smoke.jinja2 create mode 100644 templates/tests/test1.jinja2 create mode 100644 templates/tests/test2.jinja2 create mode 100644 utils/agljobtemplate.py create mode 100755 utils/create-jobs.py diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2 new file mode 100644 index 0000000..ff70116 --- /dev/null +++ b/templates/base/agl-base-defaults.jinja2 @@ -0,0 +1,40 @@ +{%- macro baseurl(object) -%} +{{ urlbase }}/{{ dl_dir }}/{{ object }} +{%- endmacro %} +{%- set dl_dir = dl_dir|default(yocto_machine) %} +{%- set boot_timeout = boot_timeout|default(10) %} +{%- set job_timeout = job_timeout|default(30) %} +{%- set action_timeout = action_timeout|default(15) %} +{%- set connection_timeout = connection_timeout|default(5) %} +{%- set deploy_timeout = deploy_timeout|default(15) %} +{%- set device_type = device_type|default(yocto_machine+"-uboot") %} +{%- set kernel_image = kernel_image|default('uImage') %} +{%- set kernel_url = kernel_url|default(baseurl(kernel_image)) %} +{%- if dtb %} +{%- set dtb_url = dtb_url|default(baseurl(dtb)) %} +{%- endif %} +{%- if modules %} +{%- set modules_url = modules_url|default(baseurl(modules)) %} +{%- set modules_compression = modules_compression|default(modules|get_extension) %} +{%- endif %} +{%- if rootfs_type == 'nbd' %} +{%- set deploy_to = "nbd" %} +{%- set boot_commands = "nbd" %} +{%- set nbdroot = nbdroot|default("core-image-minimal-" + yocto_machine + ".ext4.xz") %} +{%- set nbdroot_url = nbdroot_url|default(baseurl(nbdroot)) %} +{%- set nbdroot_compression = nbdroot_compression|default(nbdroot|get_extension) %} +{%- set nbdinitrd = nbdinitrd|default("initramfs-netboot-image-" + yocto_machine +".ext4.gz") %} +{%- set nbdinitrd_url = nbdinitrd_url|default(baseurl(nbdinitrd)) %} +{%- elif rootfs_type == 'nfs' %} +{%- set nfsrootfs = nfsrootfs|default("nfsrootfs-" + yocto_machine + ".tar.gz") %} +{%- set nfsrootfs_url = nfsrootfs_url|default(baseurl(nfsrootfs)) %} +{%- set nfsroot_compression = nfsroot_compression|default(nfsrootfs|get_extension) %} +{%- elif rootfs_type == 'ramdisk' %} +{%- set initrd = initrd|default("initramfs-boot-image-" + yocto_machine +".gz") %} +{%- set initrd_compression = initrd_compression|default(initrd|get_extension) %} +{%- endif %} +{%- if initrd %} +{%- set initrd_url = initrd_url|default(baseurl(initrd)) %} +{%- endif %} +{%- block job %} +{%- endblock %} diff --git a/templates/base/agl-base.jinja2 b/templates/base/agl-base.jinja2 new file mode 100644 index 0000000..1119aaa --- /dev/null +++ b/templates/base/agl-base.jinja2 @@ -0,0 +1,36 @@ +{%- extends 'base/agl-base-defaults.jinja2' %} +{%- block job %} +{%- block metadata %} +metadata: + image.type: 'AGL' +{% endblock %} +{%- block main %} +device_type: {{ device_type }} +job_name: {{ name }} + +timeouts: + job: + minutes: {{ job_timeout }} + action: + minutes: {{ action_timeout }} + connection: + minutes: {{ connection_timeout }} +priority: {{ priority }} +visibility: public +{% endblock %} +{%- block actions %} +actions: +{%- block deploy %} +- deploy: + timeout: + minutes: {{ deploy_timeout }} +{%- endblock %} +{%- block boot %} +{% endblock %} +{%- block tests %} +{%-for test_template in test_templates %} +{% include test_template %} +{%- endfor %} +{% endblock %} +{% endblock %} +{% endblock %} diff --git a/templates/base/agl-simple.jinja2 b/templates/base/agl-simple.jinja2 new file mode 100644 index 0000000..b2802a3 --- /dev/null +++ b/templates/base/agl-simple.jinja2 @@ -0,0 +1,7 @@ +{%- extends 'base/agl-base.jinja2' %} +{%- block deploy -%} +{{ super() }} + to: {{ deploy_to|default("tftp") }} + os: {{ deploy_os|default("oe") }} + failure_retry: 2 +{%- endblock %} diff --git a/templates/boot/agl-prompt.jinja b/templates/boot/agl-prompt.jinja new file mode 100644 index 0000000..9d8cd50 --- /dev/null +++ b/templates/boot/agl-prompt.jinja @@ -0,0 +1,5 @@ + + prompts: ["root@{{ yocto_machine }}:~"] + auto_login: + login_prompt: "login:" + username: root diff --git a/templates/boot/generic-base-boot.jinja2 b/templates/boot/generic-base-boot.jinja2 new file mode 100644 index 0000000..5011876 --- /dev/null +++ b/templates/boot/generic-base-boot.jinja2 @@ -0,0 +1,9 @@ +{%- extends 'base/agl-simple.jinja2' %} +{%- block boot %} +{{ super() }} +- boot: + timeout: + minutes: {{ boot_timeout }} + method: {{ boot_method|default("u-boot") }} +{%- include 'boot/agl-prompt.jinja' %} +{%- endblock %} diff --git a/templates/boot/generic-qemu-tmpfs.jinja2 b/templates/boot/generic-qemu-tmpfs.jinja2 new file mode 100644 index 0000000..3908c80 --- /dev/null +++ b/templates/boot/generic-qemu-tmpfs.jinja2 @@ -0,0 +1,26 @@ +{%- extends 'boot/generic-base-boot.jinja2' %} +{%- set boot_method = "qemu" %} +{%- block boot %} +{{ super() }} + media: {{ media_type|default("tmpfs") }} +{%- endblock %} +{%- block main %} +{{ super() }} +context: + no_kvm: false + arch: {{ qemu_arch }} + extra_options: ["{{ qemu_args }}"] +{% endblock %} +{%- block deploy -%} +{{ super() }} + images: + kernel: + image_arg: '-kernel {kernel} -append "{{ qemu_cmdline }}"' + url: {{ kernel_url }} +{%- if initrd_url and rootfs_type == 'ramdisk' %} + ramdisk: + image_arg: '-drive format=raw,file={ramdisk}' + url: {{ initrd_url }} + compression: {{ initrd_compression }} +{%- endif %} +{%- endblock %} diff --git a/templates/boot/generic-uboot-tftp.jinja2 b/templates/boot/generic-uboot-tftp.jinja2 new file mode 100644 index 0000000..cbdda7a --- /dev/null +++ b/templates/boot/generic-uboot-tftp.jinja2 @@ -0,0 +1,51 @@ +{%- extends 'boot/generic-base-boot.jinja2' %} +{%- set boot_method = "u-boot" %} +{%- block main %} +{{ super() }} +{%- if rootfs_type == 'nbd' %} +protocols: + lava-xnbd: + port: auto +{%- endif %} +{% endblock %} +{%- block boot %} +{{ super() }} + type: {{ uboot_type|default("bootm") }} + commands: {{ boot_commands|default("ramdisk") }} +{%- if rootfs_type == 'nbd' %} + transfer_overlay: + download_command: wget + unpack_command: tar -C / -xvpf +{%- endif %} +{%- endblock %} +{%- block deploy -%} +{{ super() }} + kernel: + url: {{ kernel_url }} +{%- if rootfs_type == 'nfs' %} + nfsrootfs: + url: {{ nfsrootfs_url }} + compression: {{ nfsroot_compression }} +{%- elif rootfs_type == 'nbd' %} + initrd: + url: {{ nbdinitrd_url }} + allow_modify: false + nbdroot: + url: {{ nbdroot_url }} + compression: {{ nbdroot_compression }} +{%- endif %} +{%- if initrd_url and rootfs_type != 'nbd' %} + ramdisk: + url: {{ initrd_url }} + compression: {{ initrd_compression }} +{%- endif %} +{%- if modules_url %} + modules: + url: {{ modules_url }} + compression: {{ modules_compression }} +{%- endif %} +{%- if dtb_url %} + dtb: + url: {{ dtb_url }} +{%- endif %} +{%- endblock %} diff --git a/templates/machines/dra7xx-evm.jinja2 b/templates/machines/dra7xx-evm.jinja2 new file mode 100644 index 0000000..3e16590 --- /dev/null +++ b/templates/machines/dra7xx-evm.jinja2 @@ -0,0 +1,6 @@ +{%- extends 'boot/generic-uboot-tftp.jinja2' %} +{%- set device_type = "ti-vayu-uboot" %} +{%- set dtb = "zImage-dra7-evm-lcd-lg.dtb" %} +{%- set kernel_image = "zImage" %} +{%- set uboot_type = "bootz" %} +{%- set nbdroot = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %} diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2 new file mode 100644 index 0000000..e9bf6a3 --- /dev/null +++ b/templates/machines/m3ulcb.jinja2 @@ -0,0 +1,6 @@ +{%- extends 'boot/generic-uboot-tftp.jinja2' %} +{%- set device_type = "r8a7796-m3ulcb" %} +{%- set kernel_image = "Image" %} +{%- set dtb = "Image-r8a7796-m3ulcb.dtb" %} +{%- set uboot_type = "booti" %} +{%- set dl_dir = "m3ulcb-nogfx" %} diff --git a/templates/machines/porter.jinja2 b/templates/machines/porter.jinja2 new file mode 100644 index 0000000..36dd96a --- /dev/null +++ b/templates/machines/porter.jinja2 @@ -0,0 +1,5 @@ +{%- extends 'boot/generic-uboot-tftp.jinja2' %} +{%- set device_type = "renesas-porter-uboot" %} +{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %} +{%- set dtb = "uImage-r8a7791-porter.dtb" %} +{%- set dl_dir = "porter-nogfx" %} diff --git a/templates/machines/qemux86-64.jinja2 b/templates/machines/qemux86-64.jinja2 new file mode 100644 index 0000000..d76106c --- /dev/null +++ b/templates/machines/qemux86-64.jinja2 @@ -0,0 +1,8 @@ +{%- extends 'boot/generic-qemu-tmpfs.jinja2' %} +{%- set device_type = "qemu" %} +{%- set kernel_image = "bzImage" %} +{%- set qemu_cmdline = "console=ttyS0,115200 root=/dev/hda debug verbose" %} +{%- set qemu_args = "-cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt -m 1048 -soundhw hda" %} +{%- set qemu_arch = "x86_64" %} +{%- set deploy_to = "tmpfs" %} +{%- set initrd = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %} diff --git a/templates/machines/raspberrypi3.jinja2 b/templates/machines/raspberrypi3.jinja2 new file mode 100644 index 0000000..ed36889 --- /dev/null +++ b/templates/machines/raspberrypi3.jinja2 @@ -0,0 +1,4 @@ +{%- extends 'boot/generic-uboot-tftp.jinja2' %} +{%- set device_type = "bcm2837-rpi-3-b" %} +{%- set dtb = "uImage-bcm2710-rpi-3-b.dtb" %} +{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %} diff --git a/templates/tests/smoke.jinja2 b/templates/tests/smoke.jinja2 new file mode 100644 index 0000000..c3e62dc --- /dev/null +++ b/templates/tests/smoke.jinja2 @@ -0,0 +1,12 @@ + +- test: + definitions: + - repository: https://git.automotivelinux.org/src/qa-testdefinitions + from: git + path: test-suites/short-smoke/busybox.yaml + name: busybox + - repository: https://git.automotivelinux.org/src/qa-testdefinitions + from: git + path: test-suites/short-smoke/smoke-tests-basic.yaml + name: smoke-tests-basic + diff --git a/templates/tests/test1.jinja2 b/templates/tests/test1.jinja2 new file mode 100644 index 0000000..8b04691 --- /dev/null +++ b/templates/tests/test1.jinja2 @@ -0,0 +1,21 @@ + +- test: + timeout: + minutes: 2 + definitions: + - repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: test1 + description: "test1" + os: + - debian + scope: + - functional + run: + steps: + - lava-test-set start test1 + - lava-test-case always-pass --shell true + from: inline + name: dummytest1 + path: inline/test1.yaml diff --git a/templates/tests/test2.jinja2 b/templates/tests/test2.jinja2 new file mode 100644 index 0000000..4dd4a14 --- /dev/null +++ b/templates/tests/test2.jinja2 @@ -0,0 +1,22 @@ + +- test: + timeout: + minutes: 2 + definitions: + - repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: test2 + description: "test2" + os: + - debian + scope: + - functional + run: + steps: + - lava-test-set start test2 + - lava-test-case always-fail --shell false + from: inline + name: dummytest2 + path: inline/test2.yaml + diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py new file mode 100644 index 0000000..749d4bf --- /dev/null +++ b/utils/agljobtemplate.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import os +import sys +import jinja2 + + +def get_extension(path): + return path.split('.')[-1] + + +class Agljobtemplate(object): + + DEFAULT_PATH = "templates" + MACHINES_DIR = "machines" + TESTS_DIR = "tests" + RFS_TYPE = ['nfs', 'nbd', 'ramdisk'] + + def __init__(self, path=DEFAULT_PATH): + self._template_path = os.path.normpath(path) + if not (os.path.isdir(self._template_path) and os.access(self._template_path, os.F_OK)): + raise OSError, "Cannot access {}".format(self._template_path) + + if self.machines is None: + raise RuntimeError, "No machine directory found at {}".format(self._template_path) + + def __list_jinjas(self, directory): + d = os.path.join(self._template_path, directory) + return [os.path.splitext(os.path.basename(f))[0] for f in os.listdir(d) if f.endswith('.jinja2')] + + @property + def machines(self): + """ List the availables machines + """ + return self.__list_jinjas(self.MACHINES_DIR) + + @property + def tests(self): + """ List the availables tests + """ + return self.__list_jinjas(self.TESTS_DIR) + + @property + def rfs_types(self): + return self.RFS_TYPE + + def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type="nbd"): + test_templates = [] + + if machine not in self.machines: + raise RuntimeError, "{} is not a available machine".format(machine) + + for t in tests: + if t in self.tests: + test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2')) + else: + raise RuntimeError, "{} is not an available test".format(t) + + # Populate jinja substitution dict + job = {} + job['name'] = job_name + job['yocto_machine'] = machine + job['priority'] = priority + job['urlbase'] = url + job['test_templates'] = test_templates + job['rootfs_type'] = rfs_type + + env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_path)) + env.filters['get_extension'] = get_extension + template = env.get_template(os.path.join(self.MACHINES_DIR, machine + ".jinja2")) + + return template.render(job) diff --git a/utils/create-jobs.py b/utils/create-jobs.py new file mode 100755 index 0000000..1c15bb6 --- /dev/null +++ b/utils/create-jobs.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import agljobtemplate +import argparse +import urlparse + + +def parse_cmdline(machines, tests, rfs_types): + parser = argparse.ArgumentParser(description="AGL create job", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0') + parser.add_argument('machine', action='store', choices=machines, + help="machine to generate the job for") + parser.add_argument('--priority', '-p', action='store', dest='priority', + help="job priority", + default='medium') + parser.add_argument('--urlbase', '-u', action='store', dest='urlbase', + help="url fetch base", + default='https://download.automotivelinux.org/AGL/upload/ci/') + parser.add_argument('--boot', action='store', dest='rfs_type', nargs=1, required=True, + choices=rfs_types, help='select boot type') + parser.add_argument('--test', dest='tests', action='store', choices=tests + ['all'], + help="add these test to the job", nargs='*', default=[]) + parser.add_argument('-o', '--output', dest='job_file', action='store', + help="destination file") + parser.add_argument('-n', '--name', dest='job_name', action='store', + help="job name", default='AGL-short-smoke-wip') + parser.add_argument('-j', '--jobid', dest='job_id', action='store', + help='job id for link creation: URLBASE/JOB_ID', required=True) + parser.add_argument('-i', '--jobidx', dest='job_index', action='store', + help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1') + + return parser.parse_args() + + +def main(): + ajt = agljobtemplate.Agljobtemplate('templates') + args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types) + + if args.tests is not None and 'all' in args.tests: + args.tests = ajt.tests + + if args.job_id is not None: + args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/') + args.job_name += ' - {}'.format(args.job_id) + + if args.job_index is not None: + args.urlbase = urlparse.urljoin(args.urlbase, args.job_index) + args.job_name += ' - {}'.format(args.job_index) + + job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority, + rfs_type=args.rfs_type[0], job_name=args.job_name) + + if args.job_file is None: + print job + else: + try: + with open(args.job_file, 'w') as j: + j.write(job) + except IOError as e: + print "{}: {}".format(e.strerror, args.job_file) + exit(e.errno) + else: + print "Job written to: {}".format(args.job_file) + +if __name__ == '__main__': + main() -- cgit 1.2.3-korg