summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKevin Hilman <khilman@baylibre.com>2017-08-22 16:07:47 -0700
committerKevin Hilman <khilman@baylibre.com>2017-09-01 15:52:07 -0700
commit05146fa425d94a77ba35b43ac3b0a5c633163d4e (patch)
tree1d68bd51a1dfe8cc6c5e19fe517dbb19f3b67725
parent263fa00cd42334c69897ebad72fcd9d6acd89aeb (diff)
initial snapshot: LAVA job creation based on templates
Change-Id: I5d545d3531f4c4190453724738076bd4eddfc4d6 Signed-off-by: Jerome Brunet <jbrunet@baylibre.com> Signed-off-by: Loys Ollivier <lollivier@baylibre.com> Signed-off-by: Kevin Hilman <khilman@baylibre.com>
-rw-r--r--templates/base/agl-base-defaults.jinja240
-rw-r--r--templates/base/agl-base.jinja236
-rw-r--r--templates/base/agl-simple.jinja27
-rw-r--r--templates/boot/agl-prompt.jinja5
-rw-r--r--templates/boot/generic-base-boot.jinja29
-rw-r--r--templates/boot/generic-qemu-tmpfs.jinja226
-rw-r--r--templates/boot/generic-uboot-tftp.jinja251
-rw-r--r--templates/machines/dra7xx-evm.jinja26
-rw-r--r--templates/machines/m3ulcb.jinja26
-rw-r--r--templates/machines/porter.jinja25
-rw-r--r--templates/machines/qemux86-64.jinja28
-rw-r--r--templates/machines/raspberrypi3.jinja24
-rw-r--r--templates/tests/smoke.jinja212
-rw-r--r--templates/tests/test1.jinja221
-rw-r--r--templates/tests/test2.jinja222
-rw-r--r--utils/agljobtemplate.py73
-rwxr-xr-xutils/create-jobs.py68
17 files changed, 399 insertions, 0 deletions
diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2
new file mode 100644
index 0000000..ff70116
--- /dev/null
+++ b/templates/base/agl-base-defaults.jinja2
@@ -0,0 +1,40 @@
+{%- macro baseurl(object) -%}
+{{ urlbase }}/{{ dl_dir }}/{{ object }}
+{%- endmacro %}
+{%- set dl_dir = dl_dir|default(yocto_machine) %}
+{%- set boot_timeout = boot_timeout|default(10) %}
+{%- set job_timeout = job_timeout|default(30) %}
+{%- set action_timeout = action_timeout|default(15) %}
+{%- set connection_timeout = connection_timeout|default(5) %}
+{%- set deploy_timeout = deploy_timeout|default(15) %}
+{%- set device_type = device_type|default(yocto_machine+"-uboot") %}
+{%- set kernel_image = kernel_image|default('uImage') %}
+{%- set kernel_url = kernel_url|default(baseurl(kernel_image)) %}
+{%- if dtb %}
+{%- set dtb_url = dtb_url|default(baseurl(dtb)) %}
+{%- endif %}
+{%- if modules %}
+{%- set modules_url = modules_url|default(baseurl(modules)) %}
+{%- set modules_compression = modules_compression|default(modules|get_extension) %}
+{%- endif %}
+{%- if rootfs_type == 'nbd' %}
+{%- set deploy_to = "nbd" %}
+{%- set boot_commands = "nbd" %}
+{%- set nbdroot = nbdroot|default("core-image-minimal-" + yocto_machine + ".ext4.xz") %}
+{%- set nbdroot_url = nbdroot_url|default(baseurl(nbdroot)) %}
+{%- set nbdroot_compression = nbdroot_compression|default(nbdroot|get_extension) %}
+{%- set nbdinitrd = nbdinitrd|default("initramfs-netboot-image-" + yocto_machine +".ext4.gz") %}
+{%- set nbdinitrd_url = nbdinitrd_url|default(baseurl(nbdinitrd)) %}
+{%- elif rootfs_type == 'nfs' %}
+{%- set nfsrootfs = nfsrootfs|default("nfsrootfs-" + yocto_machine + ".tar.gz") %}
+{%- set nfsrootfs_url = nfsrootfs_url|default(baseurl(nfsrootfs)) %}
+{%- set nfsroot_compression = nfsroot_compression|default(nfsrootfs|get_extension) %}
+{%- elif rootfs_type == 'ramdisk' %}
+{%- set initrd = initrd|default("initramfs-boot-image-" + yocto_machine +".gz") %}
+{%- set initrd_compression = initrd_compression|default(initrd|get_extension) %}
+{%- endif %}
+{%- if initrd %}
+{%- set initrd_url = initrd_url|default(baseurl(initrd)) %}
+{%- endif %}
+{%- block job %}
+{%- endblock %}
diff --git a/templates/base/agl-base.jinja2 b/templates/base/agl-base.jinja2
new file mode 100644
index 0000000..1119aaa
--- /dev/null
+++ b/templates/base/agl-base.jinja2
@@ -0,0 +1,36 @@
+{%- extends 'base/agl-base-defaults.jinja2' %}
+{%- block job %}
+{%- block metadata %}
+metadata:
+ image.type: 'AGL'
+{% endblock %}
+{%- block main %}
+device_type: {{ device_type }}
+job_name: {{ name }}
+
+timeouts:
+ job:
+ minutes: {{ job_timeout }}
+ action:
+ minutes: {{ action_timeout }}
+ connection:
+ minutes: {{ connection_timeout }}
+priority: {{ priority }}
+visibility: public
+{% endblock %}
+{%- block actions %}
+actions:
+{%- block deploy %}
+- deploy:
+ timeout:
+ minutes: {{ deploy_timeout }}
+{%- endblock %}
+{%- block boot %}
+{% endblock %}
+{%- block tests %}
+{%-for test_template in test_templates %}
+{% include test_template %}
+{%- endfor %}
+{% endblock %}
+{% endblock %}
+{% endblock %}
diff --git a/templates/base/agl-simple.jinja2 b/templates/base/agl-simple.jinja2
new file mode 100644
index 0000000..b2802a3
--- /dev/null
+++ b/templates/base/agl-simple.jinja2
@@ -0,0 +1,7 @@
+{%- extends 'base/agl-base.jinja2' %}
+{%- block deploy -%}
+{{ super() }}
+ to: {{ deploy_to|default("tftp") }}
+ os: {{ deploy_os|default("oe") }}
+ failure_retry: 2
+{%- endblock %}
diff --git a/templates/boot/agl-prompt.jinja b/templates/boot/agl-prompt.jinja
new file mode 100644
index 0000000..9d8cd50
--- /dev/null
+++ b/templates/boot/agl-prompt.jinja
@@ -0,0 +1,5 @@
+
+ prompts: ["root@{{ yocto_machine }}:~"]
+ auto_login:
+ login_prompt: "login:"
+ username: root
diff --git a/templates/boot/generic-base-boot.jinja2 b/templates/boot/generic-base-boot.jinja2
new file mode 100644
index 0000000..5011876
--- /dev/null
+++ b/templates/boot/generic-base-boot.jinja2
@@ -0,0 +1,9 @@
+{%- extends 'base/agl-simple.jinja2' %}
+{%- block boot %}
+{{ super() }}
+- boot:
+ timeout:
+ minutes: {{ boot_timeout }}
+ method: {{ boot_method|default("u-boot") }}
+{%- include 'boot/agl-prompt.jinja' %}
+{%- endblock %}
diff --git a/templates/boot/generic-qemu-tmpfs.jinja2 b/templates/boot/generic-qemu-tmpfs.jinja2
new file mode 100644
index 0000000..3908c80
--- /dev/null
+++ b/templates/boot/generic-qemu-tmpfs.jinja2
@@ -0,0 +1,26 @@
+{%- extends 'boot/generic-base-boot.jinja2' %}
+{%- set boot_method = "qemu" %}
+{%- block boot %}
+{{ super() }}
+ media: {{ media_type|default("tmpfs") }}
+{%- endblock %}
+{%- block main %}
+{{ super() }}
+context:
+ no_kvm: false
+ arch: {{ qemu_arch }}
+ extra_options: ["{{ qemu_args }}"]
+{% endblock %}
+{%- block deploy -%}
+{{ super() }}
+ images:
+ kernel:
+ image_arg: '-kernel {kernel} -append "{{ qemu_cmdline }}"'
+ url: {{ kernel_url }}
+{%- if initrd_url and rootfs_type == 'ramdisk' %}
+ ramdisk:
+ image_arg: '-drive format=raw,file={ramdisk}'
+ url: {{ initrd_url }}
+ compression: {{ initrd_compression }}
+{%- endif %}
+{%- endblock %}
diff --git a/templates/boot/generic-uboot-tftp.jinja2 b/templates/boot/generic-uboot-tftp.jinja2
new file mode 100644
index 0000000..cbdda7a
--- /dev/null
+++ b/templates/boot/generic-uboot-tftp.jinja2
@@ -0,0 +1,51 @@
+{%- extends 'boot/generic-base-boot.jinja2' %}
+{%- set boot_method = "u-boot" %}
+{%- block main %}
+{{ super() }}
+{%- if rootfs_type == 'nbd' %}
+protocols:
+ lava-xnbd:
+ port: auto
+{%- endif %}
+{% endblock %}
+{%- block boot %}
+{{ super() }}
+ type: {{ uboot_type|default("bootm") }}
+ commands: {{ boot_commands|default("ramdisk") }}
+{%- if rootfs_type == 'nbd' %}
+ transfer_overlay:
+ download_command: wget
+ unpack_command: tar -C / -xvpf
+{%- endif %}
+{%- endblock %}
+{%- block deploy -%}
+{{ super() }}
+ kernel:
+ url: {{ kernel_url }}
+{%- if rootfs_type == 'nfs' %}
+ nfsrootfs:
+ url: {{ nfsrootfs_url }}
+ compression: {{ nfsroot_compression }}
+{%- elif rootfs_type == 'nbd' %}
+ initrd:
+ url: {{ nbdinitrd_url }}
+ allow_modify: false
+ nbdroot:
+ url: {{ nbdroot_url }}
+ compression: {{ nbdroot_compression }}
+{%- endif %}
+{%- if initrd_url and rootfs_type != 'nbd' %}
+ ramdisk:
+ url: {{ initrd_url }}
+ compression: {{ initrd_compression }}
+{%- endif %}
+{%- if modules_url %}
+ modules:
+ url: {{ modules_url }}
+ compression: {{ modules_compression }}
+{%- endif %}
+{%- if dtb_url %}
+ dtb:
+ url: {{ dtb_url }}
+{%- endif %}
+{%- endblock %}
diff --git a/templates/machines/dra7xx-evm.jinja2 b/templates/machines/dra7xx-evm.jinja2
new file mode 100644
index 0000000..3e16590
--- /dev/null
+++ b/templates/machines/dra7xx-evm.jinja2
@@ -0,0 +1,6 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "ti-vayu-uboot" %}
+{%- set dtb = "zImage-dra7-evm-lcd-lg.dtb" %}
+{%- set kernel_image = "zImage" %}
+{%- set uboot_type = "bootz" %}
+{%- set nbdroot = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %}
diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2
new file mode 100644
index 0000000..e9bf6a3
--- /dev/null
+++ b/templates/machines/m3ulcb.jinja2
@@ -0,0 +1,6 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "r8a7796-m3ulcb" %}
+{%- set kernel_image = "Image" %}
+{%- set dtb = "Image-r8a7796-m3ulcb.dtb" %}
+{%- set uboot_type = "booti" %}
+{%- set dl_dir = "m3ulcb-nogfx" %}
diff --git a/templates/machines/porter.jinja2 b/templates/machines/porter.jinja2
new file mode 100644
index 0000000..36dd96a
--- /dev/null
+++ b/templates/machines/porter.jinja2
@@ -0,0 +1,5 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "renesas-porter-uboot" %}
+{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %}
+{%- set dtb = "uImage-r8a7791-porter.dtb" %}
+{%- set dl_dir = "porter-nogfx" %}
diff --git a/templates/machines/qemux86-64.jinja2 b/templates/machines/qemux86-64.jinja2
new file mode 100644
index 0000000..d76106c
--- /dev/null
+++ b/templates/machines/qemux86-64.jinja2
@@ -0,0 +1,8 @@
+{%- extends 'boot/generic-qemu-tmpfs.jinja2' %}
+{%- set device_type = "qemu" %}
+{%- set kernel_image = "bzImage" %}
+{%- set qemu_cmdline = "console=ttyS0,115200 root=/dev/hda debug verbose" %}
+{%- set qemu_args = "-cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt -m 1048 -soundhw hda" %}
+{%- set qemu_arch = "x86_64" %}
+{%- set deploy_to = "tmpfs" %}
+{%- set initrd = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %}
diff --git a/templates/machines/raspberrypi3.jinja2 b/templates/machines/raspberrypi3.jinja2
new file mode 100644
index 0000000..ed36889
--- /dev/null
+++ b/templates/machines/raspberrypi3.jinja2
@@ -0,0 +1,4 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "bcm2837-rpi-3-b" %}
+{%- set dtb = "uImage-bcm2710-rpi-3-b.dtb" %}
+{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %}
diff --git a/templates/tests/smoke.jinja2 b/templates/tests/smoke.jinja2
new file mode 100644
index 0000000..c3e62dc
--- /dev/null
+++ b/templates/tests/smoke.jinja2
@@ -0,0 +1,12 @@
+
+- test:
+ definitions:
+ - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+ from: git
+ path: test-suites/short-smoke/busybox.yaml
+ name: busybox
+ - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+ from: git
+ path: test-suites/short-smoke/smoke-tests-basic.yaml
+ name: smoke-tests-basic
+
diff --git a/templates/tests/test1.jinja2 b/templates/tests/test1.jinja2
new file mode 100644
index 0000000..8b04691
--- /dev/null
+++ b/templates/tests/test1.jinja2
@@ -0,0 +1,21 @@
+
+- test:
+ timeout:
+ minutes: 2
+ definitions:
+ - repository:
+ metadata:
+ format: Lava-Test Test Definition 1.0
+ name: test1
+ description: "test1"
+ os:
+ - debian
+ scope:
+ - functional
+ run:
+ steps:
+ - lava-test-set start test1
+ - lava-test-case always-pass --shell true
+ from: inline
+ name: dummytest1
+ path: inline/test1.yaml
diff --git a/templates/tests/test2.jinja2 b/templates/tests/test2.jinja2
new file mode 100644
index 0000000..4dd4a14
--- /dev/null
+++ b/templates/tests/test2.jinja2
@@ -0,0 +1,22 @@
+
+- test:
+ timeout:
+ minutes: 2
+ definitions:
+ - repository:
+ metadata:
+ format: Lava-Test Test Definition 1.0
+ name: test2
+ description: "test2"
+ os:
+ - debian
+ scope:
+ - functional
+ run:
+ steps:
+ - lava-test-set start test2
+ - lava-test-case always-fail --shell false
+ from: inline
+ name: dummytest2
+ path: inline/test2.yaml
+
diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
new file mode 100644
index 0000000..749d4bf
--- /dev/null
+++ b/utils/agljobtemplate.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import jinja2
+
+
+def get_extension(path):
+ return path.split('.')[-1]
+
+
+class Agljobtemplate(object):
+
+ DEFAULT_PATH = "templates"
+ MACHINES_DIR = "machines"
+ TESTS_DIR = "tests"
+ RFS_TYPE = ['nfs', 'nbd', 'ramdisk']
+
+ def __init__(self, path=DEFAULT_PATH):
+ self._template_path = os.path.normpath(path)
+ if not (os.path.isdir(self._template_path) and os.access(self._template_path, os.F_OK)):
+ raise OSError, "Cannot access {}".format(self._template_path)
+
+ if self.machines is None:
+ raise RuntimeError, "No machine directory found at {}".format(self._template_path)
+
+ def __list_jinjas(self, directory):
+ d = os.path.join(self._template_path, directory)
+ return [os.path.splitext(os.path.basename(f))[0] for f in os.listdir(d) if f.endswith('.jinja2')]
+
+ @property
+ def machines(self):
+ """ List the availables machines
+ """
+ return self.__list_jinjas(self.MACHINES_DIR)
+
+ @property
+ def tests(self):
+ """ List the availables tests
+ """
+ return self.__list_jinjas(self.TESTS_DIR)
+
+ @property
+ def rfs_types(self):
+ return self.RFS_TYPE
+
+ def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type="nbd"):
+ test_templates = []
+
+ if machine not in self.machines:
+ raise RuntimeError, "{} is not a available machine".format(machine)
+
+ for t in tests:
+ if t in self.tests:
+ test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2'))
+ else:
+ raise RuntimeError, "{} is not an available test".format(t)
+
+ # Populate jinja substitution dict
+ job = {}
+ job['name'] = job_name
+ job['yocto_machine'] = machine
+ job['priority'] = priority
+ job['urlbase'] = url
+ job['test_templates'] = test_templates
+ job['rootfs_type'] = rfs_type
+
+ env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_path))
+ env.filters['get_extension'] = get_extension
+ template = env.get_template(os.path.join(self.MACHINES_DIR, machine + ".jinja2"))
+
+ return template.render(job)
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
new file mode 100755
index 0000000..1c15bb6
--- /dev/null
+++ b/utils/create-jobs.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import agljobtemplate
+import argparse
+import urlparse
+
+
+def parse_cmdline(machines, tests, rfs_types):
+ parser = argparse.ArgumentParser(description="AGL create job",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0')
+ parser.add_argument('machine', action='store', choices=machines,
+ help="machine to generate the job for")
+ parser.add_argument('--priority', '-p', action='store', dest='priority',
+ help="job priority",
+ default='medium')
+ parser.add_argument('--urlbase', '-u', action='store', dest='urlbase',
+ help="url fetch base",
+ default='https://download.automotivelinux.org/AGL/upload/ci/')
+ parser.add_argument('--boot', action='store', dest='rfs_type', nargs=1, required=True,
+ choices=rfs_types, help='select boot type')
+ parser.add_argument('--test', dest='tests', action='store', choices=tests + ['all'],
+ help="add these test to the job", nargs='*', default=[])
+ parser.add_argument('-o', '--output', dest='job_file', action='store',
+ help="destination file")
+ parser.add_argument('-n', '--name', dest='job_name', action='store',
+ help="job name", default='AGL-short-smoke-wip')
+ parser.add_argument('-j', '--jobid', dest='job_id', action='store',
+ help='job id for link creation: URLBASE/JOB_ID', required=True)
+ parser.add_argument('-i', '--jobidx', dest='job_index', action='store',
+ help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1')
+
+ return parser.parse_args()
+
+
+def main():
+ ajt = agljobtemplate.Agljobtemplate('templates')
+ args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types)
+
+ if args.tests is not None and 'all' in args.tests:
+ args.tests = ajt.tests
+
+ if args.job_id is not None:
+ args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/')
+ args.job_name += ' - {}'.format(args.job_id)
+
+ if args.job_index is not None:
+ args.urlbase = urlparse.urljoin(args.urlbase, args.job_index)
+ args.job_name += ' - {}'.format(args.job_index)
+
+ job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority,
+ rfs_type=args.rfs_type[0], job_name=args.job_name)
+
+ if args.job_file is None:
+ print job
+ else:
+ try:
+ with open(args.job_file, 'w') as j:
+ j.write(job)
+ except IOError as e:
+ print "{}: {}".format(e.strerror, args.job_file)
+ exit(e.errno)
+ else:
+ print "Job written to: {}".format(args.job_file)
+
+if __name__ == '__main__':
+ main()