initial snapshot: LAVA job creation based on templates

Change-Id: I5d545d3531f4c4190453724738076bd4eddfc4d6
Signed-off-by: Jerome Brunet <jbrunet@baylibre.com>
Signed-off-by: Loys Ollivier <lollivier@baylibre.com>
Signed-off-by: Kevin Hilman <khilman@baylibre.com>

17 files changed, 399 insertions, 0 deletions

diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2
new file mode 100644
index 0000000..ff70116
--- /dev/null
+++ b/templates/base/agl-base-defaults.jinja2
@@ -0,0 +1,40 @@
+{%- macro baseurl(object) -%}
+{{ urlbase }}/{{ dl_dir }}/{{ object }}
+{%- endmacro %}
+{%- set dl_dir = dl_dir|default(yocto_machine) %}
+{%- set boot_timeout = boot_timeout|default(10) %}
+{%- set job_timeout = job_timeout|default(30) %}
+{%- set action_timeout = action_timeout|default(15) %}
+{%- set connection_timeout = connection_timeout|default(5) %}
+{%- set deploy_timeout = deploy_timeout|default(15) %}
+{%- set device_type = device_type|default(yocto_machine+"-uboot") %}
+{%- set kernel_image = kernel_image|default('uImage') %}
+{%- set kernel_url = kernel_url|default(baseurl(kernel_image)) %}
+{%- if dtb %}
+{%- set dtb_url = dtb_url|default(baseurl(dtb)) %}
+{%- endif %}
+{%- if modules %}
+{%- set modules_url = modules_url|default(baseurl(modules)) %}
+{%- set modules_compression = modules_compression|default(modules|get_extension) %}
+{%- endif %}
+{%- if rootfs_type == 'nbd' %}
+{%- set deploy_to = "nbd" %}
+{%- set boot_commands = "nbd" %}
+{%- set nbdroot = nbdroot|default("core-image-minimal-" + yocto_machine + ".ext4.xz") %}
+{%- set nbdroot_url = nbdroot_url|default(baseurl(nbdroot)) %}
+{%- set nbdroot_compression = nbdroot_compression|default(nbdroot|get_extension) %}
+{%- set nbdinitrd = nbdinitrd|default("initramfs-netboot-image-" + yocto_machine +".ext4.gz") %}
+{%- set nbdinitrd_url = nbdinitrd_url|default(baseurl(nbdinitrd)) %}
+{%- elif rootfs_type == 'nfs' %}
+{%- set nfsrootfs = nfsrootfs|default("nfsrootfs-" + yocto_machine + ".tar.gz") %}
+{%- set nfsrootfs_url = nfsrootfs_url|default(baseurl(nfsrootfs)) %}
+{%- set nfsroot_compression = nfsroot_compression|default(nfsrootfs|get_extension) %}
+{%- elif rootfs_type == 'ramdisk' %}
+{%- set initrd = initrd|default("initramfs-boot-image-" + yocto_machine +".gz") %}
+{%- set initrd_compression = initrd_compression|default(initrd|get_extension) %}
+{%- endif %}
+{%- if initrd %}
+{%- set initrd_url = initrd_url|default(baseurl(initrd)) %}
+{%- endif %}
+{%- block job %}
+{%- endblock %}
diff --git a/templates/base/agl-base.jinja2 b/templates/base/agl-base.jinja2
new file mode 100644
index 0000000..1119aaa
--- /dev/null
+++ b/templates/base/agl-base.jinja2
@@ -0,0 +1,36 @@
+{%- extends 'base/agl-base-defaults.jinja2' %}
+{%- block job %}
+{%- block metadata %}
+metadata:
+  image.type: 'AGL'
+{% endblock %}
+{%- block main %}
+device_type: {{ device_type }}
+job_name: {{ name }}
+
+timeouts:
+  job:
+    minutes: {{ job_timeout }}
+  action:
+    minutes: {{ action_timeout }}
+  connection:
+    minutes: {{ connection_timeout }}
+priority: {{ priority }}
+visibility: public
+{% endblock %}
+{%- block actions %}
+actions:
+{%- block deploy %}
+- deploy:
+    timeout:
+      minutes: {{ deploy_timeout }}
+{%- endblock %}
+{%- block boot %}
+{% endblock %}
+{%- block tests %}
+{%-for test_template in test_templates %}
+{% include test_template %}
+{%- endfor %}
+{% endblock %}
+{% endblock %}
+{% endblock %}
diff --git a/templates/base/agl-simple.jinja2 b/templates/base/agl-simple.jinja2
new file mode 100644
index 0000000..b2802a3
--- /dev/null
+++ b/templates/base/agl-simple.jinja2
@@ -0,0 +1,7 @@
+{%- extends 'base/agl-base.jinja2' %}
+{%- block deploy -%}
+{{ super() }}
+    to: {{ deploy_to|default("tftp") }}
+    os: {{ deploy_os|default("oe") }}
+    failure_retry: 2
+{%- endblock %}
diff --git a/templates/boot/agl-prompt.jinja b/templates/boot/agl-prompt.jinja
new file mode 100644
index 0000000..9d8cd50
--- /dev/null
+++ b/templates/boot/agl-prompt.jinja
@@ -0,0 +1,5 @@
+
+    prompts: ["root@{{ yocto_machine }}:~"]
+    auto_login:
+      login_prompt: "login:"
+      username: root
diff --git a/templates/boot/generic-base-boot.jinja2 b/templates/boot/generic-base-boot.jinja2
new file mode 100644
index 0000000..5011876
--- /dev/null
+++ b/templates/boot/generic-base-boot.jinja2
@@ -0,0 +1,9 @@
+{%- extends 'base/agl-simple.jinja2' %}
+{%- block boot %}
+{{ super() }}
+- boot:
+    timeout:
+      minutes: {{ boot_timeout }}
+    method: {{ boot_method|default("u-boot") }}
+{%- include 'boot/agl-prompt.jinja' %}
+{%- endblock %}
diff --git a/templates/boot/generic-qemu-tmpfs.jinja2 b/templates/boot/generic-qemu-tmpfs.jinja2
new file mode 100644
index 0000000..3908c80
--- /dev/null
+++ b/templates/boot/generic-qemu-tmpfs.jinja2
@@ -0,0 +1,26 @@
+{%- extends 'boot/generic-base-boot.jinja2' %}
+{%- set boot_method = "qemu" %}
+{%- block boot %}
+{{ super() }}
+    media: {{ media_type|default("tmpfs") }}
+{%- endblock %}
+{%- block main %}
+{{ super() }}
+context:
+    no_kvm: false
+    arch: {{ qemu_arch }}
+    extra_options: ["{{ qemu_args }}"]
+{% endblock %}
+{%- block deploy -%}
+{{ super() }}
+    images:
+        kernel:
+           image_arg: '-kernel {kernel} -append "{{ qemu_cmdline }}"'
+           url: {{ kernel_url }}
+{%- if initrd_url and rootfs_type == 'ramdisk' %}
+        ramdisk:
+           image_arg: '-drive format=raw,file={ramdisk}'
+           url: {{ initrd_url }}
+           compression: {{ initrd_compression }}
+{%- endif %}
+{%- endblock %}
diff --git a/templates/boot/generic-uboot-tftp.jinja2 b/templates/boot/generic-uboot-tftp.jinja2
new file mode 100644
index 0000000..cbdda7a
--- /dev/null
+++ b/templates/boot/generic-uboot-tftp.jinja2
@@ -0,0 +1,51 @@
+{%- extends 'boot/generic-base-boot.jinja2' %}
+{%- set boot_method = "u-boot" %}
+{%- block main %}
+{{ super() }}
+{%- if rootfs_type == 'nbd' %}
+protocols:
+  lava-xnbd:
+    port: auto
+{%- endif %}
+{% endblock %}
+{%- block boot %}
+{{ super() }}
+    type: {{ uboot_type|default("bootm") }}
+    commands: {{ boot_commands|default("ramdisk") }}
+{%- if rootfs_type == 'nbd' %}
+    transfer_overlay:
+      download_command: wget
+      unpack_command: tar -C / -xvpf
+{%- endif %}
+{%- endblock %}
+{%- block deploy -%}
+{{ super() }}
+    kernel:
+      url: {{ kernel_url }}
+{%- if rootfs_type == 'nfs' %}
+    nfsrootfs:
+      url: {{ nfsrootfs_url }}
+      compression: {{ nfsroot_compression }}
+{%- elif rootfs_type == 'nbd' %}
+    initrd:
+      url: {{ nbdinitrd_url }}
+      allow_modify: false
+    nbdroot:
+      url: {{ nbdroot_url }}
+      compression: {{ nbdroot_compression }}
+{%- endif %}
+{%- if initrd_url and rootfs_type != 'nbd' %}
+    ramdisk:
+      url: {{ initrd_url }}
+      compression: {{ initrd_compression }}
+{%- endif %}
+{%- if modules_url %}
+    modules:
+      url: {{ modules_url }}
+      compression: {{ modules_compression }}
+{%- endif %}
+{%- if dtb_url %}
+    dtb:
+      url: {{ dtb_url }}
+{%- endif %}
+{%- endblock %}
diff --git a/templates/machines/dra7xx-evm.jinja2 b/templates/machines/dra7xx-evm.jinja2
new file mode 100644
index 0000000..3e16590
--- /dev/null
+++ b/templates/machines/dra7xx-evm.jinja2
@@ -0,0 +1,6 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "ti-vayu-uboot" %}
+{%- set dtb = "zImage-dra7-evm-lcd-lg.dtb" %}
+{%- set kernel_image = "zImage" %}
+{%- set uboot_type = "bootz" %}
+{%- set nbdroot = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %}
diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2
new file mode 100644
index 0000000..e9bf6a3
--- /dev/null
+++ b/templates/machines/m3ulcb.jinja2
@@ -0,0 +1,6 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "r8a7796-m3ulcb" %}
+{%- set kernel_image = "Image" %}
+{%- set dtb = "Image-r8a7796-m3ulcb.dtb" %}
+{%- set uboot_type = "booti" %}
+{%- set dl_dir = "m3ulcb-nogfx" %}
diff --git a/templates/machines/porter.jinja2 b/templates/machines/porter.jinja2
new file mode 100644
index 0000000..36dd96a
--- /dev/null
+++ b/templates/machines/porter.jinja2
@@ -0,0 +1,5 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "renesas-porter-uboot" %}
+{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %}
+{%- set dtb = "uImage-r8a7791-porter.dtb" %}
+{%- set dl_dir = "porter-nogfx" %}
diff --git a/templates/machines/qemux86-64.jinja2 b/templates/machines/qemux86-64.jinja2
new file mode 100644
index 0000000..d76106c
--- /dev/null
+++ b/templates/machines/qemux86-64.jinja2
@@ -0,0 +1,8 @@
+{%- extends 'boot/generic-qemu-tmpfs.jinja2' %}
+{%- set device_type = "qemu" %}
+{%- set kernel_image = "bzImage" %}
+{%- set qemu_cmdline = "console=ttyS0,115200 root=/dev/hda debug verbose" %}
+{%- set qemu_args = "-cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt -m 1048 -soundhw hda" %}
+{%- set qemu_arch = "x86_64" %}
+{%- set deploy_to = "tmpfs" %}
+{%- set initrd = "agl-demo-platform-" + yocto_machine + ".ext4.xz" %}
diff --git a/templates/machines/raspberrypi3.jinja2 b/templates/machines/raspberrypi3.jinja2
new file mode 100644
index 0000000..ed36889
--- /dev/null
+++ b/templates/machines/raspberrypi3.jinja2
@@ -0,0 +1,4 @@
+{%- extends 'boot/generic-uboot-tftp.jinja2' %}
+{%- set device_type = "bcm2837-rpi-3-b" %}
+{%- set dtb = "uImage-bcm2710-rpi-3-b.dtb" %}
+{%- set nbdinitrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %}
diff --git a/templates/tests/smoke.jinja2 b/templates/tests/smoke.jinja2
new file mode 100644
index 0000000..c3e62dc
--- /dev/null
+++ b/templates/tests/smoke.jinja2
@@ -0,0 +1,12 @@
+
+- test:
+    definitions:
+    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+      from: git
+      path: test-suites/short-smoke/busybox.yaml
+      name: busybox
+    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+      from: git
+      path: test-suites/short-smoke/smoke-tests-basic.yaml
+      name: smoke-tests-basic
+
diff --git a/templates/tests/test1.jinja2 b/templates/tests/test1.jinja2
new file mode 100644
index 0000000..8b04691
--- /dev/null
+++ b/templates/tests/test1.jinja2
@@ -0,0 +1,21 @@
+
+- test:
+    timeout:
+      minutes: 2
+    definitions:
+    - repository:
+        metadata:
+          format: Lava-Test Test Definition 1.0
+          name: test1
+          description: "test1"
+          os:
+          - debian
+          scope:
+          - functional
+        run:
+          steps:
+          - lava-test-set start test1
+          - lava-test-case always-pass --shell true
+      from: inline
+      name: dummytest1
+      path: inline/test1.yaml
diff --git a/templates/tests/test2.jinja2 b/templates/tests/test2.jinja2
new file mode 100644
index 0000000..4dd4a14
--- /dev/null
+++ b/templates/tests/test2.jinja2
@@ -0,0 +1,22 @@
+
+- test:
+    timeout:
+      minutes: 2
+    definitions:
+    - repository:
+        metadata:
+          format: Lava-Test Test Definition 1.0
+          name: test2
+          description: "test2"
+          os:
+          - debian
+          scope:
+          - functional
+        run:
+          steps:
+          - lava-test-set start test2
+          - lava-test-case always-fail --shell false
+      from: inline
+      name: dummytest2
+      path: inline/test2.yaml
+
diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
new file mode 100644
index 0000000..749d4bf
--- /dev/null
+++ b/utils/agljobtemplate.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import jinja2
+
+
+def get_extension(path):
+    return path.split('.')[-1]
+
+
+class Agljobtemplate(object):
+
+    DEFAULT_PATH = "templates"
+    MACHINES_DIR = "machines"
+    TESTS_DIR = "tests"
+    RFS_TYPE = ['nfs', 'nbd', 'ramdisk']
+
+    def __init__(self, path=DEFAULT_PATH):
+        self._template_path = os.path.normpath(path)
+        if not (os.path.isdir(self._template_path) and os.access(self._template_path, os.F_OK)):
+            raise OSError, "Cannot access {}".format(self._template_path)
+
+        if self.machines is None:
+            raise RuntimeError, "No machine directory found at {}".format(self._template_path)
+
+    def __list_jinjas(self, directory):
+        d = os.path.join(self._template_path, directory)
+        return [os.path.splitext(os.path.basename(f))[0] for f in os.listdir(d) if f.endswith('.jinja2')]
+
+    @property
+    def machines(self):
+        """ List the availables machines
+        """
+        return self.__list_jinjas(self.MACHINES_DIR)
+
+    @property
+    def tests(self):
+        """ List the availables tests
+        """
+        return self.__list_jinjas(self.TESTS_DIR)
+
+    @property
+    def rfs_types(self):
+        return self.RFS_TYPE
+
+    def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type="nbd"):
+        test_templates = []
+
+        if machine not in self.machines:
+            raise RuntimeError, "{} is not a available machine".format(machine)
+
+        for t in tests:
+            if t in self.tests:
+                test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2'))
+            else:
+                raise RuntimeError, "{} is not an available test".format(t)
+
+        # Populate jinja substitution dict
+        job = {}
+        job['name'] = job_name
+        job['yocto_machine'] = machine
+        job['priority'] = priority
+        job['urlbase'] = url
+        job['test_templates'] = test_templates
+        job['rootfs_type'] = rfs_type
+
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_path))
+        env.filters['get_extension'] = get_extension
+        template = env.get_template(os.path.join(self.MACHINES_DIR, machine + ".jinja2"))
+
+        return template.render(job)
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
new file mode 100755
index 0000000..1c15bb6
--- /dev/null
+++ b/utils/create-jobs.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import agljobtemplate
+import argparse
+import urlparse
+
+
+def parse_cmdline(machines, tests, rfs_types):
+    parser = argparse.ArgumentParser(description="AGL create job",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0')
+    parser.add_argument('machine',  action='store', choices=machines,
+                        help="machine to generate the job for")
+    parser.add_argument('--priority', '-p', action='store', dest='priority',
+                        help="job priority",
+                        default='medium')
+    parser.add_argument('--urlbase', '-u', action='store', dest='urlbase',
+                        help="url fetch base",
+                        default='https://download.automotivelinux.org/AGL/upload/ci/')
+    parser.add_argument('--boot', action='store', dest='rfs_type', nargs=1, required=True,
+                        choices=rfs_types, help='select boot type')
+    parser.add_argument('--test', dest='tests',  action='store', choices=tests + ['all'],
+                        help="add these test to the job", nargs='*', default=[])
+    parser.add_argument('-o', '--output', dest='job_file',  action='store',
+                        help="destination file")
+    parser.add_argument('-n', '--name', dest='job_name',  action='store',
+                        help="job name", default='AGL-short-smoke-wip')
+    parser.add_argument('-j', '--jobid', dest='job_id',  action='store',
+                        help='job id for link creation: URLBASE/JOB_ID', required=True)
+    parser.add_argument('-i', '--jobidx', dest='job_index',  action='store',
+                        help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1')
+
+    return parser.parse_args()
+
+
+def main():
+    ajt = agljobtemplate.Agljobtemplate('templates')
+    args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types)
+
+    if args.tests is not None and 'all' in args.tests:
+        args.tests = ajt.tests
+
+    if args.job_id is not None:
+        args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/')
+        args.job_name += ' - {}'.format(args.job_id)
+
+        if args.job_index is not None:
+            args.urlbase = urlparse.urljoin(args.urlbase, args.job_index)
+            args.job_name += ' - {}'.format(args.job_index)
+
+    job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority,
+                         rfs_type=args.rfs_type[0], job_name=args.job_name)
+
+    if args.job_file is None:
+        print job
+    else:
+        try:
+            with open(args.job_file, 'w') as j:
+                j.write(job)
+        except IOError as e:
+            print "{}: {}".format(e.strerror, args.job_file)
+            exit(e.errno)
+        else:
+            print "Job written to: {}".format(args.job_file)
+
+if __name__ == '__main__':
+    main()