Merge changes I44ae2990,Ied4e9f80,I9ce801a7

* changes:
  templates: fix the rootfs name for m3ulcb and rpi3 daily builds
  utils: define test-plans based on build-type
  utils: change the url / build behavior scheme

7 files changed, 63 insertions, 41 deletions

diff --git a/README.md b/README.md
index 5c0c8fd..dd978c9 100644
--- a/README.md
+++ b/README.md
@@ -86,13 +86,14 @@ _Examples:_
 ```bash
 ./utils/create-jobs.py --machine m3ulcb
 ./utils/create-jobs.py --machine qemux86-64
-./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine m3ulcb
-./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine qemux86-64
-./utils/create-jobs.py --url daily --branch master --version latest --machine m3ulcb
-./utils/create-jobs.py --url daily --branch master --version latest --machine raspberrypi3
-./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine raspberrypi3
-./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine m3ulcb
+./utils/create-jobs.py --build-type release --branch eel --version 4.99.1 --machine m3ulcb
+./utils/create-jobs.py --build-type release --branch eel --version 4.99.1 --machine qemux86-64
+./utils/create-jobs.py --build-type daily --branch master --version latest --machine m3ulcb
+./utils/create-jobs.py --build-type daily --branch master --version latest --machine raspberrypi3
+./utils/create-jobs.py --build-type ci --changeid 13079 --patchset 1 --machine raspberrypi3
+./utils/create-jobs.py --build-type ci --changeid 13079 --patchset 1 --machine m3ulcb
 ./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --machine raspberrypi3
+./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --build-type release --machine raspberrypi3
 ```
 The full list of arguments with default values is available using the helper:\
 `$ ./utils/create-jobs.py --help`
diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2
index d432b63..3f13f08 100644
--- a/templates/base/agl-base-defaults.jinja2
+++ b/templates/base/agl-base-defaults.jinja2
@@ -3,6 +3,7 @@
 {%- endmacro %}
 {%- set action_timeout = action_timeout|default(15) %}
 {%- set boot_timeout = boot_timeout|default(10) %}
+{%- set build_type = build_type|default("default") %}
 {%- set connection_timeout = connection_timeout|default(5) %}
 {%- set deploy_timeout = deploy_timeout|default(15) %}
 {%- set device_type = device_type|default(yocto_machine+"-uboot") %}
diff --git a/templates/URLs/default.cfg b/templates/config/default.cfg
index 4c2adec..ba43270 100644
--- a/templates/URLs/default.cfg
+++ b/templates/config/default.cfg
@@ -2,10 +2,14 @@
 style = AGL
 [default]
 urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/
+test_plan = ["health-test","smoke"]
 [release]
 urlbase = https://download.automotivelinux.org/AGL/release/
+test_plan = ["all"]
 [weekly]
 [daily]
 urlbase = https://download.automotivelinux.org/AGL/snapshots/
+test_plan = ["health-test","smoke", "yocto-ptest"]
 [ci]
 urlbase = https://download.automotivelinux.org/AGL/upload/ci/
+test_plan = ["health-test","smoke"]
diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2
index 41b6677..f337a8d 100644
--- a/templates/machines/m3ulcb.jinja2
+++ b/templates/machines/m3ulcb.jinja2
@@ -6,3 +6,6 @@
 {%- set kernel_image = kernel_image|default("Image") %}
 {%- set rootfs_type = rootfs_type|default("nbd") %}
 {%- set uboot_type = "booti" %}
+{%- if build_type == 'daily' %}
+    {%- set rfs_image = rfs_image|default("agl-image-ivi-crosssdk-m3ulcb.ext4.xz") %}
+{%- endif %}
diff --git a/templates/machines/raspberrypi3.jinja2 b/templates/machines/raspberrypi3.jinja2
index e415c2c..0e743a8 100644
--- a/templates/machines/raspberrypi3.jinja2
+++ b/templates/machines/raspberrypi3.jinja2
@@ -9,4 +9,8 @@
     {%- set initrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %}
 {%- endif %}
 {%- set dtb = dtb|default("uImage-bcm2710-rpi-3-b.dtb") %}
-{%- set rfs_image = rfs_image|default("agl-demo-platform-raspberrypi3.ext4.xz") %}
+{%- if build_type == 'daily' %}
+    {%- set rfs_image = rfs_image|default("agl-demo-platform-crosssdk-raspberrypi3.ext4.xz") %}
+{%- else %}
+    {%- set rfs_image = rfs_image|default("agl-demo-platform-raspberrypi3.ext4.xz") %}
+{%- endif %}
diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
index f41dded..dec7fad 100644
--- a/utils/agljobtemplate.py
+++ b/utils/agljobtemplate.py
@@ -5,19 +5,20 @@ import os
 import jinja2
 import ConfigParser
 import urlparse
+import ast
 
 
 def get_extension(path):
     return path.split('.')[-1]
 
 
-def parse_url_file(template_path, url_file, url_type):
-    url_file_path = template_path + '/URLs/' + url_file
+def parse_cfg_file(template_path, cfg_file, build_type):
+    url_file_path = template_path + '/config/' + cfg_file
     try:
         with open(url_file_path):
             cfg = ConfigParser.ConfigParser()
             cfg.read(url_file_path)
-            return cfg.get(url_type, 'urlbase'), cfg.get('infra', 'style')
+            return cfg.items(build_type), cfg.get('infra', 'style')
     except IOError as err:
         raise err
 
@@ -86,34 +87,34 @@ class Agljobtemplate(object):
                    job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
                    lava_callback=None, kci_callback=None,
                    rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None,
+                   build_type=None,
                    build_version=None):
-        test_templates = []
 
         if machine not in self.machines:
             raise RuntimeError("{} is not a available machine".format(machine))
 
-        for t in tests:
-            if t in self.tests:
-                test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2'))
-            else:
-                raise RuntimeError("{} is not an available test".format(t))
-
         # Populate jinja substitution dict
         job = {}
         job['name'] = job_name
         job['yocto_machine'] = machine
         job['priority'] = priority
+        job['build_type'] = build_type
 
-        if url[:4] != 'http':
-            # If not a full url, read from config file
-            url_base, infra = parse_url_file(self._template_path, 'default.cfg', url)
+        defaults, infra = parse_cfg_file(self._template_path, 'default.cfg', build_type)
+
+        # If the user doesn't specify an URL, use the default one from the build-type
+        if url is None:
             if infra == 'AGL':
+                url_base = ''
+                for section in defaults:
+                    if section[0] == "urlbase":
+                        url_base = section[1]
                 # If not set, create a build_version from other args
                 if (not build_version) and (url_branch) and (url_version):
-                    build_version = 'AGL-' + url + '-' + url_branch + '-' + url_version
+                    build_version = 'AGL-' + build_type + '-' + url_branch + '-' + url_version
 
                 url_fragment = ''
-                if (url != 'default'):
+                if (build_type != 'default'):
                     url_fragment += url_branch + '/' + url_version + '/'
 
                 if (machine == 'm3ulcb') or (machine == 'porter'):
@@ -121,11 +122,27 @@ class Agljobtemplate(object):
                 else:
                     url_fragment += machine
 
-                if (url != 'ci'):
+                if (build_type != 'ci'):
                     url_fragment += '/deploy/images/' + machine
 
                 url = urlparse.urljoin(url_base, url_fragment)
 
+        test_templates = []
+        # If the user doesn't specify tests, use the default ones from the build-type
+        if not tests:
+            if infra == 'AGL':
+                for section in defaults:
+                    if section[0] == "test_plan":
+                        tests = ast.literal_eval(section[1])
+
+        if 'all' in tests:
+            tests = self.tests
+        for t in tests:
+            if t in self.tests:
+                test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2'))
+            else:
+                raise RuntimeError("{} is not an available test".format(t))
+
         job['urlbase'] = url
 
         job['test_templates'] = test_templates
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
index f3415a1..384d935 100755
--- a/utils/create-jobs.py
+++ b/utils/create-jobs.py
@@ -17,12 +17,14 @@ def parse_cmdline(machines, tests, rfs_types):
                         help="job priority",
                         default='medium')
     parser.add_argument('--url', '-u', action='store', dest='url',
-                        help="url fetch base",
-                        default='release')
+                        help="If using a custom URL specify it there. The files will be fetched from this URL.")
+    parser.add_argument('--build-type', dest='build_type', action='store',
+                        help="The type of build, can be one of: {'ci', 'daily', 'weekly', 'release'}.",
+                        default="default")
     parser.add_argument('--branch', '--changeid', dest='url_branch', action='store',
-                        help='The branch (or changeid) to generate the job for.')
+                        help='The build branch (or changeid).')
     parser.add_argument('--version', '--patchset', dest='url_version', action='store',
-                        help='The version (or patchset) to generate the job for.')
+                        help='The build version (or patchset).')
     parser.add_argument('--boot', action='store', dest='rfs_type',
                         choices=rfs_types, help='select boot type')
     parser.add_argument('--callback-from', action='store', dest='callback_from',
@@ -46,24 +48,16 @@ def parse_cmdline(machines, tests, rfs_types):
     parser.add_argument('--modules-img', dest='modules_img', action='store',
                         help="The name of the modules to use (such as modules.tar.xz)")
     parser.add_argument('--build-version', dest='build_version', action='store',
-                        help="the version number of the AGL build.")
+                        help="the version number of the build.")
 
     args = parser.parse_args()
 
     if args.callback_to and not args.callback_from:
         parser.error("When using '--callback-to', '--callback-from' is mandatory.")
 
-    if (args.url == 'release'):
-        if (args.url_branch is None) and (args.url_version is None):
-            args.url = 'default'
-        elif (args.url_branch is not None) != (args.url_version is not None):
-            parser.error("Both arguments: '--branch' and '--version' needs to be set")
-    elif (args.url == 'daily'):
+    if (args.build_type is not "default") and (args.url is None):
         if (not args.url_branch) or (not args.url_version):
-            parser.error("The argument '--url daily' requires '--branch' and '--version' to be set")
-    elif (args.url == 'ci'):
-        if (not args.url_branch) or (not args.url_version):
-            parser.error("The argument '--url ci' requires '--changeid' and '--patchset' to be set.")
+            parser.error("when using '--build-type' either '--url' or '--branch' and '--version' arguments needs to be set.")
 
     return args
 
@@ -74,9 +68,6 @@ def main():
     ajt = agljobtemplate.Agljobtemplate(templates_dir)
     args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types)
 
-    if args.tests is not None and 'all' in args.tests:
-        args.tests = ajt.tests
-
     job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version,
                          machine=args.machine, tests=args.tests, priority=args.priority,
                          rfs_type=args.rfs_type, job_name=args.job_name,
@@ -85,6 +76,7 @@ def main():
                          kernel_image=args.kernel_img,
                          dtb_image=args.dtb_img,
                          modules_image=args.modules_img,
+                         build_type=args.build_type,
                          build_version=args.build_version)
 
     if args.job_file is None: