New machine, url, branch, version arguments

- Machine is now a mandatory argument: --machine

- Change in the URL scheme usage:
Now the argument is --url, it should be used with --branch & --version
(or --changeid and --patchset which are aliases).
The default url is the latest AGL release (dab - 4.0.2)

- Build version is derived from the URL, branch and version if not set
through command line.

Example usage:
./utils/create-jobs.py --machine m3ulcb
./utils/create-jobs.py --machine qemux86-64
./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine m3ulcb
./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine qemux86-64
./utils/create-jobs.py --url daily --branch master --version latest --machine m3ulcb
./utils/create-jobs.py --url daily --branch master --version latest --machine raspberrypi3
./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine raspberrypi3
./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine m3ulcb
./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --machine raspberrypi3

Update the README.md following changes on the url scheme and --machine
argument.

Bug-AGL: SPEC-975

Change-Id: Idd0d539627d56e8d28e1e4990570a929b3fa0f5d
Signed-off-by: lollivier <lollivier@baylibre.com>

2 files changed, 60 insertions, 20 deletions

diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
index 528a3b9..b4d44af 100644
--- a/utils/agljobtemplate.py
+++ b/utils/agljobtemplate.py
@@ -4,11 +4,21 @@
 import os
 import jinja2
 import ConfigParser
+import urlparse
 
 
 def get_extension(path):
     return path.split('.')[-1]
 
+def parse_url_file(template_path, url_file, url_type):
+    url_file_path = template_path + '/URLs/' + url_file
+    try:
+        with open(url_file_path):
+            cfg = ConfigParser.ConfigParser()
+            cfg.read(url_file_path)
+            return cfg.get(url_type, 'urlbase'), cfg.get('infra', 'style')
+    except IOError:
+        raise IOError, "Unable to read from file {}".format(url_file_path)
 
 def parse_callback_file(template_path, callback_file, job):
     callback_file_path = template_path + '/callback/' + callback_file + '.cfg'
@@ -59,7 +69,8 @@ class Agljobtemplate(object):
     def rfs_types(self):
         return self.RFS_TYPE
 
-    def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
+    def render_job(self, url, machine, url_branch=None, url_version=None,
+                   job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
                    kci_callback=None, rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None,
                    build_version=None):
         test_templates = []
@@ -78,9 +89,32 @@ class Agljobtemplate(object):
         job['name'] = job_name
         job['yocto_machine'] = machine
         job['priority'] = priority
+
+        if url[:4] != 'http':
+            # If not a full url, read from config file
+            url_base, infra = parse_url_file(self._template_path, 'default.cfg', url)
+            if infra == 'AGL':
+                url_fragment = ''
+                if (url != 'default'):
+                    url_fragment += url_branch + '/' + url_version + '/'
+
+                if (machine == 'm3ulcb') or (machine == 'porter'):
+                    url_fragment += machine + '-nogfx'
+                else:
+                    url_fragment += machine
+
+                if (url != 'ci'):
+                    url_fragment += '/deploy/images/' + machine
+
+                url = urlparse.urljoin(url_base, url_fragment)
+
         job['urlbase'] = url
+
         job['test_templates'] = test_templates
 
+        if (not build_version) and (url_branch) and (url_version):
+            build_version = 'AGL-' + url + '-' + url_branch + '-' + url_version
+
         if build_version is not None:
             job['kernel_version'] = build_version
 
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
index 347c62b..67afe4c 100755
--- a/utils/create-jobs.py
+++ b/utils/create-jobs.py
@@ -3,21 +3,25 @@
 
 import agljobtemplate
 import argparse
-import urlparse
 import os
 
 def parse_cmdline(machines, tests, rfs_types):
     parser = argparse.ArgumentParser(description="AGL create job",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0')
-    parser.add_argument('machine',  action='store', choices=machines,
-                        help="machine to generate the job for")
+    parser.add_argument('-v', action='version', version='%(prog)s 1.0')
+    parser.add_argument('--machine',  action='store', choices=machines,
+                        help="machine to generate the job for",
+                        required=True)
     parser.add_argument('--priority', '-p', action='store', dest='priority',
                         help="job priority",
                         default='medium')
-    parser.add_argument('--urlbase', '-u', action='store', dest='urlbase',
+    parser.add_argument('--url', '-u', action='store', dest='url',
                         help="url fetch base",
-                        default='https://download.automotivelinux.org/AGL/upload/ci/')
+                        default='release')
+    parser.add_argument('--branch', '--changeid', dest='url_branch',  action='store',
+                        help='The branch (or changeid) to generate the job for.')
+    parser.add_argument('--version', '--patchset', dest='url_version',  action='store',
+                        help='The version (or patchset) to generate the job for.')
     parser.add_argument('--boot', action='store', dest='rfs_type',
                         choices=rfs_types, help='select boot type')
     parser.add_argument('--callback', action='store', dest='callback',
@@ -28,10 +32,6 @@ def parse_cmdline(machines, tests, rfs_types):
                         help="destination file")
     parser.add_argument('-n', '--name', dest='job_name',  action='store',
                         help="job name", default='AGL-short-smoke-wip')
-    parser.add_argument('-j', '--jobid', dest='job_id',  action='store',
-                        help='job id for link creation: URLBASE/JOB_ID')
-    parser.add_argument('-i', '--jobidx', dest='job_index',  action='store',
-                        help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1')
     parser.add_argument('--rootfs-img', dest='rootfs_img',  action='store',
                         help="The name of the root file system image (such as agl-demo-platform-raspberrypi3.ext4.xz)")
     parser.add_argument('--kernel-img', dest='kernel_img',  action='store',
@@ -45,6 +45,19 @@ def parse_cmdline(machines, tests, rfs_types):
 
     args = parser.parse_args()
 
+    if (args.url == 'release'):
+        if (args.url_branch is None) and (args.url_version is None):
+            args.url = 'default'
+        elif (args.url_branch is not None) != (args.url_version is not None):
+            parser.error("Both arguments: '--branch' and '--version' needs to be set")
+    elif (args.url == 'daily'):
+        if (not args.url_branch) or (not args.url_version):
+            parser.error("The argument '--url daily' requires '--branch' and '--version' to be set")
+    elif (args.url == 'ci'):
+        if (not args.url_branch) or (not args.url_version):
+            parser.error("The argument '--url ci' requires '--changeid' and '--patchset' to be set. "
+                         + "For more information on how to use patchset and changeid use --help.")
+
     return args
 
 
@@ -57,15 +70,8 @@ def main():
     if args.tests is not None and 'all' in args.tests:
         args.tests = ajt.tests
 
-    if args.job_id is not None:
-        args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/')
-        args.job_name += ' - {}'.format(args.job_id)
-
-        if args.job_index is not None:
-            args.urlbase = urlparse.urljoin(args.urlbase, args.job_index)
-            args.job_name += ' - {}'.format(args.job_index)
-
-    job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority,
+    job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version,
+                         machine=args.machine, tests=args.tests, priority=args.priority,
                          rfs_type=args.rfs_type, job_name=args.job_name, kci_callback=args.callback,
                          rfs_image=args.rootfs_img,
                          kernel_image=args.kernel_img,