aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorlollivier <lollivier@baylibre.com>2017-10-30 16:08:23 +0100
committerlollivier <lollivier@baylibre.com>2017-11-06 10:27:51 +0100
commit223394c59690707a528fa512a61db8d95f117660 (patch)
treef973756ed7ea9415a5c6fb6cbbb2a01f1290242b
parent848b44f4497613a0fcbae6a02380a9d6670d804f (diff)
New machine, url, branch, version arguments
- Machine is now a mandatory argument: --machine - Change in the URL scheme usage: Now the argument is --url, it should be used with --branch & --version (or --changeid and --patchset which are aliases). The default url is the latest AGL release (dab - 4.0.2) - Build version is derived from the URL, branch and version if not set through command line. Example usage: ./utils/create-jobs.py --machine m3ulcb ./utils/create-jobs.py --machine qemux86-64 ./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine m3ulcb ./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine qemux86-64 ./utils/create-jobs.py --url daily --branch master --version latest --machine m3ulcb ./utils/create-jobs.py --url daily --branch master --version latest --machine raspberrypi3 ./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine raspberrypi3 ./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine m3ulcb ./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --machine raspberrypi3 Update the README.md following changes on the url scheme and --machine argument. Bug-AGL: SPEC-975 Change-Id: Idd0d539627d56e8d28e1e4990570a929b3fa0f5d Signed-off-by: lollivier <lollivier@baylibre.com>
-rw-r--r--README.md58
-rw-r--r--templates/URLs/default.cfg11
-rw-r--r--templates/base/agl-base-defaults.jinja23
-rw-r--r--templates/machines/m3ulcb.jinja21
-rw-r--r--templates/machines/porter.jinja21
-rw-r--r--utils/agljobtemplate.py36
-rwxr-xr-xutils/create-jobs.py44
7 files changed, 111 insertions, 43 deletions
diff --git a/README.md b/README.md
index e23e40c..c95696d 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,9 @@ for submitting jobs.
## Prerequisites
- Python >= 2.7.1
+- jinja >= 2.9
+ - You can check if a version is/was installed with pip using: ```pip show jinja2```
+ - If the version is too old, you can update it using: ```sudo pip install --upgrade jinja2```
## Usage instructions
The tool for generating job is located in the ./utils folder, it is named "create-jobs.py".
@@ -18,26 +21,43 @@ The tool for generating job is located in the ./utils folder, it is named "creat
Command line tool to generate AGL jobs for LAVA.
##### Required arguments:
-- Machine name
+- ```./utils/create-jobs.py --machine machine-name```
+ - Available machine names: ```{dra7xx-evm,qemux86-64,porter,m3ulcb,raspberrypi3}```
+ - For an up to date list of machine names run: ```./utils/create-jobs.py --help```
##### Artifacts fetching from URL:
-The tool will create an URL to fetch the build artifacts as follows: "URL_BASE/MACHINE_NAME".
-
-Optionnal extra parameters can be used to extend the URL_BASE: `--jobid` and `--jobidx`.
-The fetching URL will then be constructed like this: "URL_BASE/JOB_ID/JOB_INDEX/MACHINE_NAME"
-
-
-The default URL_BASE is the AGL CI build repo.\
-The job id and index parameters sould be passed to create a valid fetching URL from this repo.\
-If using another URL these parameters can be omitted.
-
-##### Example:
-From default URL (https://download.automotivelinux.org/AGL/upload/ci/):\
-`$ ./utils/create-jobs.py raspberrypi3 --jobid 10763 --jobidx 3`\
-From other URLs:\
-`$ ./utils/create-jobs.py raspberrypi3 --urlbase http://www.baylibre.com/pub/agl/ci/`\
-`$ ./utils/create-jobs.py raspberrypi3 --urlbase https://download.automotivelinux.org/AGL/snapshots/master/latest/raspberrypi3/deploy/images/`\
-`$ ./utils/create-jobs.py raspberrypi3 --urlbase https://download.automotivelinux.org/AGL/release/dab/4.0.0/raspberrypi3/deploy/images/`
-
+Amongst other things, this tool is used to generate URLs for fetching build artifacts from specific locations.\
+The default location is: https://download.automotivelinux.org/AGL/\
+The default build is dab version 4.0.2
+
+The user can override these defaults using the command line:
+- ```--url <release, daily, ci or http://my-custom-url....>```
+ - Available url options and their corresponding URL:
+ - release: https://download.automotivelinux.org/AGL/release/
+ - daily: https://download.automotivelinux.org/AGL/snapshots/
+ - ci: https://download.automotivelinux.org/AGL/upload/ci/
+
+If using the url argument the user has to specify other arguments depending on the url:
+- release: ```--branch and --version```
+- daily: ```--branch and --version```
+- ci: ```--changeid and --patchset```
+
+If using a custom url these argument are not used and should not be set. The tool will suppose that the custom url points directly to build artifacts for the target machine.
+
+_Examples:_
+```
+./utils/create-jobs.py --machine m3ulcb
+./utils/create-jobs.py --machine qemux86-64
+./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine m3ulcb
+./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine qemux86-64
+./utils/create-jobs.py --url daily --branch master --version latest --machine m3ulcb
+./utils/create-jobs.py --url daily --branch master --version latest --machine raspberrypi3
+./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine raspberrypi3
+./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine m3ulcb
+./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --machine raspberrypi3
+```
The full list of arguments with default values is available using the helper:\
`$ ./utils/create-jobs.py --help`
+
+##### Add tests to a job
+To add tests to a job description please refer to the specific documentation: [releng-scripts-folder]/doc/test-documentation.md
diff --git a/templates/URLs/default.cfg b/templates/URLs/default.cfg
new file mode 100644
index 0000000..4c2adec
--- /dev/null
+++ b/templates/URLs/default.cfg
@@ -0,0 +1,11 @@
+[infra]
+style = AGL
+[default]
+urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/
+[release]
+urlbase = https://download.automotivelinux.org/AGL/release/
+[weekly]
+[daily]
+urlbase = https://download.automotivelinux.org/AGL/snapshots/
+[ci]
+urlbase = https://download.automotivelinux.org/AGL/upload/ci/
diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2
index 462fb86..d432b63 100644
--- a/templates/base/agl-base-defaults.jinja2
+++ b/templates/base/agl-base-defaults.jinja2
@@ -1,12 +1,11 @@
{%- macro baseurl(object) -%}
-{{ urlbase }}/{{ dl_dir }}/{{ object }}
+{{ urlbase }}/{{ object }}
{%- endmacro %}
{%- set action_timeout = action_timeout|default(15) %}
{%- set boot_timeout = boot_timeout|default(10) %}
{%- set connection_timeout = connection_timeout|default(5) %}
{%- set deploy_timeout = deploy_timeout|default(15) %}
{%- set device_type = device_type|default(yocto_machine+"-uboot") %}
-{%- set dl_dir = dl_dir|default(yocto_machine) %}
{%- set image_type = image_type|default("AGL") %}
{%- set job_timeout = job_timeout|default(30) %}
{%- set kernel_defconfig = kernel_defconfig|default("defconfig+CONFIG_AGL=y") %}
diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2
index 16a3e76..41b6677 100644
--- a/templates/machines/m3ulcb.jinja2
+++ b/templates/machines/m3ulcb.jinja2
@@ -2,7 +2,6 @@
{%- set device_arch = "arm64" %}
{%- set device_mach = "renesas" %}
{%- set device_type = "r8a7796-m3ulcb" %}
-{%- set dl_dir = "m3ulcb-nogfx" %}
{%- set dtb = dtb|default("Image-r8a7796-m3ulcb.dtb") %}
{%- set kernel_image = kernel_image|default("Image") %}
{%- set rootfs_type = rootfs_type|default("nbd") %}
diff --git a/templates/machines/porter.jinja2 b/templates/machines/porter.jinja2
index 1ad6e7b..b455922 100644
--- a/templates/machines/porter.jinja2
+++ b/templates/machines/porter.jinja2
@@ -1,6 +1,5 @@
{%- extends 'boot/generic-uboot-tftp.jinja2' %}
{%- set device_type = "renesas-porter-uboot" %}
-{%- set dl_dir = "porter-nogfx" %}
{%- set dtb = dtb|default("uImage-r8a7791-porter.dtb") %}
{%- set initrd = initrd|default("initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot") %}
{%- set rootfs_type = rootfs_type|default("nbd") %}
diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
index 528a3b9..b4d44af 100644
--- a/utils/agljobtemplate.py
+++ b/utils/agljobtemplate.py
@@ -4,11 +4,21 @@
import os
import jinja2
import ConfigParser
+import urlparse
def get_extension(path):
return path.split('.')[-1]
+def parse_url_file(template_path, url_file, url_type):
+ url_file_path = template_path + '/URLs/' + url_file
+ try:
+ with open(url_file_path):
+ cfg = ConfigParser.ConfigParser()
+ cfg.read(url_file_path)
+ return cfg.get(url_type, 'urlbase'), cfg.get('infra', 'style')
+ except IOError:
+ raise IOError, "Unable to read from file {}".format(url_file_path)
def parse_callback_file(template_path, callback_file, job):
callback_file_path = template_path + '/callback/' + callback_file + '.cfg'
@@ -59,7 +69,8 @@ class Agljobtemplate(object):
def rfs_types(self):
return self.RFS_TYPE
- def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
+ def render_job(self, url, machine, url_branch=None, url_version=None,
+ job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
kci_callback=None, rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None,
build_version=None):
test_templates = []
@@ -78,9 +89,32 @@ class Agljobtemplate(object):
job['name'] = job_name
job['yocto_machine'] = machine
job['priority'] = priority
+
+ if url[:4] != 'http':
+ # If not a full url, read from config file
+ url_base, infra = parse_url_file(self._template_path, 'default.cfg', url)
+ if infra == 'AGL':
+ url_fragment = ''
+ if (url != 'default'):
+ url_fragment += url_branch + '/' + url_version + '/'
+
+ if (machine == 'm3ulcb') or (machine == 'porter'):
+ url_fragment += machine + '-nogfx'
+ else:
+ url_fragment += machine
+
+ if (url != 'ci'):
+ url_fragment += '/deploy/images/' + machine
+
+ url = urlparse.urljoin(url_base, url_fragment)
+
job['urlbase'] = url
+
job['test_templates'] = test_templates
+ if (not build_version) and (url_branch) and (url_version):
+ build_version = 'AGL-' + url + '-' + url_branch + '-' + url_version
+
if build_version is not None:
job['kernel_version'] = build_version
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
index 347c62b..67afe4c 100755
--- a/utils/create-jobs.py
+++ b/utils/create-jobs.py
@@ -3,21 +3,25 @@
import agljobtemplate
import argparse
-import urlparse
import os
def parse_cmdline(machines, tests, rfs_types):
parser = argparse.ArgumentParser(description="AGL create job",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0')
- parser.add_argument('machine', action='store', choices=machines,
- help="machine to generate the job for")
+ parser.add_argument('-v', action='version', version='%(prog)s 1.0')
+ parser.add_argument('--machine', action='store', choices=machines,
+ help="machine to generate the job for",
+ required=True)
parser.add_argument('--priority', '-p', action='store', dest='priority',
help="job priority",
default='medium')
- parser.add_argument('--urlbase', '-u', action='store', dest='urlbase',
+ parser.add_argument('--url', '-u', action='store', dest='url',
help="url fetch base",
- default='https://download.automotivelinux.org/AGL/upload/ci/')
+ default='release')
+ parser.add_argument('--branch', '--changeid', dest='url_branch', action='store',
+ help='The branch (or changeid) to generate the job for.')
+ parser.add_argument('--version', '--patchset', dest='url_version', action='store',
+ help='The version (or patchset) to generate the job for.')
parser.add_argument('--boot', action='store', dest='rfs_type',
choices=rfs_types, help='select boot type')
parser.add_argument('--callback', action='store', dest='callback',
@@ -28,10 +32,6 @@ def parse_cmdline(machines, tests, rfs_types):
help="destination file")
parser.add_argument('-n', '--name', dest='job_name', action='store',
help="job name", default='AGL-short-smoke-wip')
- parser.add_argument('-j', '--jobid', dest='job_id', action='store',
- help='job id for link creation: URLBASE/JOB_ID')
- parser.add_argument('-i', '--jobidx', dest='job_index', action='store',
- help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1')
parser.add_argument('--rootfs-img', dest='rootfs_img', action='store',
help="The name of the root file system image (such as agl-demo-platform-raspberrypi3.ext4.xz)")
parser.add_argument('--kernel-img', dest='kernel_img', action='store',
@@ -45,6 +45,19 @@ def parse_cmdline(machines, tests, rfs_types):
args = parser.parse_args()
+ if (args.url == 'release'):
+ if (args.url_branch is None) and (args.url_version is None):
+ args.url = 'default'
+ elif (args.url_branch is not None) != (args.url_version is not None):
+ parser.error("Both arguments: '--branch' and '--version' needs to be set")
+ elif (args.url == 'daily'):
+ if (not args.url_branch) or (not args.url_version):
+ parser.error("The argument '--url daily' requires '--branch' and '--version' to be set")
+ elif (args.url == 'ci'):
+ if (not args.url_branch) or (not args.url_version):
+ parser.error("The argument '--url ci' requires '--changeid' and '--patchset' to be set. "
+ + "For more information on how to use patchset and changeid use --help.")
+
return args
@@ -57,15 +70,8 @@ def main():
if args.tests is not None and 'all' in args.tests:
args.tests = ajt.tests
- if args.job_id is not None:
- args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/')
- args.job_name += ' - {}'.format(args.job_id)
-
- if args.job_index is not None:
- args.urlbase = urlparse.urljoin(args.urlbase, args.job_index)
- args.job_name += ' - {}'.format(args.job_index)
-
- job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority,
+ job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version,
+ machine=args.machine, tests=args.tests, priority=args.priority,
rfs_type=args.rfs_type, job_name=args.job_name, kci_callback=args.callback,
rfs_image=args.rootfs_img,
kernel_image=args.kernel_img,