diff options
| -rw-r--r-- | README.md | 58 | ||||
| -rw-r--r-- | templates/URLs/default.cfg | 11 | ||||
| -rw-r--r-- | templates/base/agl-base-defaults.jinja2 | 3 | ||||
| -rw-r--r-- | templates/machines/m3ulcb.jinja2 | 1 | ||||
| -rw-r--r-- | templates/machines/porter.jinja2 | 1 | ||||
| -rw-r--r-- | utils/agljobtemplate.py | 36 | ||||
| -rwxr-xr-x | utils/create-jobs.py | 44 |
7 files changed, 111 insertions, 43 deletions
@@ -9,6 +9,9 @@ for submitting jobs. ## Prerequisites - Python >= 2.7.1 +- jinja >= 2.9 + - You can check if a version is/was installed with pip using: ```pip show jinja2``` + - If the version is too old, you can update it using: ```sudo pip install --upgrade jinja2``` ## Usage instructions The tool for generating job is located in the ./utils folder, it is named "create-jobs.py". @@ -18,26 +21,43 @@ The tool for generating job is located in the ./utils folder, it is named "creat Command line tool to generate AGL jobs for LAVA. ##### Required arguments: -- Machine name +- ```./utils/create-jobs.py --machine machine-name``` + - Available machine names: ```{dra7xx-evm,qemux86-64,porter,m3ulcb,raspberrypi3}``` + - For an up to date list of machine names run: ```./utils/create-jobs.py --help``` ##### Artifacts fetching from URL: -The tool will create an URL to fetch the build artifacts as follows: "URL_BASE/MACHINE_NAME". - -Optionnal extra parameters can be used to extend the URL_BASE: `--jobid` and `--jobidx`. -The fetching URL will then be constructed like this: "URL_BASE/JOB_ID/JOB_INDEX/MACHINE_NAME" - - -The default URL_BASE is the AGL CI build repo.\ -The job id and index parameters sould be passed to create a valid fetching URL from this repo.\ -If using another URL these parameters can be omitted. - -##### Example: -From default URL (https://download.automotivelinux.org/AGL/upload/ci/):\ -`$ ./utils/create-jobs.py raspberrypi3 --jobid 10763 --jobidx 3`\ -From other URLs:\ -`$ ./utils/create-jobs.py raspberrypi3 --urlbase http://www.baylibre.com/pub/agl/ci/`\ -`$ ./utils/create-jobs.py raspberrypi3 --urlbase https://download.automotivelinux.org/AGL/snapshots/master/latest/raspberrypi3/deploy/images/`\ -`$ ./utils/create-jobs.py raspberrypi3 --urlbase https://download.automotivelinux.org/AGL/release/dab/4.0.0/raspberrypi3/deploy/images/` - +Amongst other things, this tool is used to generate URLs for fetching build artifacts from specific locations.\ +The default location is: https://download.automotivelinux.org/AGL/\ +The default build is dab version 4.0.2 + +The user can override these defaults using the command line: +- ```--url <release, daily, ci or http://my-custom-url....>``` + - Available url options and their corresponding URL: + - release: https://download.automotivelinux.org/AGL/release/ + - daily: https://download.automotivelinux.org/AGL/snapshots/ + - ci: https://download.automotivelinux.org/AGL/upload/ci/ + +If using the url argument the user has to specify other arguments depending on the url: +- release: ```--branch and --version``` +- daily: ```--branch and --version``` +- ci: ```--changeid and --patchset``` + +If using a custom url these argument are not used and should not be set. The tool will suppose that the custom url points directly to build artifacts for the target machine. + +_Examples:_ +``` +./utils/create-jobs.py --machine m3ulcb +./utils/create-jobs.py --machine qemux86-64 +./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine m3ulcb +./utils/create-jobs.py --url release --branch eel --version 4.99.1 --machine qemux86-64 +./utils/create-jobs.py --url daily --branch master --version latest --machine m3ulcb +./utils/create-jobs.py --url daily --branch master --version latest --machine raspberrypi3 +./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine raspberrypi3 +./utils/create-jobs.py --url ci --changeid 11533 --patchset 2 --machine m3ulcb +./utils/create-jobs.py --url http://baylibre.com/pub/agl/ci/raspberrypi3 --machine raspberrypi3 +``` The full list of arguments with default values is available using the helper:\ `$ ./utils/create-jobs.py --help` + +##### Add tests to a job +To add tests to a job description please refer to the specific documentation: [releng-scripts-folder]/doc/test-documentation.md diff --git a/templates/URLs/default.cfg b/templates/URLs/default.cfg new file mode 100644 index 0000000..4c2adec --- /dev/null +++ b/templates/URLs/default.cfg @@ -0,0 +1,11 @@ +[infra] +style = AGL +[default] +urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/ +[release] +urlbase = https://download.automotivelinux.org/AGL/release/ +[weekly] +[daily] +urlbase = https://download.automotivelinux.org/AGL/snapshots/ +[ci] +urlbase = https://download.automotivelinux.org/AGL/upload/ci/ diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2 index 462fb86..d432b63 100644 --- a/templates/base/agl-base-defaults.jinja2 +++ b/templates/base/agl-base-defaults.jinja2 @@ -1,12 +1,11 @@ {%- macro baseurl(object) -%} -{{ urlbase }}/{{ dl_dir }}/{{ object }} +{{ urlbase }}/{{ object }} {%- endmacro %} {%- set action_timeout = action_timeout|default(15) %} {%- set boot_timeout = boot_timeout|default(10) %} {%- set connection_timeout = connection_timeout|default(5) %} {%- set deploy_timeout = deploy_timeout|default(15) %} {%- set device_type = device_type|default(yocto_machine+"-uboot") %} -{%- set dl_dir = dl_dir|default(yocto_machine) %} {%- set image_type = image_type|default("AGL") %} {%- set job_timeout = job_timeout|default(30) %} {%- set kernel_defconfig = kernel_defconfig|default("defconfig+CONFIG_AGL=y") %} diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2 index 16a3e76..41b6677 100644 --- a/templates/machines/m3ulcb.jinja2 +++ b/templates/machines/m3ulcb.jinja2 @@ -2,7 +2,6 @@ {%- set device_arch = "arm64" %} {%- set device_mach = "renesas" %} {%- set device_type = "r8a7796-m3ulcb" %} -{%- set dl_dir = "m3ulcb-nogfx" %} {%- set dtb = dtb|default("Image-r8a7796-m3ulcb.dtb") %} {%- set kernel_image = kernel_image|default("Image") %} {%- set rootfs_type = rootfs_type|default("nbd") %} diff --git a/templates/machines/porter.jinja2 b/templates/machines/porter.jinja2 index 1ad6e7b..b455922 100644 --- a/templates/machines/porter.jinja2 +++ b/templates/machines/porter.jinja2 @@ -1,6 +1,5 @@ {%- extends 'boot/generic-uboot-tftp.jinja2' %} {%- set device_type = "renesas-porter-uboot" %} -{%- set dl_dir = "porter-nogfx" %} {%- set dtb = dtb|default("uImage-r8a7791-porter.dtb") %} {%- set initrd = initrd|default("initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot") %} {%- set rootfs_type = rootfs_type|default("nbd") %} diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py index 528a3b9..b4d44af 100644 --- a/utils/agljobtemplate.py +++ b/utils/agljobtemplate.py @@ -4,11 +4,21 @@ import os import jinja2 import ConfigParser +import urlparse def get_extension(path): return path.split('.')[-1] +def parse_url_file(template_path, url_file, url_type): + url_file_path = template_path + '/URLs/' + url_file + try: + with open(url_file_path): + cfg = ConfigParser.ConfigParser() + cfg.read(url_file_path) + return cfg.get(url_type, 'urlbase'), cfg.get('infra', 'style') + except IOError: + raise IOError, "Unable to read from file {}".format(url_file_path) def parse_callback_file(template_path, callback_file, job): callback_file_path = template_path + '/callback/' + callback_file + '.cfg' @@ -59,7 +69,8 @@ class Agljobtemplate(object): def rfs_types(self): return self.RFS_TYPE - def render_job(self, url, machine, job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None, + def render_job(self, url, machine, url_branch=None, url_version=None, + job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None, kci_callback=None, rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None, build_version=None): test_templates = [] @@ -78,9 +89,32 @@ class Agljobtemplate(object): job['name'] = job_name job['yocto_machine'] = machine job['priority'] = priority + + if url[:4] != 'http': + # If not a full url, read from config file + url_base, infra = parse_url_file(self._template_path, 'default.cfg', url) + if infra == 'AGL': + url_fragment = '' + if (url != 'default'): + url_fragment += url_branch + '/' + url_version + '/' + + if (machine == 'm3ulcb') or (machine == 'porter'): + url_fragment += machine + '-nogfx' + else: + url_fragment += machine + + if (url != 'ci'): + url_fragment += '/deploy/images/' + machine + + url = urlparse.urljoin(url_base, url_fragment) + job['urlbase'] = url + job['test_templates'] = test_templates + if (not build_version) and (url_branch) and (url_version): + build_version = 'AGL-' + url + '-' + url_branch + '-' + url_version + if build_version is not None: job['kernel_version'] = build_version diff --git a/utils/create-jobs.py b/utils/create-jobs.py index 347c62b..67afe4c 100755 --- a/utils/create-jobs.py +++ b/utils/create-jobs.py @@ -3,21 +3,25 @@ import agljobtemplate import argparse -import urlparse import os def parse_cmdline(machines, tests, rfs_types): parser = argparse.ArgumentParser(description="AGL create job", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0') - parser.add_argument('machine', action='store', choices=machines, - help="machine to generate the job for") + parser.add_argument('-v', action='version', version='%(prog)s 1.0') + parser.add_argument('--machine', action='store', choices=machines, + help="machine to generate the job for", + required=True) parser.add_argument('--priority', '-p', action='store', dest='priority', help="job priority", default='medium') - parser.add_argument('--urlbase', '-u', action='store', dest='urlbase', + parser.add_argument('--url', '-u', action='store', dest='url', help="url fetch base", - default='https://download.automotivelinux.org/AGL/upload/ci/') + default='release') + parser.add_argument('--branch', '--changeid', dest='url_branch', action='store', + help='The branch (or changeid) to generate the job for.') + parser.add_argument('--version', '--patchset', dest='url_version', action='store', + help='The version (or patchset) to generate the job for.') parser.add_argument('--boot', action='store', dest='rfs_type', choices=rfs_types, help='select boot type') parser.add_argument('--callback', action='store', dest='callback', @@ -28,10 +32,6 @@ def parse_cmdline(machines, tests, rfs_types): help="destination file") parser.add_argument('-n', '--name', dest='job_name', action='store', help="job name", default='AGL-short-smoke-wip') - parser.add_argument('-j', '--jobid', dest='job_id', action='store', - help='job id for link creation: URLBASE/JOB_ID') - parser.add_argument('-i', '--jobidx', dest='job_index', action='store', - help='job index for link creation: URLBASE/JOB_ID/JOB_INDEX', default='1') parser.add_argument('--rootfs-img', dest='rootfs_img', action='store', help="The name of the root file system image (such as agl-demo-platform-raspberrypi3.ext4.xz)") parser.add_argument('--kernel-img', dest='kernel_img', action='store', @@ -45,6 +45,19 @@ def parse_cmdline(machines, tests, rfs_types): args = parser.parse_args() + if (args.url == 'release'): + if (args.url_branch is None) and (args.url_version is None): + args.url = 'default' + elif (args.url_branch is not None) != (args.url_version is not None): + parser.error("Both arguments: '--branch' and '--version' needs to be set") + elif (args.url == 'daily'): + if (not args.url_branch) or (not args.url_version): + parser.error("The argument '--url daily' requires '--branch' and '--version' to be set") + elif (args.url == 'ci'): + if (not args.url_branch) or (not args.url_version): + parser.error("The argument '--url ci' requires '--changeid' and '--patchset' to be set. " + + "For more information on how to use patchset and changeid use --help.") + return args @@ -57,15 +70,8 @@ def main(): if args.tests is not None and 'all' in args.tests: args.tests = ajt.tests - if args.job_id is not None: - args.urlbase = urlparse.urljoin(args.urlbase, args.job_id + '/') - args.job_name += ' - {}'.format(args.job_id) - - if args.job_index is not None: - args.urlbase = urlparse.urljoin(args.urlbase, args.job_index) - args.job_name += ' - {}'.format(args.job_index) - - job = ajt.render_job(args.urlbase, args.machine, tests=args.tests, priority=args.priority, + job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version, + machine=args.machine, tests=args.tests, priority=args.priority, rfs_type=args.rfs_type, job_name=args.job_name, kci_callback=args.callback, rfs_image=args.rootfs_img, kernel_image=args.kernel_img, |