From 788b31a122c3823d5e756524187975a2ad14cac6 Mon Sep 17 00:00:00 2001 From: Loys Ollivier Date: Wed, 3 Jan 2018 14:53:55 +0100 Subject: utils: change the url / build behavior scheme This is the first patch of the serie to differentiate builds and urls. As of now we were using the --url option to specify a build-type. Add a new option --build-type which specifies which type of build it is. From this build-type if no url is specified, the url can be defaulted by a configuration file. This configuration file is specific per user, e.g. AGL, others... Change-Id: I9ce801a7518b78ee859c6c3bbcad3a89e884e832 Signed-off-by: Loys Ollivier --- utils/agljobtemplate.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'utils/agljobtemplate.py') diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py index f41dded..25efab6 100644 --- a/utils/agljobtemplate.py +++ b/utils/agljobtemplate.py @@ -11,13 +11,18 @@ def get_extension(path): return path.split('.')[-1] -def parse_url_file(template_path, url_file, url_type): +def parse_url_file(template_path, url_file, build_type): url_file_path = template_path + '/URLs/' + url_file try: with open(url_file_path): cfg = ConfigParser.ConfigParser() cfg.read(url_file_path) - return cfg.get(url_type, 'urlbase'), cfg.get('infra', 'style') + config = cfg.items(build_type) + for section in config: + if section[0] == "test_plan": + print section[1] + print build_type + return cfg.get(build_type, 'urlbase'), cfg.get('infra', 'style') except IOError as err: raise err @@ -86,6 +91,7 @@ class Agljobtemplate(object): job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None, lava_callback=None, kci_callback=None, rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None, + build_type=None, build_version=None): test_templates = [] @@ -104,16 +110,16 @@ class Agljobtemplate(object): job['yocto_machine'] = machine job['priority'] = priority - if url[:4] != 'http': - # If not a full url, read from config file - url_base, infra = parse_url_file(self._template_path, 'default.cfg', url) + # If the user doesn't specify an URL, use the default one from the build-type + if url is None: + url_base, infra = parse_url_file(self._template_path, 'default.cfg', build_type) if infra == 'AGL': # If not set, create a build_version from other args if (not build_version) and (url_branch) and (url_version): - build_version = 'AGL-' + url + '-' + url_branch + '-' + url_version + build_version = 'AGL-' + build_type + '-' + url_branch + '-' + url_version url_fragment = '' - if (url != 'default'): + if (build_type != 'default'): url_fragment += url_branch + '/' + url_version + '/' if (machine == 'm3ulcb') or (machine == 'porter'): @@ -121,7 +127,7 @@ class Agljobtemplate(object): else: url_fragment += machine - if (url != 'ci'): + if (build_type != 'ci'): url_fragment += '/deploy/images/' + machine url = urlparse.urljoin(url_base, url_fragment) -- cgit 1.2.3-korg From 927d952dae1faa84a747ccd5b9885ed21ccc69bd Mon Sep 17 00:00:00 2001 From: Loys Ollivier Date: Wed, 3 Jan 2018 15:56:12 +0100 Subject: utils: define test-plans based on build-type As the list of test is expending, we want to be able to specify test plans grouping several test definitions. These test plans are build type specific. Create a config file that specifies the test plans for each build type. If the user does not specify the tests to run then use the default ones for the build-type. Change-Id: Ied4e9f80d3e42787174189cd08499a2906e500ef Signed-off-by: Loys Ollivier --- templates/URLs/default.cfg | 11 ----------- templates/config/default.cfg | 15 +++++++++++++++ utils/agljobtemplate.py | 42 ++++++++++++++++++++++++++---------------- utils/create-jobs.py | 3 --- 4 files changed, 41 insertions(+), 30 deletions(-) delete mode 100644 templates/URLs/default.cfg create mode 100644 templates/config/default.cfg (limited to 'utils/agljobtemplate.py') diff --git a/templates/URLs/default.cfg b/templates/URLs/default.cfg deleted file mode 100644 index 4c2adec..0000000 --- a/templates/URLs/default.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[infra] -style = AGL -[default] -urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/ -[release] -urlbase = https://download.automotivelinux.org/AGL/release/ -[weekly] -[daily] -urlbase = https://download.automotivelinux.org/AGL/snapshots/ -[ci] -urlbase = https://download.automotivelinux.org/AGL/upload/ci/ diff --git a/templates/config/default.cfg b/templates/config/default.cfg new file mode 100644 index 0000000..ba43270 --- /dev/null +++ b/templates/config/default.cfg @@ -0,0 +1,15 @@ +[infra] +style = AGL +[default] +urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/ +test_plan = ["health-test","smoke"] +[release] +urlbase = https://download.automotivelinux.org/AGL/release/ +test_plan = ["all"] +[weekly] +[daily] +urlbase = https://download.automotivelinux.org/AGL/snapshots/ +test_plan = ["health-test","smoke", "yocto-ptest"] +[ci] +urlbase = https://download.automotivelinux.org/AGL/upload/ci/ +test_plan = ["health-test","smoke"] diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py index 25efab6..a392845 100644 --- a/utils/agljobtemplate.py +++ b/utils/agljobtemplate.py @@ -5,24 +5,20 @@ import os import jinja2 import ConfigParser import urlparse +import ast def get_extension(path): return path.split('.')[-1] -def parse_url_file(template_path, url_file, build_type): - url_file_path = template_path + '/URLs/' + url_file +def parse_cfg_file(template_path, cfg_file, build_type): + url_file_path = template_path + '/config/' + cfg_file try: with open(url_file_path): cfg = ConfigParser.ConfigParser() cfg.read(url_file_path) - config = cfg.items(build_type) - for section in config: - if section[0] == "test_plan": - print section[1] - print build_type - return cfg.get(build_type, 'urlbase'), cfg.get('infra', 'style') + return cfg.items(build_type), cfg.get('infra', 'style') except IOError as err: raise err @@ -93,27 +89,25 @@ class Agljobtemplate(object): rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None, build_type=None, build_version=None): - test_templates = [] if machine not in self.machines: raise RuntimeError("{} is not a available machine".format(machine)) - for t in tests: - if t in self.tests: - test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2')) - else: - raise RuntimeError("{} is not an available test".format(t)) - # Populate jinja substitution dict job = {} job['name'] = job_name job['yocto_machine'] = machine job['priority'] = priority + defaults, infra = parse_cfg_file(self._template_path, 'default.cfg', build_type) + # If the user doesn't specify an URL, use the default one from the build-type if url is None: - url_base, infra = parse_url_file(self._template_path, 'default.cfg', build_type) if infra == 'AGL': + url_base = '' + for section in defaults: + if section[0] == "urlbase": + url_base = section[1] # If not set, create a build_version from other args if (not build_version) and (url_branch) and (url_version): build_version = 'AGL-' + build_type + '-' + url_branch + '-' + url_version @@ -132,6 +126,22 @@ class Agljobtemplate(object): url = urlparse.urljoin(url_base, url_fragment) + test_templates = [] + # If the user doesn't specify tests, use the default ones from the build-type + if not tests: + if infra == 'AGL': + for section in defaults: + if section[0] == "test_plan": + tests = ast.literal_eval(section[1]) + + if 'all' in tests: + tests = self.tests + for t in tests: + if t in self.tests: + test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2')) + else: + raise RuntimeError("{} is not an available test".format(t)) + job['urlbase'] = url job['test_templates'] = test_templates diff --git a/utils/create-jobs.py b/utils/create-jobs.py index 28fefa4..ad0c428 100755 --- a/utils/create-jobs.py +++ b/utils/create-jobs.py @@ -68,9 +68,6 @@ def main(): ajt = agljobtemplate.Agljobtemplate(templates_dir) args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types) - if args.tests is not None and 'all' in args.tests: - args.tests = ajt.tests - job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version, machine=args.machine, tests=args.tests, priority=args.priority, rfs_type=args.rfs_type, job_name=args.job_name, -- cgit 1.2.3-korg From 7df990a4080d77ebce1d9d867b207627e7059e85 Mon Sep 17 00:00:00 2001 From: Loys Ollivier Date: Wed, 3 Jan 2018 16:11:49 +0100 Subject: templates: fix the rootfs name for m3ulcb and rpi3 daily builds The rootfs filename has changed for the daily builds. Change the default to the new filename for both platforms. Change-Id: I44ae29902d882ab6f38dc14bba11cc8aee88f8c2 Signed-off-by: Loys Ollivier --- templates/base/agl-base-defaults.jinja2 | 1 + templates/machines/m3ulcb.jinja2 | 3 +++ templates/machines/raspberrypi3.jinja2 | 6 +++++- utils/agljobtemplate.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) (limited to 'utils/agljobtemplate.py') diff --git a/templates/base/agl-base-defaults.jinja2 b/templates/base/agl-base-defaults.jinja2 index d432b63..3f13f08 100644 --- a/templates/base/agl-base-defaults.jinja2 +++ b/templates/base/agl-base-defaults.jinja2 @@ -3,6 +3,7 @@ {%- endmacro %} {%- set action_timeout = action_timeout|default(15) %} {%- set boot_timeout = boot_timeout|default(10) %} +{%- set build_type = build_type|default("default") %} {%- set connection_timeout = connection_timeout|default(5) %} {%- set deploy_timeout = deploy_timeout|default(15) %} {%- set device_type = device_type|default(yocto_machine+"-uboot") %} diff --git a/templates/machines/m3ulcb.jinja2 b/templates/machines/m3ulcb.jinja2 index 41b6677..f337a8d 100644 --- a/templates/machines/m3ulcb.jinja2 +++ b/templates/machines/m3ulcb.jinja2 @@ -6,3 +6,6 @@ {%- set kernel_image = kernel_image|default("Image") %} {%- set rootfs_type = rootfs_type|default("nbd") %} {%- set uboot_type = "booti" %} +{%- if build_type == 'daily' %} + {%- set rfs_image = rfs_image|default("agl-image-ivi-crosssdk-m3ulcb.ext4.xz") %} +{%- endif %} diff --git a/templates/machines/raspberrypi3.jinja2 b/templates/machines/raspberrypi3.jinja2 index e415c2c..0e743a8 100644 --- a/templates/machines/raspberrypi3.jinja2 +++ b/templates/machines/raspberrypi3.jinja2 @@ -9,4 +9,8 @@ {%- set initrd = "initramfs-netboot-image-" + yocto_machine +".ext4.gz.u-boot" %} {%- endif %} {%- set dtb = dtb|default("uImage-bcm2710-rpi-3-b.dtb") %} -{%- set rfs_image = rfs_image|default("agl-demo-platform-raspberrypi3.ext4.xz") %} +{%- if build_type == 'daily' %} + {%- set rfs_image = rfs_image|default("agl-demo-platform-crosssdk-raspberrypi3.ext4.xz") %} +{%- else %} + {%- set rfs_image = rfs_image|default("agl-demo-platform-raspberrypi3.ext4.xz") %} +{%- endif %} diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py index a392845..dec7fad 100644 --- a/utils/agljobtemplate.py +++ b/utils/agljobtemplate.py @@ -98,6 +98,7 @@ class Agljobtemplate(object): job['name'] = job_name job['yocto_machine'] = machine job['priority'] = priority + job['build_type'] = build_type defaults, infra = parse_cfg_file(self._template_path, 'default.cfg', build_type) -- cgit 1.2.3-korg