From 788b31a122c3823d5e756524187975a2ad14cac6 Mon Sep 17 00:00:00 2001 From: Loys Ollivier Date: Wed, 3 Jan 2018 14:53:55 +0100 Subject: utils: change the url / build behavior scheme This is the first patch of the serie to differentiate builds and urls. As of now we were using the --url option to specify a build-type. Add a new option --build-type which specifies which type of build it is. From this build-type if no url is specified, the url can be defaulted by a configuration file. This configuration file is specific per user, e.g. AGL, others... Change-Id: I9ce801a7518b78ee859c6c3bbcad3a89e884e832 Signed-off-by: Loys Ollivier --- utils/create-jobs.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'utils/create-jobs.py') diff --git a/utils/create-jobs.py b/utils/create-jobs.py index e9701a5..28fefa4 100755 --- a/utils/create-jobs.py +++ b/utils/create-jobs.py @@ -17,12 +17,14 @@ def parse_cmdline(machines, tests, rfs_types): help="job priority", default='medium') parser.add_argument('--url', '-u', action='store', dest='url', - help="url fetch base", - default='release') + help="If using a custom URL specify it there. The files will be fetched from this URL.") + parser.add_argument('--build-type', dest='build_type', action='store', + help="The type of build, can be one of: {'ci', 'daily', 'weekly', 'release'}.", + default="default") parser.add_argument('--branch', '--changeid', dest='url_branch', action='store', - help='The branch (or changeid) to generate the job for.') + help='The build branch (or changeid).') parser.add_argument('--version', '--patchset', dest='url_version', action='store', - help='The version (or patchset) to generate the job for.') + help='The build version (or patchset).') parser.add_argument('--boot', action='store', dest='rfs_type', choices=rfs_types, help='select boot type') parser.add_argument('--callback-from', action='store', dest='callback_from', @@ -46,24 +48,16 @@ def parse_cmdline(machines, tests, rfs_types): parser.add_argument('--modules-img', dest='modules_img', action='store', help="The name of the modules to use (such as modules.tar.xz)") parser.add_argument('--build-version', dest='build_version', action='store', - help="the version number of the AGL build.") + help="the version number of the build.") args = parser.parse_args() if (bool(args.callback_from) ^ bool(args.callback_to)): parser.error("To specify callbacks both '--callback-to' and '--callback-from' are mandatory.") - if (args.url == 'release'): - if (args.url_branch is None) and (args.url_version is None): - args.url = 'default' - elif (args.url_branch is not None) != (args.url_version is not None): - parser.error("Both arguments: '--branch' and '--version' needs to be set") - elif (args.url == 'daily'): + if (args.build_type is not "default") and (args.url is None): if (not args.url_branch) or (not args.url_version): - parser.error("The argument '--url daily' requires '--branch' and '--version' to be set") - elif (args.url == 'ci'): - if (not args.url_branch) or (not args.url_version): - parser.error("The argument '--url ci' requires '--changeid' and '--patchset' to be set.") + parser.error("when using '--build-type' either '--url' or '--branch' and '--version' arguments needs to be set.") return args @@ -85,6 +79,7 @@ def main(): kernel_image=args.kernel_img, dtb_image=args.dtb_img, modules_image=args.modules_img, + build_type=args.build_type, build_version=args.build_version) if args.job_file is None: -- cgit 1.2.3-korg From 927d952dae1faa84a747ccd5b9885ed21ccc69bd Mon Sep 17 00:00:00 2001 From: Loys Ollivier Date: Wed, 3 Jan 2018 15:56:12 +0100 Subject: utils: define test-plans based on build-type As the list of test is expending, we want to be able to specify test plans grouping several test definitions. These test plans are build type specific. Create a config file that specifies the test plans for each build type. If the user does not specify the tests to run then use the default ones for the build-type. Change-Id: Ied4e9f80d3e42787174189cd08499a2906e500ef Signed-off-by: Loys Ollivier --- templates/URLs/default.cfg | 11 ----------- templates/config/default.cfg | 15 +++++++++++++++ utils/agljobtemplate.py | 42 ++++++++++++++++++++++++++---------------- utils/create-jobs.py | 3 --- 4 files changed, 41 insertions(+), 30 deletions(-) delete mode 100644 templates/URLs/default.cfg create mode 100644 templates/config/default.cfg (limited to 'utils/create-jobs.py') diff --git a/templates/URLs/default.cfg b/templates/URLs/default.cfg deleted file mode 100644 index 4c2adec..0000000 --- a/templates/URLs/default.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[infra] -style = AGL -[default] -urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/ -[release] -urlbase = https://download.automotivelinux.org/AGL/release/ -[weekly] -[daily] -urlbase = https://download.automotivelinux.org/AGL/snapshots/ -[ci] -urlbase = https://download.automotivelinux.org/AGL/upload/ci/ diff --git a/templates/config/default.cfg b/templates/config/default.cfg new file mode 100644 index 0000000..ba43270 --- /dev/null +++ b/templates/config/default.cfg @@ -0,0 +1,15 @@ +[infra] +style = AGL +[default] +urlbase = https://download.automotivelinux.org/AGL/release/dab/4.0.2/ +test_plan = ["health-test","smoke"] +[release] +urlbase = https://download.automotivelinux.org/AGL/release/ +test_plan = ["all"] +[weekly] +[daily] +urlbase = https://download.automotivelinux.org/AGL/snapshots/ +test_plan = ["health-test","smoke", "yocto-ptest"] +[ci] +urlbase = https://download.automotivelinux.org/AGL/upload/ci/ +test_plan = ["health-test","smoke"] diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py index 25efab6..a392845 100644 --- a/utils/agljobtemplate.py +++ b/utils/agljobtemplate.py @@ -5,24 +5,20 @@ import os import jinja2 import ConfigParser import urlparse +import ast def get_extension(path): return path.split('.')[-1] -def parse_url_file(template_path, url_file, build_type): - url_file_path = template_path + '/URLs/' + url_file +def parse_cfg_file(template_path, cfg_file, build_type): + url_file_path = template_path + '/config/' + cfg_file try: with open(url_file_path): cfg = ConfigParser.ConfigParser() cfg.read(url_file_path) - config = cfg.items(build_type) - for section in config: - if section[0] == "test_plan": - print section[1] - print build_type - return cfg.get(build_type, 'urlbase'), cfg.get('infra', 'style') + return cfg.items(build_type), cfg.get('infra', 'style') except IOError as err: raise err @@ -93,27 +89,25 @@ class Agljobtemplate(object): rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None, build_type=None, build_version=None): - test_templates = [] if machine not in self.machines: raise RuntimeError("{} is not a available machine".format(machine)) - for t in tests: - if t in self.tests: - test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2')) - else: - raise RuntimeError("{} is not an available test".format(t)) - # Populate jinja substitution dict job = {} job['name'] = job_name job['yocto_machine'] = machine job['priority'] = priority + defaults, infra = parse_cfg_file(self._template_path, 'default.cfg', build_type) + # If the user doesn't specify an URL, use the default one from the build-type if url is None: - url_base, infra = parse_url_file(self._template_path, 'default.cfg', build_type) if infra == 'AGL': + url_base = '' + for section in defaults: + if section[0] == "urlbase": + url_base = section[1] # If not set, create a build_version from other args if (not build_version) and (url_branch) and (url_version): build_version = 'AGL-' + build_type + '-' + url_branch + '-' + url_version @@ -132,6 +126,22 @@ class Agljobtemplate(object): url = urlparse.urljoin(url_base, url_fragment) + test_templates = [] + # If the user doesn't specify tests, use the default ones from the build-type + if not tests: + if infra == 'AGL': + for section in defaults: + if section[0] == "test_plan": + tests = ast.literal_eval(section[1]) + + if 'all' in tests: + tests = self.tests + for t in tests: + if t in self.tests: + test_templates.append(os.path.join(self.TESTS_DIR, t + '.jinja2')) + else: + raise RuntimeError("{} is not an available test".format(t)) + job['urlbase'] = url job['test_templates'] = test_templates diff --git a/utils/create-jobs.py b/utils/create-jobs.py index 28fefa4..ad0c428 100755 --- a/utils/create-jobs.py +++ b/utils/create-jobs.py @@ -68,9 +68,6 @@ def main(): ajt = agljobtemplate.Agljobtemplate(templates_dir) args = parse_cmdline(ajt.machines, ajt.tests, ajt.rfs_types) - if args.tests is not None and 'all' in args.tests: - args.tests = ajt.tests - job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version, machine=args.machine, tests=args.tests, priority=args.priority, rfs_type=args.rfs_type, job_name=args.job_name, -- cgit 1.2.3-korg