diff options
Diffstat (limited to 'roms/u-boot/tools/buildman/builder.py')
| -rw-r--r-- | roms/u-boot/tools/buildman/builder.py | 1744 |
1 files changed, 1744 insertions, 0 deletions
diff --git a/roms/u-boot/tools/buildman/builder.py b/roms/u-boot/tools/buildman/builder.py new file mode 100644 index 000000000..ce852eb03 --- /dev/null +++ b/roms/u-boot/tools/buildman/builder.py @@ -0,0 +1,1744 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2013 The Chromium OS Authors. +# +# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> +# + +import collections +from datetime import datetime, timedelta +import glob +import os +import re +import queue +import shutil +import signal +import string +import sys +import threading +import time + +from buildman import builderthread +from buildman import toolchain +from patman import command +from patman import gitutil +from patman import terminal +from patman.terminal import Print + +""" +Theory of Operation + +Please see README for user documentation, and you should be familiar with +that before trying to make sense of this. + +Buildman works by keeping the machine as busy as possible, building different +commits for different boards on multiple CPUs at once. + +The source repo (self.git_dir) contains all the commits to be built. Each +thread works on a single board at a time. It checks out the first commit, +configures it for that board, then builds it. Then it checks out the next +commit and builds it (typically without re-configuring). When it runs out +of commits, it gets another job from the builder and starts again with that +board. + +Clearly the builder threads could work either way - they could check out a +commit and then built it for all boards. Using separate directories for each +commit/board pair they could leave their build product around afterwards +also. + +The intent behind building a single board for multiple commits, is to make +use of incremental builds. Since each commit is built incrementally from +the previous one, builds are faster. Reconfiguring for a different board +removes all intermediate object files. + +Many threads can be working at once, but each has its own working directory. +When a thread finishes a build, it puts the output files into a result +directory. + +The base directory used by buildman is normally '../<branch>', i.e. +a directory higher than the source repository and named after the branch +being built. + +Within the base directory, we have one subdirectory for each commit. Within +that is one subdirectory for each board. Within that is the build output for +that commit/board combination. + +Buildman also create working directories for each thread, in a .bm-work/ +subdirectory in the base dir. + +As an example, say we are building branch 'us-net' for boards 'sandbox' and +'seaboard', and say that us-net has two commits. We will have directories +like this: + +us-net/ base directory + 01_g4ed4ebc_net--Add-tftp-speed-/ + sandbox/ + u-boot.bin + seaboard/ + u-boot.bin + 02_g4ed4ebc_net--Check-tftp-comp/ + sandbox/ + u-boot.bin + seaboard/ + u-boot.bin + .bm-work/ + 00/ working directory for thread 0 (contains source checkout) + build/ build output + 01/ working directory for thread 1 + build/ build output + ... +u-boot/ source directory + .git/ repository +""" + +"""Holds information about a particular error line we are outputing + + char: Character representation: '+': error, '-': fixed error, 'w+': warning, + 'w-' = fixed warning + boards: List of Board objects which have line in the error/warning output + errline: The text of the error line +""" +ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline') + +# Possible build outcomes +OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4)) + +# Translate a commit subject into a valid filename (and handle unicode) +trans_valid_chars = str.maketrans('/: ', '---') + +BASE_CONFIG_FILENAMES = [ + 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' +] + +EXTRA_CONFIG_FILENAMES = [ + '.config', '.config-spl', '.config-tpl', + 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', + 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', +] + +class Config: + """Holds information about configuration settings for a board.""" + def __init__(self, config_filename, target): + self.target = target + self.config = {} + for fname in config_filename: + self.config[fname] = {} + + def Add(self, fname, key, value): + self.config[fname][key] = value + + def __hash__(self): + val = 0 + for fname in self.config: + for key, value in self.config[fname].items(): + print(key, value) + val = val ^ hash(key) & hash(value) + return val + +class Environment: + """Holds information about environment variables for a board.""" + def __init__(self, target): + self.target = target + self.environment = {} + + def Add(self, key, value): + self.environment[key] = value + +class Builder: + """Class for building U-Boot for a particular commit. + + Public members: (many should ->private) + already_done: Number of builds already completed + base_dir: Base directory to use for builder + checkout: True to check out source, False to skip that step. + This is used for testing. + col: terminal.Color() object + count: Number of commits to build + do_make: Method to call to invoke Make + fail: Number of builds that failed due to error + force_build: Force building even if a build already exists + force_config_on_failure: If a commit fails for a board, disable + incremental building for the next commit we build for that + board, so that we will see all warnings/errors again. + force_build_failures: If a previously-built build (i.e. built on + a previous run of buildman) is marked as failed, rebuild it. + git_dir: Git directory containing source repository + num_jobs: Number of jobs to run at once (passed to make as -j) + num_threads: Number of builder threads to run + out_queue: Queue of results to process + re_make_err: Compiled regular expression for ignore_lines + queue: Queue of jobs to run + threads: List of active threads + toolchains: Toolchains object to use for building + upto: Current commit number we are building (0.count-1) + warned: Number of builds that produced at least one warning + force_reconfig: Reconfigure U-Boot on each comiit. This disables + incremental building, where buildman reconfigures on the first + commit for a baord, and then just does an incremental build for + the following commits. In fact buildman will reconfigure and + retry for any failing commits, so generally the only effect of + this option is to slow things down. + in_tree: Build U-Boot in-tree instead of specifying an output + directory separate from the source code. This option is really + only useful for testing in-tree builds. + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + thread_exceptions: List of exceptions raised by thread jobs + + Private members: + _base_board_dict: Last-summarised Dict of boards + _base_err_lines: Last-summarised list of errors + _base_warn_lines: Last-summarised list of warnings + _build_period_us: Time taken for a single build (float object). + _complete_delay: Expected delay until completion (timedelta) + _next_delay_update: Next time we plan to display a progress update + (datatime) + _show_unknown: Show unknown boards (those not built) in summary + _start_time: Start time for the build + _timestamps: List of timestamps for the completion of the last + last _timestamp_count builds. Each is a datetime object. + _timestamp_count: Number of timestamps to keep in our list. + _working_dir: Base working directory containing all threads + _single_builder: BuilderThread object for the singer builder, if + threading is not being used + """ + class Outcome: + """Records a build outcome for a single make invocation + + Public Members: + rc: Outcome value (OUTCOME_...) + err_lines: List of error lines or [] if none + sizes: Dictionary of image size information, keyed by filename + - Each value is itself a dictionary containing + values for 'text', 'data' and 'bss', being the integer + size in bytes of each section. + func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each + value is itself a dictionary: + key: function name + value: Size of function in bytes + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. + """ + def __init__(self, rc, err_lines, sizes, func_sizes, config, + environment): + self.rc = rc + self.err_lines = err_lines + self.sizes = sizes + self.func_sizes = func_sizes + self.config = config + self.environment = environment + + def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False, full_path=False, verbose_build=False, + mrproper=False, per_board_out_dir=False, + config_only=False, squash_config_y=False, + warnings_as_errors=False, work_in_output=False, + test_thread_exceptions=False): + """Create a new Builder object + + Args: + toolchains: Toolchains object to use for building + base_dir: Base directory to use for builder + git_dir: Git directory containing source repository + num_threads: Number of builder threads to run + num_jobs: Number of jobs to run at once (passed to make as -j) + gnu_make: the command name of GNU Make. + checkout: True to check out source, False to skip that step. + This is used for testing. + show_unknown: Show unknown boards (those not built) in summary + step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + verbose_build: Run build with V=1 and don't use 'make -s' + mrproper: Always run 'make mrproper' when configuring + per_board_out_dir: Build in a separate persistent directory per + board rather than a thread-specific directory + config_only: Only configure each build, don't build it + squash_config_y: Convert CONFIG options with the value 'y' to '1' + warnings_as_errors: Treat all compiler warnings as errors + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + test_thread_exceptions: Uses for tests only, True to make the + threads raise an exception instead of reporting their result. + This simulates a failure in the code somewhere + """ + self.toolchains = toolchains + self.base_dir = base_dir + if work_in_output: + self._working_dir = base_dir + else: + self._working_dir = os.path.join(base_dir, '.bm-work') + self.threads = [] + self.do_make = self.Make + self.gnu_make = gnu_make + self.checkout = checkout + self.num_threads = num_threads + self.num_jobs = num_jobs + self.already_done = 0 + self.force_build = False + self.git_dir = git_dir + self._show_unknown = show_unknown + self._timestamp_count = 10 + self._build_period_us = None + self._complete_delay = None + self._next_delay_update = datetime.now() + self._start_time = datetime.now() + self.force_config_on_failure = True + self.force_build_failures = False + self.force_reconfig = False + self._step = step + self.in_tree = False + self._error_lines = 0 + self.no_subdirs = no_subdirs + self.full_path = full_path + self.verbose_build = verbose_build + self.config_only = config_only + self.squash_config_y = squash_config_y + self.config_filenames = BASE_CONFIG_FILENAMES + self.work_in_output = work_in_output + if not self.squash_config_y: + self.config_filenames += EXTRA_CONFIG_FILENAMES + + self.warnings_as_errors = warnings_as_errors + self.col = terminal.Color() + + self._re_function = re.compile('(.*): In function.*') + self._re_files = re.compile('In file included from.*') + self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') + self._re_dtb_warning = re.compile('(.*): Warning .*') + self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n', + re.MULTILINE | re.DOTALL) + + self.thread_exceptions = [] + self.test_thread_exceptions = test_thread_exceptions + if self.num_threads: + self._single_builder = None + self.queue = queue.Queue() + self.out_queue = queue.Queue() + for i in range(self.num_threads): + t = builderthread.BuilderThread( + self, i, mrproper, per_board_out_dir, + test_exception=test_thread_exceptions) + t.setDaemon(True) + t.start() + self.threads.append(t) + + t = builderthread.ResultThread(self) + t.setDaemon(True) + t.start() + self.threads.append(t) + else: + self._single_builder = builderthread.BuilderThread( + self, -1, mrproper, per_board_out_dir) + + ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] + self.re_make_err = re.compile('|'.join(ignore_lines)) + + # Handle existing graceful with SIGINT / Ctrl-C + signal.signal(signal.SIGINT, self.signal_handler) + + def __del__(self): + """Get rid of all threads created by the builder""" + for t in self.threads: + del t + + def signal_handler(self, signal, frame): + sys.exit(1) + + def SetDisplayOptions(self, show_errors=False, show_sizes=False, + show_detail=False, show_bloat=False, + list_error_boards=False, show_config=False, + show_environment=False, filter_dtb_warnings=False, + filter_migration_warnings=False): + """Setup display options for the builder. + + Args: + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show size delta detail for each board if show_sizes + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + show_config: Show config deltas + show_environment: Show environment deltas + filter_dtb_warnings: Filter out any warnings from the device-tree + compiler + filter_migration_warnings: Filter out any warnings about migrating + a board to driver model + """ + self._show_errors = show_errors + self._show_sizes = show_sizes + self._show_detail = show_detail + self._show_bloat = show_bloat + self._list_error_boards = list_error_boards + self._show_config = show_config + self._show_environment = show_environment + self._filter_dtb_warnings = filter_dtb_warnings + self._filter_migration_warnings = filter_migration_warnings + + def _AddTimestamp(self): + """Add a new timestamp to the list and record the build period. + + The build period is the length of time taken to perform a single + build (one board, one commit). + """ + now = datetime.now() + self._timestamps.append(now) + count = len(self._timestamps) + delta = self._timestamps[-1] - self._timestamps[0] + seconds = delta.total_seconds() + + # If we have enough data, estimate build period (time taken for a + # single build) and therefore completion time. + if count > 1 and self._next_delay_update < now: + self._next_delay_update = now + timedelta(seconds=2) + if seconds > 0: + self._build_period = float(seconds) / count + todo = self.count - self.upto + self._complete_delay = timedelta(microseconds= + self._build_period * todo * 1000000) + # Round it + self._complete_delay -= timedelta( + microseconds=self._complete_delay.microseconds) + + if seconds > 60: + self._timestamps.popleft() + count -= 1 + + def SelectCommit(self, commit, checkout=True): + """Checkout the selected commit for this build + """ + self.commit = commit + if checkout and self.checkout: + gitutil.Checkout(commit.hash) + + def Make(self, commit, brd, stage, cwd, *args, **kwargs): + """Run make + + Args: + commit: Commit object that is being built + brd: Board object that is being built + stage: Stage that we are at (mrproper, config, build) + cwd: Directory where make should be run + args: Arguments to pass to make + kwargs: Arguments to pass to command.RunPipe() + """ + cmd = [self.gnu_make] + list(args) + result = command.RunPipe([cmd], capture=True, capture_stderr=True, + cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs) + if self.verbose_build: + result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout + result.combined = '%s\n' % (' '.join(cmd)) + result.combined + return result + + def ProcessResult(self, result): + """Process the result of a build, showing progress information + + Args: + result: A CommandResult object, which indicates the result for + a single build + """ + col = terminal.Color() + if result: + target = result.brd.target + + self.upto += 1 + if result.return_code != 0: + self.fail += 1 + elif result.stderr: + self.warned += 1 + if result.already_done: + self.already_done += 1 + if self._verbose: + terminal.PrintClear() + boards_selected = {target : result.brd} + self.ResetResultSummary(boards_selected) + self.ProduceResultSummary(result.commit_upto, self.commits, + boards_selected) + else: + target = '(starting)' + + # Display separate counts for ok, warned and fail + ok = self.upto - self.warned - self.fail + line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) + line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) + line += self.col.Color(self.col.RED, '%5d' % self.fail) + + line += ' /%-5d ' % self.count + remaining = self.count - self.upto + if remaining: + line += self.col.Color(self.col.MAGENTA, ' -%-5d ' % remaining) + else: + line += ' ' * 8 + + # Add our current completion time estimate + self._AddTimestamp() + if self._complete_delay: + line += '%s : ' % self._complete_delay + + line += target + terminal.PrintClear() + Print(line, newline=False, limit_to_line=True) + + def _GetOutputDir(self, commit_upto): + """Get the name of the output directory for a commit number + + The output directory is typically .../<branch>/<commit>. + + Args: + commit_upto: Commit number to use (0..self.count-1) + """ + if self.work_in_output: + return self._working_dir + + commit_dir = None + if self.commits: + commit = self.commits[commit_upto] + subject = commit.subject.translate(trans_valid_chars) + # See _GetOutputSpaceRemovals() which parses this name + commit_dir = ('%02d_g%s_%s' % (commit_upto + 1, + commit.hash, subject[:20])) + elif not self.no_subdirs: + commit_dir = 'current' + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir) + + def GetBuildDir(self, commit_upto, target): + """Get the name of the build directory for a commit number + + The build directory is typically .../<branch>/<commit>/<target>. + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + output_dir = self._GetOutputDir(commit_upto) + if self.work_in_output: + return output_dir + return os.path.join(output_dir, target) + + def GetDoneFile(self, commit_upto, target): + """Get the name of the done file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + return os.path.join(self.GetBuildDir(commit_upto, target), 'done') + + def GetSizesFile(self, commit_upto, target): + """Get the name of the sizes file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') + + def GetFuncSizesFile(self, commit_upto, target, elf_fname): + """Get the name of the funcsizes file for a commit number and ELF file + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + elf_fname: Filename of elf image + """ + return os.path.join(self.GetBuildDir(commit_upto, target), + '%s.sizes' % elf_fname.replace('/', '-')) + + def GetObjdumpFile(self, commit_upto, target, elf_fname): + """Get the name of the objdump file for a commit number and ELF file + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + elf_fname: Filename of elf image + """ + return os.path.join(self.GetBuildDir(commit_upto, target), + '%s.objdump' % elf_fname.replace('/', '-')) + + def GetErrFile(self, commit_upto, target): + """Get the name of the err file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + output_dir = self.GetBuildDir(commit_upto, target) + return os.path.join(output_dir, 'err') + + def FilterErrors(self, lines): + """Filter out errors in which we have no interest + + We should probably use map(). + + Args: + lines: List of error lines, each a string + Returns: + New list with only interesting lines included + """ + out_lines = [] + if self._filter_migration_warnings: + text = '\n'.join(lines) + text = self._re_migration_warning.sub('', text) + lines = text.splitlines() + for line in lines: + if self.re_make_err.search(line): + continue + if self._filter_dtb_warnings and self._re_dtb_warning.search(line): + continue + out_lines.append(line) + return out_lines + + def ReadFuncSizes(self, fname, fd): + """Read function sizes from the output of 'nm' + + Args: + fd: File containing data to read + fname: Filename we are reading from (just for errors) + + Returns: + Dictionary containing size of each function in bytes, indexed by + function name. + """ + sym = {} + for line in fd.readlines(): + try: + if line.strip(): + size, type, name = line[:-1].split() + except: + Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) + continue + if type in 'tTdDbB': + # function names begin with '.' on 64-bit powerpc + if '.' in name[1:]: + name = 'static.' + name.split('.')[0] + sym[name] = sym.get(name, 0) + int(size, 16) + return sym + + def _ProcessConfig(self, fname): + """Read in a .config, autoconf.mk or autoconf.h file + + This function handles all config file types. It ignores comments and + any #defines which don't start with CONFIG_. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: Config name (e.g. CONFIG_DM) + value: Config value (e.g. 1) + """ + config = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd: + line = line.strip() + if line.startswith('#define'): + values = line[8:].split(' ', 1) + if len(values) > 1: + key, value = values + else: + key = values[0] + value = '1' if self.squash_config_y else '' + if not key.startswith('CONFIG_'): + continue + elif not line or line[0] in ['#', '*', '/']: + continue + else: + key, value = line.split('=', 1) + if self.squash_config_y and value == 'y': + value = '1' + config[key] = value + return config + + def _ProcessEnvironment(self, fname): + """Read in a uboot.env file + + This function reads in environment variables from a file. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: environment variable (e.g. bootlimit) + value: value of environment variable (e.g. 1) + """ + environment = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd.read().split('\0'): + try: + key, value = line.split('=', 1) + environment[key] = value + except ValueError: + # ignore lines we can't parse + pass + return environment + + def GetBuildOutcome(self, commit_upto, target, read_func_sizes, + read_config, read_environment): + """Work out the outcome of a build. + + Args: + commit_upto: Commit number to check (0..n-1) + target: Target board to check + read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files + + Returns: + Outcome object + """ + done_file = self.GetDoneFile(commit_upto, target) + sizes_file = self.GetSizesFile(commit_upto, target) + sizes = {} + func_sizes = {} + config = {} + environment = {} + if os.path.exists(done_file): + with open(done_file, 'r') as fd: + try: + return_code = int(fd.readline()) + except ValueError: + # The file may be empty due to running out of disk space. + # Try a rebuild + return_code = 1 + err_lines = [] + err_file = self.GetErrFile(commit_upto, target) + if os.path.exists(err_file): + with open(err_file, 'r') as fd: + err_lines = self.FilterErrors(fd.readlines()) + + # Decide whether the build was ok, failed or created warnings + if return_code: + rc = OUTCOME_ERROR + elif len(err_lines): + rc = OUTCOME_WARNING + else: + rc = OUTCOME_OK + + # Convert size information to our simple format + if os.path.exists(sizes_file): + with open(sizes_file, 'r') as fd: + for line in fd.readlines(): + values = line.split() + rodata = 0 + if len(values) > 6: + rodata = int(values[6], 16) + size_dict = { + 'all' : int(values[0]) + int(values[1]) + + int(values[2]), + 'text' : int(values[0]) - rodata, + 'data' : int(values[1]), + 'bss' : int(values[2]), + 'rodata' : rodata, + } + sizes[values[5]] = size_dict + + if read_func_sizes: + pattern = self.GetFuncSizesFile(commit_upto, target, '*') + for fname in glob.glob(pattern): + with open(fname, 'r') as fd: + dict_name = os.path.basename(fname).replace('.sizes', + '') + func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) + + if read_config: + output_dir = self.GetBuildDir(commit_upto, target) + for name in self.config_filenames: + fname = os.path.join(output_dir, name) + config[name] = self._ProcessConfig(fname) + + if read_environment: + output_dir = self.GetBuildDir(commit_upto, target) + fname = os.path.join(output_dir, 'uboot.env') + environment = self._ProcessEnvironment(fname) + + return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, + environment) + + return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) + + def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, + read_config, read_environment): + """Calculate a summary of the results of building a commit. + + Args: + board_selected: Dict containing boards to summarise + commit_upto: Commit number to summarize (0..self.count-1) + read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files + + Returns: + Tuple: + Dict containing boards which passed building this commit. + keyed by board.target + List containing a summary of error lines + Dict keyed by error line, containing a list of the Board + objects with that error + List containing a summary of warning lines + Dict keyed by error line, containing a list of the Board + objects with that warning + Dictionary keyed by board.target. Each value is a dictionary: + key: filename - e.g. '.config' + value is itself a dictionary: + key: config name + value: config value + Dictionary keyed by board.target. Each value is a dictionary: + key: environment variable + value: value of environment variable + """ + def AddLine(lines_summary, lines_boards, line, board): + line = line.rstrip() + if line in lines_boards: + lines_boards[line].append(board) + else: + lines_boards[line] = [board] + lines_summary.append(line) + + board_dict = {} + err_lines_summary = [] + err_lines_boards = {} + warn_lines_summary = [] + warn_lines_boards = {} + config = {} + environment = {} + + for board in boards_selected.values(): + outcome = self.GetBuildOutcome(commit_upto, board.target, + read_func_sizes, read_config, + read_environment) + board_dict[board.target] = outcome + last_func = None + last_was_warning = False + for line in outcome.err_lines: + if line: + if (self._re_function.match(line) or + self._re_files.match(line)): + last_func = line + else: + is_warning = (self._re_warning.match(line) or + self._re_dtb_warning.match(line)) + is_note = self._re_note.match(line) + if is_warning or (last_was_warning and is_note): + if last_func: + AddLine(warn_lines_summary, warn_lines_boards, + last_func, board) + AddLine(warn_lines_summary, warn_lines_boards, + line, board) + else: + if last_func: + AddLine(err_lines_summary, err_lines_boards, + last_func, board) + AddLine(err_lines_summary, err_lines_boards, + line, board) + last_was_warning = is_warning + last_func = None + tconfig = Config(self.config_filenames, board.target) + for fname in self.config_filenames: + if outcome.config: + for key, value in outcome.config[fname].items(): + tconfig.Add(fname, key, value) + config[board.target] = tconfig + + tenvironment = Environment(board.target) + if outcome.environment: + for key, value in outcome.environment.items(): + tenvironment.Add(key, value) + environment[board.target] = tenvironment + + return (board_dict, err_lines_summary, err_lines_boards, + warn_lines_summary, warn_lines_boards, config, environment) + + def AddOutcome(self, board_dict, arch_list, changes, char, color): + """Add an output to our list of outcomes for each architecture + + This simple function adds failing boards (changes) to the + relevant architecture string, so we can print the results out + sorted by architecture. + + Args: + board_dict: Dict containing all boards + arch_list: Dict keyed by arch name. Value is a string containing + a list of board names which failed for that arch. + changes: List of boards to add to arch_list + color: terminal.Colour object + """ + done_arch = {} + for target in changes: + if target in board_dict: + arch = board_dict[target].arch + else: + arch = 'unknown' + str = self.col.Color(color, ' ' + target) + if not arch in done_arch: + str = ' %s %s' % (self.col.Color(color, char), str) + done_arch[arch] = True + if not arch in arch_list: + arch_list[arch] = str + else: + arch_list[arch] += str + + + def ColourNum(self, num): + color = self.col.RED if num > 0 else self.col.GREEN + if num == 0: + return '0' + return self.col.Color(color, str(num)) + + def ResetResultSummary(self, board_selected): + """Reset the results summary ready for use. + + Set up the base board list to be all those selected, and set the + error lines to empty. + + Following this, calls to PrintResultSummary() will use this + information to work out what has changed. + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + """ + self._base_board_dict = {} + for board in board_selected: + self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, + {}) + self._base_err_lines = [] + self._base_warn_lines = [] + self._base_err_line_boards = {} + self._base_warn_line_boards = {} + self._base_config = None + self._base_environment = None + + def PrintFuncSizeDetail(self, fname, old, new): + grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 + delta, common = [], {} + + for a in old: + if a in new: + common[a] = 1 + + for name in old: + if name not in common: + remove += 1 + down += old[name] + delta.append([-old[name], name]) + + for name in new: + if name not in common: + add += 1 + up += new[name] + delta.append([new[name], name]) + + for name in common: + diff = new.get(name, 0) - old.get(name, 0) + if diff > 0: + grow, up = grow + 1, up + diff + elif diff < 0: + shrink, down = shrink + 1, down - diff + delta.append([diff, name]) + + delta.sort() + delta.reverse() + + args = [add, -remove, grow, -shrink, up, -down, up - down] + if max(args) == 0 and min(args) == 0: + return + args = [self.ColourNum(x) for x in args] + indent = ' ' * 15 + Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % + tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) + Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', + 'delta')) + for diff, name in delta: + if diff: + color = self.col.RED if diff > 0 else self.col.GREEN + msg = '%s %-38s %7s %7s %+7d' % (indent, name, + old.get(name, '-'), new.get(name,'-'), diff) + Print(msg, colour=color) + + + def PrintSizeDetail(self, target_list, show_bloat): + """Show details size information for each board + + Args: + target_list: List of targets, each a dict containing: + 'target': Target name + 'total_diff': Total difference in bytes across all areas + <part_name>: Difference for that part + show_bloat: Show detail for each function + """ + targets_by_diff = sorted(target_list, reverse=True, + key=lambda x: x['_total_diff']) + for result in targets_by_diff: + printed_target = False + for name in sorted(result): + diff = result[name] + if name.startswith('_'): + continue + if diff != 0: + color = self.col.RED if diff > 0 else self.col.GREEN + msg = ' %s %+d' % (name, diff) + if not printed_target: + Print('%10s %-15s:' % ('', result['_target']), + newline=False) + printed_target = True + Print(msg, colour=color, newline=False) + if printed_target: + Print() + if show_bloat: + target = result['_target'] + outcome = result['_outcome'] + base_outcome = self._base_board_dict[target] + for fname in outcome.func_sizes: + self.PrintFuncSizeDetail(fname, + base_outcome.func_sizes[fname], + outcome.func_sizes[fname]) + + + def PrintSizeSummary(self, board_selected, board_dict, show_detail, + show_bloat): + """Print a summary of image sizes broken down by section. + + The summary takes the form of one line per architecture. The + line contains deltas for each of the sections (+ means the section + got bigger, - means smaller). The numbers are the average number + of bytes that a board in this section increased by. + + For example: + powerpc: (622 boards) text -0.0 + arm: (285 boards) text -0.0 + nds32: (3 boards) text -8.0 + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + board_dict: Dict containing boards for which we built this + commit, keyed by board.target. The value is an Outcome object. + show_detail: Show size delta detail for each board + show_bloat: Show detail for each function + """ + arch_list = {} + arch_count = {} + + # Calculate changes in size for different image parts + # The previous sizes are in Board.sizes, for each board + for target in board_dict: + if target not in board_selected: + continue + base_sizes = self._base_board_dict[target].sizes + outcome = board_dict[target] + sizes = outcome.sizes + + # Loop through the list of images, creating a dict of size + # changes for each image/part. We end up with something like + # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} + # which means that U-Boot data increased by 5 bytes and SPL + # text decreased by 4. + err = {'_target' : target} + for image in sizes: + if image in base_sizes: + base_image = base_sizes[image] + # Loop through the text, data, bss parts + for part in sorted(sizes[image]): + diff = sizes[image][part] - base_image[part] + col = None + if diff: + if image == 'u-boot': + name = part + else: + name = image + ':' + part + err[name] = diff + arch = board_selected[target].arch + if not arch in arch_count: + arch_count[arch] = 1 + else: + arch_count[arch] += 1 + if not sizes: + pass # Only add to our list when we have some stats + elif not arch in arch_list: + arch_list[arch] = [err] + else: + arch_list[arch].append(err) + + # We now have a list of image size changes sorted by arch + # Print out a summary of these + for arch, target_list in arch_list.items(): + # Get total difference for each type + totals = {} + for result in target_list: + total = 0 + for name, diff in result.items(): + if name.startswith('_'): + continue + total += diff + if name in totals: + totals[name] += diff + else: + totals[name] = diff + result['_total_diff'] = total + result['_outcome'] = board_dict[result['_target']] + + count = len(target_list) + printed_arch = False + for name in sorted(totals): + diff = totals[name] + if diff: + # Display the average difference in this name for this + # architecture + avg_diff = float(diff) / count + color = self.col.RED if avg_diff > 0 else self.col.GREEN + msg = ' %s %+1.1f' % (name, avg_diff) + if not printed_arch: + Print('%10s: (for %d/%d boards)' % (arch, count, + arch_count[arch]), newline=False) + printed_arch = True + Print(msg, colour=color, newline=False) + + if printed_arch: + Print() + if show_detail: + self.PrintSizeDetail(target_list, show_bloat) + + + def PrintResultSummary(self, board_selected, board_dict, err_lines, + err_line_boards, warn_lines, warn_line_boards, + config, environment, show_sizes, show_detail, + show_bloat, show_config, show_environment): + """Compare results with the base results and display delta. + + Only boards mentioned in board_selected will be considered. This + function is intended to be called repeatedly with the results of + each commit. It therefore shows a 'diff' between what it saw in + the last call and what it sees now. + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + board_dict: Dict containing boards for which we built this + commit, keyed by board.target. The value is an Outcome object. + err_lines: A list of errors for this commit, or [] if there is + none, or we don't want to print errors + err_line_boards: Dict keyed by error line, containing a list of + the Board objects with that error + warn_lines: A list of warnings for this commit, or [] if there is + none, or we don't want to print errors + warn_line_boards: Dict keyed by warning line, containing a list of + the Board objects with that warning + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. + show_sizes: Show image size deltas + show_detail: Show size delta detail for each board if show_sizes + show_bloat: Show detail for each function + show_config: Show config changes + show_environment: Show environment changes + """ + def _BoardList(line, line_boards): + """Helper function to get a line of boards containing a line + + Args: + line: Error line to search for + line_boards: boards to search, each a Board + Return: + List of boards with that error line, or [] if the user has not + requested such a list + """ + boards = [] + board_set = set() + if self._list_error_boards: + for board in line_boards[line]: + if not board in board_set: + boards.append(board) + board_set.add(board) + return boards + + def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, + char): + """Calculate the required output based on changes in errors + + Args: + base_lines: List of errors/warnings for previous commit + base_line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in the previous commit + lines: List of errors/warning for this commit, each a str + line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in this commit + char: Character representing error ('') or warning ('w'). The + broken ('+') or fixed ('-') characters are added in this + function + + Returns: + Tuple + List of ErrLine objects for 'better' lines + List of ErrLine objects for 'worse' lines + """ + better_lines = [] + worse_lines = [] + for line in lines: + if line not in base_lines: + errline = ErrLine(char + '+', _BoardList(line, line_boards), + line) + worse_lines.append(errline) + for line in base_lines: + if line not in lines: + errline = ErrLine(char + '-', + _BoardList(line, base_line_boards), line) + better_lines.append(errline) + return better_lines, worse_lines + + def _CalcConfig(delta, name, config): + """Calculate configuration changes + + Args: + delta: Type of the delta, e.g. '+' + name: name of the file which changed (e.g. .config) + config: configuration change dictionary + key: config name + value: config value + Returns: + String containing the configuration changes which can be + printed + """ + out = '' + for key in sorted(config.keys()): + out += '%s=%s ' % (key, config[key]) + return '%s %s: %s' % (delta, name, out) + + def _AddConfig(lines, name, config_plus, config_minus, config_change): + """Add changes in configuration to a list + + Args: + lines: list to add to + name: config file name + config_plus: configurations added, dictionary + key: config name + value: config value + config_minus: configurations removed, dictionary + key: config name + value: config value + config_change: configurations changed, dictionary + key: config name + value: config value + """ + if config_plus: + lines.append(_CalcConfig('+', name, config_plus)) + if config_minus: + lines.append(_CalcConfig('-', name, config_minus)) + if config_change: + lines.append(_CalcConfig('c', name, config_change)) + + def _OutputConfigInfo(lines): + for line in lines: + if not line: + continue + if line[0] == '+': + col = self.col.GREEN + elif line[0] == '-': + col = self.col.RED + elif line[0] == 'c': + col = self.col.YELLOW + Print(' ' + line, newline=True, colour=col) + + def _OutputErrLines(err_lines, colour): + """Output the line of error/warning lines, if not empty + + Also increments self._error_lines if err_lines not empty + + Args: + err_lines: List of ErrLine objects, each an error or warning + line, possibly including a list of boards with that + error/warning + colour: Colour to use for output + """ + if err_lines: + out_list = [] + for line in err_lines: + boards = '' + names = [board.target for board in line.boards] + board_str = ' '.join(names) if names else '' + if board_str: + out = self.col.Color(colour, line.char + '(') + out += self.col.Color(self.col.MAGENTA, board_str, + bright=False) + out += self.col.Color(colour, ') %s' % line.errline) + else: + out = self.col.Color(colour, line.char + line.errline) + out_list.append(out) + Print('\n'.join(out_list)) + self._error_lines += 1 + + + ok_boards = [] # List of boards fixed since last commit + warn_boards = [] # List of boards with warnings since last commit + err_boards = [] # List of new broken boards since last commit + new_boards = [] # List of boards that didn't exist last time + unknown_boards = [] # List of boards that were not built + + for target in board_dict: + if target not in board_selected: + continue + + # If the board was built last time, add its outcome to a list + if target in self._base_board_dict: + base_outcome = self._base_board_dict[target].rc + outcome = board_dict[target] + if outcome.rc == OUTCOME_UNKNOWN: + unknown_boards.append(target) + elif outcome.rc < base_outcome: + if outcome.rc == OUTCOME_WARNING: + warn_boards.append(target) + else: + ok_boards.append(target) + elif outcome.rc > base_outcome: + if outcome.rc == OUTCOME_WARNING: + warn_boards.append(target) + else: + err_boards.append(target) + else: + new_boards.append(target) + + # Get a list of errors and warnings that have appeared, and disappeared + better_err, worse_err = _CalcErrorDelta(self._base_err_lines, + self._base_err_line_boards, err_lines, err_line_boards, '') + better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, + self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') + + # Display results by arch + if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, + worse_err, better_err, worse_warn, better_warn)): + arch_list = {} + self.AddOutcome(board_selected, arch_list, ok_boards, '', + self.col.GREEN) + self.AddOutcome(board_selected, arch_list, warn_boards, 'w+', + self.col.YELLOW) + self.AddOutcome(board_selected, arch_list, err_boards, '+', + self.col.RED) + self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE) + if self._show_unknown: + self.AddOutcome(board_selected, arch_list, unknown_boards, '?', + self.col.MAGENTA) + for arch, target_list in arch_list.items(): + Print('%10s: %s' % (arch, target_list)) + self._error_lines += 1 + _OutputErrLines(better_err, colour=self.col.GREEN) + _OutputErrLines(worse_err, colour=self.col.RED) + _OutputErrLines(better_warn, colour=self.col.CYAN) + _OutputErrLines(worse_warn, colour=self.col.YELLOW) + + if show_sizes: + self.PrintSizeSummary(board_selected, board_dict, show_detail, + show_bloat) + + if show_environment and self._base_environment: + lines = [] + + for target in board_dict: + if target not in board_selected: + continue + + tbase = self._base_environment[target] + tenvironment = environment[target] + environment_plus = {} + environment_minus = {} + environment_change = {} + base = tbase.environment + for key, value in tenvironment.environment.items(): + if key not in base: + environment_plus[key] = value + for key, value in base.items(): + if key not in tenvironment.environment: + environment_minus[key] = value + for key, value in base.items(): + new_value = tenvironment.environment.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + environment_change[key] = desc + + _AddConfig(lines, target, environment_plus, environment_minus, + environment_change) + + _OutputConfigInfo(lines) + + if show_config and self._base_config: + summary = {} + arch_config_plus = {} + arch_config_minus = {} + arch_config_change = {} + arch_list = [] + + for target in board_dict: + if target not in board_selected: + continue + arch = board_selected[target].arch + if arch not in arch_list: + arch_list.append(arch) + + for arch in arch_list: + arch_config_plus[arch] = {} + arch_config_minus[arch] = {} + arch_config_change[arch] = {} + for name in self.config_filenames: + arch_config_plus[arch][name] = {} + arch_config_minus[arch][name] = {} + arch_config_change[arch][name] = {} + + for target in board_dict: + if target not in board_selected: + continue + + arch = board_selected[target].arch + + all_config_plus = {} + all_config_minus = {} + all_config_change = {} + tbase = self._base_config[target] + tconfig = config[target] + lines = [] + for name in self.config_filenames: + if not tconfig.config[name]: + continue + config_plus = {} + config_minus = {} + config_change = {} + base = tbase.config[name] + for key, value in tconfig.config[name].items(): + if key not in base: + config_plus[key] = value + all_config_plus[key] = value + for key, value in base.items(): + if key not in tconfig.config[name]: + config_minus[key] = value + all_config_minus[key] = value + for key, value in base.items(): + new_value = tconfig.config.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + config_change[key] = desc + all_config_change[key] = desc + + arch_config_plus[arch][name].update(config_plus) + arch_config_minus[arch][name].update(config_minus) + arch_config_change[arch][name].update(config_change) + + _AddConfig(lines, name, config_plus, config_minus, + config_change) + _AddConfig(lines, 'all', all_config_plus, all_config_minus, + all_config_change) + summary[target] = '\n'.join(lines) + + lines_by_target = {} + for target, lines in summary.items(): + if lines in lines_by_target: + lines_by_target[lines].append(target) + else: + lines_by_target[lines] = [target] + + for arch in arch_list: + lines = [] + all_plus = {} + all_minus = {} + all_change = {} + for name in self.config_filenames: + all_plus.update(arch_config_plus[arch][name]) + all_minus.update(arch_config_minus[arch][name]) + all_change.update(arch_config_change[arch][name]) + _AddConfig(lines, name, arch_config_plus[arch][name], + arch_config_minus[arch][name], + arch_config_change[arch][name]) + _AddConfig(lines, 'all', all_plus, all_minus, all_change) + #arch_summary[target] = '\n'.join(lines) + if lines: + Print('%s:' % arch) + _OutputConfigInfo(lines) + + for lines, targets in lines_by_target.items(): + if not lines: + continue + Print('%s :' % ' '.join(sorted(targets))) + _OutputConfigInfo(lines.split('\n')) + + + # Save our updated information for the next call to this function + self._base_board_dict = board_dict + self._base_err_lines = err_lines + self._base_warn_lines = warn_lines + self._base_err_line_boards = err_line_boards + self._base_warn_line_boards = warn_line_boards + self._base_config = config + self._base_environment = environment + + # Get a list of boards that did not get built, if needed + not_built = [] + for board in board_selected: + if not board in board_dict: + not_built.append(board) + if not_built: + Print("Boards not built (%d): %s" % (len(not_built), + ', '.join(not_built))) + + def ProduceResultSummary(self, commit_upto, commits, board_selected): + (board_dict, err_lines, err_line_boards, warn_lines, + warn_line_boards, config, environment) = self.GetResultSummary( + board_selected, commit_upto, + read_func_sizes=self._show_bloat, + read_config=self._show_config, + read_environment=self._show_environment) + if commits: + msg = '%02d: %s' % (commit_upto + 1, + commits[commit_upto].subject) + Print(msg, colour=self.col.BLUE) + self.PrintResultSummary(board_selected, board_dict, + err_lines if self._show_errors else [], err_line_boards, + warn_lines if self._show_errors else [], warn_line_boards, + config, environment, self._show_sizes, self._show_detail, + self._show_bloat, self._show_config, self._show_environment) + + def ShowSummary(self, commits, board_selected): + """Show a build summary for U-Boot for a given board list. + + Reset the result summary, then repeatedly call GetResultSummary on + each commit's results, then display the differences we see. + + Args: + commit: Commit objects to summarise + board_selected: Dict containing boards to summarise + """ + self.commit_count = len(commits) if commits else 1 + self.commits = commits + self.ResetResultSummary(board_selected) + self._error_lines = 0 + + for commit_upto in range(0, self.commit_count, self._step): + self.ProduceResultSummary(commit_upto, commits, board_selected) + if not self._error_lines: + Print('(no errors to report)', colour=self.col.GREEN) + + + def SetupBuild(self, board_selected, commits): + """Set up ready to start a build. + + Args: + board_selected: Selected boards to build + commits: Selected commits to build + """ + # First work out how many commits we will build + count = (self.commit_count + self._step - 1) // self._step + self.count = len(board_selected) * count + self.upto = self.warned = self.fail = 0 + self._timestamps = collections.deque() + + def GetThreadDir(self, thread_num): + """Get the directory path to the working dir for a thread. + + Args: + thread_num: Number of thread to check (-1 for main process, which + is treated as 0) + """ + if self.work_in_output: + return self._working_dir + return os.path.join(self._working_dir, '%02d' % max(thread_num, 0)) + + def _PrepareThread(self, thread_num, setup_git): + """Prepare the working directory for a thread. + + This clones or fetches the repo into the thread's work directory. + Optionally, it can create a linked working tree of the repo in the + thread's work directory instead. + + Args: + thread_num: Thread number (0, 1, ...) + setup_git: + 'clone' to set up a git clone + 'worktree' to set up a git worktree + """ + thread_dir = self.GetThreadDir(thread_num) + builderthread.Mkdir(thread_dir) + git_dir = os.path.join(thread_dir, '.git') + + # Create a worktree or a git repo clone for this thread if it + # doesn't already exist + if setup_git and self.git_dir: + src_dir = os.path.abspath(self.git_dir) + if os.path.isdir(git_dir): + # This is a clone of the src_dir repo, we can keep using + # it but need to fetch from src_dir. + Print('\rFetching repo for thread %d' % thread_num, + newline=False) + gitutil.Fetch(git_dir, thread_dir) + terminal.PrintClear() + elif os.path.isfile(git_dir): + # This is a worktree of the src_dir repo, we don't need to + # create it again or update it in any way. + pass + elif os.path.exists(git_dir): + # Don't know what could trigger this, but we probably + # can't create a git worktree/clone here. + raise ValueError('Git dir %s exists, but is not a file ' + 'or a directory.' % git_dir) + elif setup_git == 'worktree': + Print('\rChecking out worktree for thread %d' % thread_num, + newline=False) + gitutil.AddWorktree(src_dir, thread_dir) + terminal.PrintClear() + elif setup_git == 'clone' or setup_git == True: + Print('\rCloning repo for thread %d' % thread_num, + newline=False) + gitutil.Clone(src_dir, thread_dir) + terminal.PrintClear() + else: + raise ValueError("Can't setup git repo with %s." % setup_git) + + def _PrepareWorkingSpace(self, max_threads, setup_git): + """Prepare the working directory for use. + + Set up the git repo for each thread. Creates a linked working tree + if git-worktree is available, or clones the repo if it isn't. + + Args: + max_threads: Maximum number of threads we expect to need. If 0 then + 1 is set up, since the main process still needs somewhere to + work + setup_git: True to set up a git worktree or a git clone + """ + builderthread.Mkdir(self._working_dir) + if setup_git and self.git_dir: + src_dir = os.path.abspath(self.git_dir) + if gitutil.CheckWorktreeIsAvailable(src_dir): + setup_git = 'worktree' + # If we previously added a worktree but the directory for it + # got deleted, we need to prune its files from the repo so + # that we can check out another in its place. + gitutil.PruneWorktrees(src_dir) + else: + setup_git = 'clone' + + # Always do at least one thread + for thread in range(max(max_threads, 1)): + self._PrepareThread(thread, setup_git) + + def _GetOutputSpaceRemovals(self): + """Get the output directories ready to receive files. + + Figure out what needs to be deleted in the output directory before it + can be used. We only delete old buildman directories which have the + expected name pattern. See _GetOutputDir(). + + Returns: + List of full paths of directories to remove + """ + if not self.commits: + return + dir_list = [] + for commit_upto in range(self.commit_count): + dir_list.append(self._GetOutputDir(commit_upto)) + + to_remove = [] + for dirname in glob.glob(os.path.join(self.base_dir, '*')): + if dirname not in dir_list: + leaf = dirname[len(self.base_dir) + 1:] + m = re.match('[0-9]+_g[0-9a-f]+_.*', leaf) + if m: + to_remove.append(dirname) + return to_remove + + def _PrepareOutputSpace(self): + """Get the output directories ready to receive files. + + We delete any output directories which look like ones we need to + create. Having left over directories is confusing when the user wants + to check the output manually. + """ + to_remove = self._GetOutputSpaceRemovals() + if to_remove: + Print('Removing %d old build directories...' % len(to_remove), + newline=False) + for dirname in to_remove: + shutil.rmtree(dirname) + terminal.PrintClear() + + def BuildBoards(self, commits, board_selected, keep_outputs, verbose): + """Build all commits for a list of boards + + Args: + commits: List of commits to be build, each a Commit object + boards_selected: Dict of selected boards, key is target name, + value is Board object + keep_outputs: True to save build output files + verbose: Display build results as they are completed + Returns: + Tuple containing: + - number of boards that failed to build + - number of boards that issued warnings + - list of thread exceptions raised + """ + self.commit_count = len(commits) if commits else 1 + self.commits = commits + self._verbose = verbose + + self.ResetResultSummary(board_selected) + builderthread.Mkdir(self.base_dir, parents = True) + self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), + commits is not None) + self._PrepareOutputSpace() + Print('\rStarting build...', newline=False) + self.SetupBuild(board_selected, commits) + self.ProcessResult(None) + self.thread_exceptions = [] + # Create jobs to build all commits for each board + for brd in board_selected.values(): + job = builderthread.BuilderJob() + job.board = brd + job.commits = commits + job.keep_outputs = keep_outputs + job.work_in_output = self.work_in_output + job.step = self._step + if self.num_threads: + self.queue.put(job) + else: + results = self._single_builder.RunJob(job) + + if self.num_threads: + term = threading.Thread(target=self.queue.join) + term.setDaemon(True) + term.start() + while term.is_alive(): + term.join(100) + + # Wait until we have processed all output + self.out_queue.join() + Print() + + msg = 'Completed: %d total built' % self.count + if self.already_done: + msg += ' (%d previously' % self.already_done + if self.already_done != self.count: + msg += ', %d newly' % (self.count - self.already_done) + msg += ')' + duration = datetime.now() - self._start_time + if duration > timedelta(microseconds=1000000): + if duration.microseconds >= 500000: + duration = duration + timedelta(seconds=1) + duration = duration - timedelta(microseconds=duration.microseconds) + rate = float(self.count) / duration.total_seconds() + msg += ', duration %s, rate %1.2f' % (duration, rate) + Print(msg) + if self.thread_exceptions: + Print('Failed: %d thread exceptions' % len(self.thread_exceptions), + colour=self.col.RED) + + return (self.fail, self.warned, self.thread_exceptions) |