diff options
Diffstat (limited to 'external/poky/meta/lib/oeqa/utils')
20 files changed, 319 insertions, 91 deletions
diff --git a/external/poky/meta/lib/oeqa/utils/__init__.py b/external/poky/meta/lib/oeqa/utils/__init__.py index d38a3230..70fbe7b5 100644 --- a/external/poky/meta/lib/oeqa/utils/__init__.py +++ b/external/poky/meta/lib/oeqa/utils/__init__.py @@ -1,3 +1,6 @@ +# +# SPDX-License-Identifier: MIT +# # Enable other layers to have modules in the same named directory from pkgutil import extend_path __path__ = extend_path(__path__, __name__) diff --git a/external/poky/meta/lib/oeqa/utils/buildproject.py b/external/poky/meta/lib/oeqa/utils/buildproject.py index 01a803ab..e6d80cc8 100644 --- a/external/poky/meta/lib/oeqa/utils/buildproject.py +++ b/external/poky/meta/lib/oeqa/utils/buildproject.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013-2016 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides a class for automating build tests for projects diff --git a/external/poky/meta/lib/oeqa/utils/commands.py b/external/poky/meta/lib/oeqa/utils/commands.py index 2e6a2289..f7f8c16b 100644 --- a/external/poky/meta/lib/oeqa/utils/commands.py +++ b/external/poky/meta/lib/oeqa/utils/commands.py @@ -1,6 +1,8 @@ +# # Copyright (c) 2013-2014 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # DESCRIPTION # This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest @@ -93,7 +95,9 @@ class Command(object): # reason, the main process will still exit, which will then # kill the write thread. if self.data: - threading.Thread(target=writeThread, daemon=True).start() + thread = threading.Thread(target=writeThread, daemon=True) + thread.start() + self.threads.append(thread) if self.process.stderr: thread = threading.Thread(target=readStderrThread) thread.start() @@ -170,8 +174,11 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, if native_sysroot: extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ (native_sysroot, native_sysroot, native_sysroot) + extra_libpaths = "%s/lib:%s/usr/lib" % \ + (native_sysroot, native_sysroot) nenv = dict(options.get('env', os.environ)) nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') + nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '') options['env'] = nenv cmd = Command(command, timeout=timeout, output_log=output_log, **options) @@ -310,15 +317,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, try: tinfoil.logger.setLevel(logging.WARNING) import oeqa.targetcontrol - tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") - tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") + recipedata = tinfoil.parse_recipe(pn) + recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") + recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") # Tell QemuTarget() whether need find rootfs/kernel or not if launch_cmd: - tinfoil.config_data.setVar("FIND_ROOTFS", '0') + recipedata.setVar("FIND_ROOTFS", '0') else: - tinfoil.config_data.setVar("FIND_ROOTFS", '1') + recipedata.setVar("FIND_ROOTFS", '1') - recipedata = tinfoil.parse_recipe(pn) for key, value in overrides.items(): recipedata.setVar(key, value) @@ -335,8 +342,8 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemu.deploy() try: qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) - except bb.build.FuncFailed: - msg = 'Failed to start QEMU - see the logs in %s' % logdir + except Exception as e: + msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir if os.path.exists(qemu.qemurunnerlog): with open(qemu.qemurunnerlog, 'r') as f: msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read()) @@ -346,10 +353,7 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, finally: targetlogger.removeHandler(handler) - try: - qemu.stop() - except: - pass + qemu.stop() def updateEnv(env_file): """ diff --git a/external/poky/meta/lib/oeqa/utils/decorators.py b/external/poky/meta/lib/oeqa/utils/decorators.py index d8768969..aabf4110 100644 --- a/external/poky/meta/lib/oeqa/utils/decorators.py +++ b/external/poky/meta/lib/oeqa/utils/decorators.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Some custom decorators that can be used by unittests # Most useful is skipUnlessPassed which can be used for diff --git a/external/poky/meta/lib/oeqa/utils/dump.py b/external/poky/meta/lib/oeqa/utils/dump.py index 79c22b75..09a44329 100644 --- a/external/poky/meta/lib/oeqa/utils/dump.py +++ b/external/poky/meta/lib/oeqa/utils/dump.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import os import sys import errno @@ -67,8 +71,11 @@ class HostDumper(BaseDumper): def dump_host(self, dump_dir=""): if dump_dir: self.dump_dir = dump_dir + env = os.environ.copy() + env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin' + env['COLUMNS'] = '9999' for cmd in self.cmds: - result = runCmd(cmd, ignore_status=True) + result = runCmd(cmd, ignore_status=True, env=env) self._write_dump(cmd.split()[0], result.output) class TargetDumper(BaseDumper): diff --git a/external/poky/meta/lib/oeqa/utils/ftools.py b/external/poky/meta/lib/oeqa/utils/ftools.py index a7233d4c..3093419c 100644 --- a/external/poky/meta/lib/oeqa/utils/ftools.py +++ b/external/poky/meta/lib/oeqa/utils/ftools.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import os import re import errno diff --git a/external/poky/meta/lib/oeqa/utils/git.py b/external/poky/meta/lib/oeqa/utils/git.py index 757e3f0c..ea35a766 100644 --- a/external/poky/meta/lib/oeqa/utils/git.py +++ b/external/poky/meta/lib/oeqa/utils/git.py @@ -1,7 +1,7 @@ # # Copyright (C) 2016 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT # """Git repository interactions""" import os diff --git a/external/poky/meta/lib/oeqa/utils/gitarchive.py b/external/poky/meta/lib/oeqa/utils/gitarchive.py index 9520b2e1..6e8040eb 100644 --- a/external/poky/meta/lib/oeqa/utils/gitarchive.py +++ b/external/poky/meta/lib/oeqa/utils/gitarchive.py @@ -4,14 +4,7 @@ # Copyright (c) 2017, Intel Corporation. # Copyright (c) 2019, Linux Foundation # -# This program is free software; you can redistribute it and/or modify it -# under the terms and conditions of the GNU General Public License, -# version 2, as published by the Free Software Foundation. -# -# This program is distributed in the hope it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. +# SPDX-License-Identifier: GPL-2.0-only # import os diff --git a/external/poky/meta/lib/oeqa/utils/httpserver.py b/external/poky/meta/lib/oeqa/utils/httpserver.py index a48d4994..58d3c3b3 100644 --- a/external/poky/meta/lib/oeqa/utils/httpserver.py +++ b/external/poky/meta/lib/oeqa/utils/httpserver.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import http.server import multiprocessing import os @@ -18,10 +22,10 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): class HTTPService(object): - def __init__(self, root_dir, host='', logger=None): + def __init__(self, root_dir, host='', port=0, logger=None): self.root_dir = root_dir self.host = host - self.port = 0 + self.port = port self.logger = logger def start(self): diff --git a/external/poky/meta/lib/oeqa/utils/logparser.py b/external/poky/meta/lib/oeqa/utils/logparser.py index 32fde14a..60e16d50 100644 --- a/external/poky/meta/lib/oeqa/utils/logparser.py +++ b/external/poky/meta/lib/oeqa/utils/logparser.py @@ -1,4 +1,6 @@ -#!/usr/bin/env python +# +# SPDX-License-Identifier: MIT +# import sys import os @@ -13,7 +15,7 @@ class PtestParser(object): def parse(self, logfile): test_regex = {} test_regex['PASSED'] = re.compile(r"^PASS:(.+)") - test_regex['FAILED'] = re.compile(r"^FAIL:(.+)") + test_regex['FAILED'] = re.compile(r"^FAIL:([^(]+)") test_regex['SKIPPED'] = re.compile(r"^SKIP:(.+)") section_regex = {} @@ -23,13 +25,20 @@ class PtestParser(object): section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)") section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest") + # Cache markers so we don't take the re.search() hit all the time. + markers = ("PASS:", "FAIL:", "SKIP:", "BEGIN:", "END:", "DURATION:", "ERROR: Exit", "TIMEOUT:") + def newsection(): - return { 'name': "No-section", 'log': "" } + return { 'name': "No-section", 'log': [] } current_section = newsection() with open(logfile, errors='replace') as f: for line in f: + if not line.startswith(markers): + current_section['log'].append(line) + continue + result = section_regex['begin'].search(line) if result: current_section['name'] = result.group(1) @@ -59,14 +68,19 @@ class PtestParser(object): current_section[t] = result.group(1) continue - current_section['log'] = current_section['log'] + line + current_section['log'].append(line) for t in test_regex: result = test_regex[t].search(line) if result: if current_section['name'] not in self.results: self.results[current_section['name']] = {} - self.results[current_section['name']][result.group(1)] = t + self.results[current_section['name']][result.group(1).strip()] = t + + # Python performance for repeatedly joining long strings is poor, do it all at once at the end. + # For 2.1 million lines in a log this reduces 18 hours to 12s. + for section in self.sections: + self.sections[section]['log'] = "".join(self.sections[section]['log']) return self.results, self.sections @@ -86,3 +100,65 @@ class PtestParser(object): status = self.results[section][test_name] f.write(status + ": " + test_name + "\n") + +# ltp log parsing +class LtpParser(object): + def __init__(self): + self.results = {} + self.section = {'duration': "", 'log': ""} + + def parse(self, logfile): + test_regex = {} + test_regex['PASSED'] = re.compile(r"PASS") + test_regex['FAILED'] = re.compile(r"FAIL") + test_regex['SKIPPED'] = re.compile(r"SKIP") + + with open(logfile, errors='replace') as f: + for line in f: + for t in test_regex: + result = test_regex[t].search(line) + if result: + self.results[line.split()[0].strip()] = t + + for test in self.results: + result = self.results[test] + self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) + + return self.results, self.section + + +# ltp Compliance log parsing +class LtpComplianceParser(object): + def __init__(self): + self.results = {} + self.section = {'duration': "", 'log': ""} + + def parse(self, logfile): + test_regex = {} + test_regex['PASSED'] = re.compile(r"^PASS") + test_regex['FAILED'] = re.compile(r"^FAIL") + test_regex['SKIPPED'] = re.compile(r"(?:UNTESTED)|(?:UNSUPPORTED)") + + section_regex = {} + section_regex['test'] = re.compile(r"^Testing") + + with open(logfile, errors='replace') as f: + for line in f: + result = section_regex['test'].search(line) + if result: + self.name = "" + self.name = line.split()[1].strip() + self.results[self.name] = "PASSED" + failed = 0 + + failed_result = test_regex['FAILED'].search(line) + if failed_result: + failed = line.split()[1].strip() + if int(failed) > 0: + self.results[self.name] = "FAILED" + + for test in self.results: + result = self.results[test] + self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) + + return self.results, self.section diff --git a/external/poky/meta/lib/oeqa/utils/metadata.py b/external/poky/meta/lib/oeqa/utils/metadata.py index b7def772..8013aa68 100644 --- a/external/poky/meta/lib/oeqa/utils/metadata.py +++ b/external/poky/meta/lib/oeqa/utils/metadata.py @@ -1,6 +1,6 @@ # Copyright (C) 2016 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT # # Functions to get metadata from the testing host used # for analytics of test results. @@ -72,8 +72,11 @@ def git_rev_info(path): info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip() except subprocess.CalledProcessError: pass + try: + info['commit_count'] = int(subprocess.check_output(["git", "rev-list", "--count", "HEAD"], cwd=path).decode('utf-8').strip()) + except subprocess.CalledProcessError: + pass return info - try: repo = Repo(path, search_parent_directories=True) except (InvalidGitRepositoryError, NoSuchPathError): diff --git a/external/poky/meta/lib/oeqa/utils/network.py b/external/poky/meta/lib/oeqa/utils/network.py index 2768f6c5..59d01723 100644 --- a/external/poky/meta/lib/oeqa/utils/network.py +++ b/external/poky/meta/lib/oeqa/utils/network.py @@ -1,7 +1,11 @@ +# +# SPDX-License-Identifier: MIT +# + import socket -def get_free_port(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +def get_free_port(udp = False): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM if not udp else socket.SOCK_DGRAM) s.bind(('', 0)) addr = s.getsockname() s.close() diff --git a/external/poky/meta/lib/oeqa/utils/nfs.py b/external/poky/meta/lib/oeqa/utils/nfs.py new file mode 100644 index 00000000..a37686c9 --- /dev/null +++ b/external/poky/meta/lib/oeqa/utils/nfs.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: MIT +import os +import sys +import tempfile +import contextlib +import socket +from oeqa.utils.commands import bitbake, get_bb_var, Command +from oeqa.utils.network import get_free_port + +@contextlib.contextmanager +def unfs_server(directory, logger = None): + unfs_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "unfs3-native") + if not os.path.exists(os.path.join(unfs_sysroot, "usr", "bin", "unfsd")): + # build native tool + bitbake("unfs3-native -c addto_recipe_sysroot") + + exports = None + cmd = None + try: + # create the exports file + with tempfile.NamedTemporaryFile(delete = False) as exports: + exports.write("{0} (rw,no_root_squash,no_all_squash,insecure)\n".format(directory).encode()) + + # find some ports for the server + nfsport, mountport = get_free_port(udp = True), get_free_port(udp = True) + + nenv = dict(os.environ) + nenv['PATH'] = "{0}/sbin:{0}/usr/sbin:{0}/usr/bin:".format(unfs_sysroot) + nenv.get('PATH', '') + cmd = Command(["unfsd", "-d", "-p", "-N", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)], + bg = True, env = nenv, output_log = logger) + cmd.run() + yield nfsport, mountport + finally: + if cmd is not None: + cmd.stop() + if exports is not None: + # clean up exports file + os.unlink(exports.name) + diff --git a/external/poky/meta/lib/oeqa/utils/package_manager.py b/external/poky/meta/lib/oeqa/utils/package_manager.py index 1495f873..2d358f71 100644 --- a/external/poky/meta/lib/oeqa/utils/package_manager.py +++ b/external/poky/meta/lib/oeqa/utils/package_manager.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import os import json import shutil diff --git a/external/poky/meta/lib/oeqa/utils/qemurunner.py b/external/poky/meta/lib/oeqa/utils/qemurunner.py index 49564f9a..519aa9aa 100644 --- a/external/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/external/poky/meta/lib/oeqa/utils/qemurunner.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # This module provides a class for starting qemu images using runqemu. # It's used by testimage.bbclass. @@ -19,6 +21,7 @@ import threading import codecs import logging from oeqa.utils.dump import HostDumper +from collections import defaultdict # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -28,10 +31,12 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm, logger): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, + use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None): # Popen object for runqemu self.runqemu = None + self.runqemu_exited = False # pid of the qemu process that runqemu will start self.qemupid = None # target ip - from the command line or runqemu output @@ -51,14 +56,40 @@ class QemuRunner: self.logged = False self.thread = None self.use_kvm = use_kvm + self.use_ovmf = use_ovmf + self.use_slirp = use_slirp + self.serial_ports = serial_ports self.msg = '' + self.boot_patterns = boot_patterns self.runqemutime = 120 - self.qemu_pidfile = 'pidfile_'+str(os.getpid()) + if not workdir: + workdir = os.getcwd() + self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) + self.monitorpipe = None self.logger = logger + # Enable testing other OS's + # Set commands for target communication, and default to Linux ALWAYS + # Other OS's or baremetal applications need to provide their + # own implementation passing it through QemuRunner's constructor + # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag] + # provided variables, where <flag> is one of the mentioned below. + accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished'] + default_boot_patterns = defaultdict(str) + # Default to the usual paterns used to communicate with the target + default_boot_patterns['search_reached_prompt'] = b' login:' + default_boot_patterns['send_login_user'] = 'root\n' + default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#" + default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#" + + # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n" + for pattern in accepted_patterns: + if not self.boot_patterns[pattern]: + self.boot_patterns[pattern] = default_boot_patterns[pattern] + def create_socket(self): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -93,11 +124,10 @@ class QemuRunner: def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: - self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) - self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.error('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout)) self.stop() self._dump_host() - raise SystemExit def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): env = os.environ.copy() @@ -121,7 +151,7 @@ class QemuRunner: env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image if not launch_cmd: - launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams) + launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '') if self.use_kvm: self.logger.debug('Using kvm for runqemu') launch_cmd += ' kvm' @@ -129,13 +159,18 @@ class QemuRunner: self.logger.debug('Not using kvm for runqemu') if not self.display: launch_cmd += ' nographic' - launch_cmd += ' %s %s' % (self.machine, self.rootfs) + if self.use_slirp: + launch_cmd += ' slirp' + if self.use_ovmf: + launch_cmd += ' ovmf' + launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs) return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): try: - self.threadsock, threadport = self.create_socket() + if self.serial_ports >= 2: + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) @@ -149,11 +184,14 @@ class QemuRunner: # and analyze descendents in order to determine it. if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) - self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile) + self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' - launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) + if self.serial_ports >= 2: + launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams) + else: + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) @@ -199,20 +237,34 @@ class QemuRunner: endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): + if self.runqemu_exited: + return False if self.runqemu.returncode: # No point waiting any longer - self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() - self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) + self.logger.warning("Output from runqemu:\n%s" % self.getOutput(output)) self.stop() return False time.sleep(0.5) + if self.runqemu_exited: + return False + if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) + + qemu_pid = None + if os.path.isfile(self.qemu_pidfile): + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + + self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s" + % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid)))) + # Dump all processes to help us to figure out what is going on... - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() @@ -230,21 +282,26 @@ class QemuRunner: self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" % (time.time() - (endtime - self.runqemutime), self.qemupid, time.strftime("%D %H:%M:%S"))) + cmdline = '' if get_ip: - cmdline = '' with open('/proc/%s/cmdline' % self.qemupid) as p: cmdline = p.read() # It is needed to sanitize the data received # because is possible to have control characters cmdline = re_control_char.sub(' ', cmdline) try: - ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - self.ip = ips[0] - self.server_ip = ips[1] + if self.use_slirp: + tcp_ports = cmdline.split("hostfwd=tcp::")[1] + host_port = tcp_ports[:tcp_ports.find('-')] + self.ip = "localhost:%s" % host_port + else: + ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + self.ip = ips[0] + self.server_ip = ips[1] self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output - match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE|re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() @@ -263,14 +320,15 @@ class QemuRunner: self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, self.threadsock, self.logger) - self.thread.start() - if not self.thread.connection_established.wait(self.boottime): - self.logger.error("Didn't receive a console connection from qemu. " - "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, out)) - self.stop_thread() - return False + if self.serial_ports >= 2: + self.thread = LoggingThread(self.log, self.threadsock, self.logger) + self.thread.start() + if not self.thread.connection_established.wait(self.boottime): + self.logger.error("Didn't receive a console connection from qemu. " + "Here is the qemu command line used:\n%s\nand " + "output from runqemu:\n%s" % (cmdline, out)) + self.stop_thread() + return False self.logger.debug("Output from runqemu:\n%s", out) self.logger.debug("Waiting at most %d seconds for login banner (%s)" % @@ -298,8 +356,12 @@ class QemuRunner: data = data + sock.recv(1024) if data: bootlog += data + if self.serial_ports < 2: + # this socket has mixed console/kernel data, log it to logfile + self.log(data) + data = b'' - if b' login:' in bootlog: + if self.boot_patterns['search_reached_prompt'] in bootlog: self.server_socket = qemusock stopread = True reachedlogin = True @@ -317,22 +379,22 @@ class QemuRunner: if not reachedlogin: if time.time() >= endtime: - self.logger.debug("Target didn't reach login banner in %d seconds (%s)" % + self.logger.warning("Target didn't reach login banner in %d seconds (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) tail = lambda l: "\n".join(l.splitlines()[-25:]) bootlog = bootlog.decode("utf-8") # in case bootlog is empty, use tail qemu log store at self.msg lines = tail(bootlog if bootlog else self.msg) - self.logger.debug("Last 25 lines of text:\n%s" % lines) - self.logger.debug("Check full boot log: %s" % self.logfile) + self.logger.warning("Last 25 lines of text:\n%s" % lines) + self.logger.warning("Check full boot log: %s" % self.logfile) self._dump_host() self.stop() return False # If we are not able to login the tests can continue try: - (status, output) = self.run_serial("root\n", raw=True) - if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): + (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True) + if re.search(self.boot_patterns['search_login_succeeded'], output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: @@ -344,11 +406,11 @@ class QemuRunner: else: self.logger.debug("Couldn't configure guest networking") else: - self.logger.debug("Couldn't login into serial console" + self.logger.warning("Couldn't login into serial console" " as root using blank password") - self.logger.debug("The output:\n%s" % output) + self.logger.warning("The output:\n%s" % output) except: - self.logger.debug("Serial console failed while trying to login") + self.logger.warning("Serial console failed while trying to login") return True def stop(self): @@ -373,7 +435,7 @@ class QemuRunner: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) self.runqemu.stdin.close() self.runqemu.stdout.close() - self.runqemu = None + self.runqemu_exited = True if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() @@ -384,7 +446,11 @@ class QemuRunner: self.qemupid = None self.ip = None if os.path.exists(self.qemu_pidfile): - os.remove(self.qemu_pidfile) + try: + os.remove(self.qemu_pidfile) + except FileNotFoundError as e: + # We raced, ignore + pass if self.monitorpipe: self.monitorpipe.close() @@ -402,7 +468,7 @@ class QemuRunner: self.thread.join() def restart(self, qemuparams = None): - self.logger.debug("Restarting qemu process") + self.logger.warning("Restarting qemu process") if self.runqemu.poll() is None: self.stop() if self.start(qemuparams): @@ -410,16 +476,23 @@ class QemuRunner: return False def is_alive(self): - if not self.runqemu or self.runqemu.poll() is not None: + if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited: return False if os.path.isfile(self.qemu_pidfile): - f = open(self.qemu_pidfile, 'r') - qemu_pid = f.read() - f.close() - qemupid = int(qemu_pid) - if os.path.exists("/proc/" + str(qemupid)): - self.qemupid = qemupid - return True + # when handling pidfile, qemu creates the file, stat it, lock it and then write to it + # so it's possible that the file has been created but the content is empty + pidfile_timeout = time.time() + 3 + while time.time() < pidfile_timeout: + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + # file created but not yet written contents + if not qemu_pid: + time.sleep(0.5) + continue + else: + if os.path.exists("/proc/" + qemu_pid): + self.qemupid = int(qemu_pid) + return True return False def run_serial(self, command, raw=False, timeout=60): @@ -446,7 +519,7 @@ class QemuRunner: if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + if re.search(self.boot_patterns['search_cmd_finished'], data): break else: raise Exception("No data on serial console socket") diff --git a/external/poky/meta/lib/oeqa/utils/qemutinyrunner.py b/external/poky/meta/lib/oeqa/utils/qemutinyrunner.py index 5aa99d06..5c92941c 100644 --- a/external/poky/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/external/poky/meta/lib/oeqa/utils/qemutinyrunner.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2015 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # This module provides a class for starting qemu images of poky tiny. # It's used by testimage.bbclass. @@ -136,7 +138,7 @@ class QemuTinyRunner(QemuRunner): # # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command'], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} diff --git a/external/poky/meta/lib/oeqa/utils/sshcontrol.py b/external/poky/meta/lib/oeqa/utils/sshcontrol.py index d292893c..36c2ecb3 100644 --- a/external/poky/meta/lib/oeqa/utils/sshcontrol.py +++ b/external/poky/meta/lib/oeqa/utils/sshcontrol.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides a class for setting up ssh connections, # running commands and copying files to/from a target. @@ -22,7 +23,7 @@ class SSHProcess(object): "stdin": None, "shell": False, "bufsize": -1, - "preexec_fn": os.setsid, + "start_new_session": True, } self.options = dict(self.defaultopts) self.options.update(options) diff --git a/external/poky/meta/lib/oeqa/utils/subprocesstweak.py b/external/poky/meta/lib/oeqa/utils/subprocesstweak.py index 1f7d11b5..b47975a4 100644 --- a/external/poky/meta/lib/oeqa/utils/subprocesstweak.py +++ b/external/poky/meta/lib/oeqa/utils/subprocesstweak.py @@ -1,3 +1,6 @@ +# +# SPDX-License-Identifier: MIT +# import subprocess class OETestCalledProcessError(subprocess.CalledProcessError): diff --git a/external/poky/meta/lib/oeqa/utils/targetbuild.py b/external/poky/meta/lib/oeqa/utils/targetbuild.py index b8db7b2a..1055810c 100644 --- a/external/poky/meta/lib/oeqa/utils/targetbuild.py +++ b/external/poky/meta/lib/oeqa/utils/targetbuild.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides a class for automating build tests for projects diff --git a/external/poky/meta/lib/oeqa/utils/testexport.py b/external/poky/meta/lib/oeqa/utils/testexport.py index be2a2110..e89d130a 100644 --- a/external/poky/meta/lib/oeqa/utils/testexport.py +++ b/external/poky/meta/lib/oeqa/utils/testexport.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2015 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides functions to help with exporting binaries obtained from built targets |