diff options
Diffstat (limited to 'external/poky/meta/lib/oeqa/utils/qemurunner.py')
| -rw-r--r-- | external/poky/meta/lib/oeqa/utils/qemurunner.py | 169 |
1 files changed, 121 insertions, 48 deletions
diff --git a/external/poky/meta/lib/oeqa/utils/qemurunner.py b/external/poky/meta/lib/oeqa/utils/qemurunner.py index 49564f9a..519aa9aa 100644 --- a/external/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/external/poky/meta/lib/oeqa/utils/qemurunner.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # This module provides a class for starting qemu images using runqemu. # It's used by testimage.bbclass. @@ -19,6 +21,7 @@ import threading import codecs import logging from oeqa.utils.dump import HostDumper +from collections import defaultdict # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -28,10 +31,12 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm, logger): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, + use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None): # Popen object for runqemu self.runqemu = None + self.runqemu_exited = False # pid of the qemu process that runqemu will start self.qemupid = None # target ip - from the command line or runqemu output @@ -51,14 +56,40 @@ class QemuRunner: self.logged = False self.thread = None self.use_kvm = use_kvm + self.use_ovmf = use_ovmf + self.use_slirp = use_slirp + self.serial_ports = serial_ports self.msg = '' + self.boot_patterns = boot_patterns self.runqemutime = 120 - self.qemu_pidfile = 'pidfile_'+str(os.getpid()) + if not workdir: + workdir = os.getcwd() + self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) + self.monitorpipe = None self.logger = logger + # Enable testing other OS's + # Set commands for target communication, and default to Linux ALWAYS + # Other OS's or baremetal applications need to provide their + # own implementation passing it through QemuRunner's constructor + # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag] + # provided variables, where <flag> is one of the mentioned below. + accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished'] + default_boot_patterns = defaultdict(str) + # Default to the usual paterns used to communicate with the target + default_boot_patterns['search_reached_prompt'] = b' login:' + default_boot_patterns['send_login_user'] = 'root\n' + default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#" + default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#" + + # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n" + for pattern in accepted_patterns: + if not self.boot_patterns[pattern]: + self.boot_patterns[pattern] = default_boot_patterns[pattern] + def create_socket(self): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -93,11 +124,10 @@ class QemuRunner: def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: - self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) - self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.error('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout)) self.stop() self._dump_host() - raise SystemExit def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): env = os.environ.copy() @@ -121,7 +151,7 @@ class QemuRunner: env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image if not launch_cmd: - launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams) + launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '') if self.use_kvm: self.logger.debug('Using kvm for runqemu') launch_cmd += ' kvm' @@ -129,13 +159,18 @@ class QemuRunner: self.logger.debug('Not using kvm for runqemu') if not self.display: launch_cmd += ' nographic' - launch_cmd += ' %s %s' % (self.machine, self.rootfs) + if self.use_slirp: + launch_cmd += ' slirp' + if self.use_ovmf: + launch_cmd += ' ovmf' + launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs) return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): try: - self.threadsock, threadport = self.create_socket() + if self.serial_ports >= 2: + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) @@ -149,11 +184,14 @@ class QemuRunner: # and analyze descendents in order to determine it. if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) - self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile) + self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' - launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) + if self.serial_ports >= 2: + launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams) + else: + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) @@ -199,20 +237,34 @@ class QemuRunner: endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): + if self.runqemu_exited: + return False if self.runqemu.returncode: # No point waiting any longer - self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() - self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) + self.logger.warning("Output from runqemu:\n%s" % self.getOutput(output)) self.stop() return False time.sleep(0.5) + if self.runqemu_exited: + return False + if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) + + qemu_pid = None + if os.path.isfile(self.qemu_pidfile): + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + + self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s" + % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid)))) + # Dump all processes to help us to figure out what is going on... - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() @@ -230,21 +282,26 @@ class QemuRunner: self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" % (time.time() - (endtime - self.runqemutime), self.qemupid, time.strftime("%D %H:%M:%S"))) + cmdline = '' if get_ip: - cmdline = '' with open('/proc/%s/cmdline' % self.qemupid) as p: cmdline = p.read() # It is needed to sanitize the data received # because is possible to have control characters cmdline = re_control_char.sub(' ', cmdline) try: - ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - self.ip = ips[0] - self.server_ip = ips[1] + if self.use_slirp: + tcp_ports = cmdline.split("hostfwd=tcp::")[1] + host_port = tcp_ports[:tcp_ports.find('-')] + self.ip = "localhost:%s" % host_port + else: + ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + self.ip = ips[0] + self.server_ip = ips[1] self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output - match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE|re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() @@ -263,14 +320,15 @@ class QemuRunner: self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, self.threadsock, self.logger) - self.thread.start() - if not self.thread.connection_established.wait(self.boottime): - self.logger.error("Didn't receive a console connection from qemu. " - "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, out)) - self.stop_thread() - return False + if self.serial_ports >= 2: + self.thread = LoggingThread(self.log, self.threadsock, self.logger) + self.thread.start() + if not self.thread.connection_established.wait(self.boottime): + self.logger.error("Didn't receive a console connection from qemu. " + "Here is the qemu command line used:\n%s\nand " + "output from runqemu:\n%s" % (cmdline, out)) + self.stop_thread() + return False self.logger.debug("Output from runqemu:\n%s", out) self.logger.debug("Waiting at most %d seconds for login banner (%s)" % @@ -298,8 +356,12 @@ class QemuRunner: data = data + sock.recv(1024) if data: bootlog += data + if self.serial_ports < 2: + # this socket has mixed console/kernel data, log it to logfile + self.log(data) + data = b'' - if b' login:' in bootlog: + if self.boot_patterns['search_reached_prompt'] in bootlog: self.server_socket = qemusock stopread = True reachedlogin = True @@ -317,22 +379,22 @@ class QemuRunner: if not reachedlogin: if time.time() >= endtime: - self.logger.debug("Target didn't reach login banner in %d seconds (%s)" % + self.logger.warning("Target didn't reach login banner in %d seconds (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) tail = lambda l: "\n".join(l.splitlines()[-25:]) bootlog = bootlog.decode("utf-8") # in case bootlog is empty, use tail qemu log store at self.msg lines = tail(bootlog if bootlog else self.msg) - self.logger.debug("Last 25 lines of text:\n%s" % lines) - self.logger.debug("Check full boot log: %s" % self.logfile) + self.logger.warning("Last 25 lines of text:\n%s" % lines) + self.logger.warning("Check full boot log: %s" % self.logfile) self._dump_host() self.stop() return False # If we are not able to login the tests can continue try: - (status, output) = self.run_serial("root\n", raw=True) - if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): + (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True) + if re.search(self.boot_patterns['search_login_succeeded'], output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: @@ -344,11 +406,11 @@ class QemuRunner: else: self.logger.debug("Couldn't configure guest networking") else: - self.logger.debug("Couldn't login into serial console" + self.logger.warning("Couldn't login into serial console" " as root using blank password") - self.logger.debug("The output:\n%s" % output) + self.logger.warning("The output:\n%s" % output) except: - self.logger.debug("Serial console failed while trying to login") + self.logger.warning("Serial console failed while trying to login") return True def stop(self): @@ -373,7 +435,7 @@ class QemuRunner: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) self.runqemu.stdin.close() self.runqemu.stdout.close() - self.runqemu = None + self.runqemu_exited = True if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() @@ -384,7 +446,11 @@ class QemuRunner: self.qemupid = None self.ip = None if os.path.exists(self.qemu_pidfile): - os.remove(self.qemu_pidfile) + try: + os.remove(self.qemu_pidfile) + except FileNotFoundError as e: + # We raced, ignore + pass if self.monitorpipe: self.monitorpipe.close() @@ -402,7 +468,7 @@ class QemuRunner: self.thread.join() def restart(self, qemuparams = None): - self.logger.debug("Restarting qemu process") + self.logger.warning("Restarting qemu process") if self.runqemu.poll() is None: self.stop() if self.start(qemuparams): @@ -410,16 +476,23 @@ class QemuRunner: return False def is_alive(self): - if not self.runqemu or self.runqemu.poll() is not None: + if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited: return False if os.path.isfile(self.qemu_pidfile): - f = open(self.qemu_pidfile, 'r') - qemu_pid = f.read() - f.close() - qemupid = int(qemu_pid) - if os.path.exists("/proc/" + str(qemupid)): - self.qemupid = qemupid - return True + # when handling pidfile, qemu creates the file, stat it, lock it and then write to it + # so it's possible that the file has been created but the content is empty + pidfile_timeout = time.time() + 3 + while time.time() < pidfile_timeout: + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + # file created but not yet written contents + if not qemu_pid: + time.sleep(0.5) + continue + else: + if os.path.exists("/proc/" + qemu_pid): + self.qemupid = int(qemu_pid) + return True return False def run_serial(self, command, raw=False, timeout=60): @@ -446,7 +519,7 @@ class QemuRunner: if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + if re.search(self.boot_patterns['search_cmd_finished'], data): break else: raise Exception("No data on serial console socket") |