diff options
Diffstat (limited to 'external/poky/bitbake/lib/bb/cooker.py')
| -rw-r--r-- | external/poky/bitbake/lib/bb/cooker.py | 234 |
1 files changed, 105 insertions, 129 deletions
diff --git a/external/poky/bitbake/lib/bb/cooker.py b/external/poky/bitbake/lib/bb/cooker.py index 78658a16..d90bd394 100644 --- a/external/poky/bitbake/lib/bb/cooker.py +++ b/external/poky/bitbake/lib/bb/cooker.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2003, 2004 Phil Blundell @@ -9,22 +6,10 @@ # Copyright (C) 2005 ROAD GmbH # Copyright (C) 2006 - 2007 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - import sys, os, glob, os.path, re, time -import atexit import itertools import logging import multiprocessing @@ -32,19 +17,17 @@ import sre_constants import threading from io import StringIO, UnsupportedOperation from contextlib import closing -from functools import wraps from collections import defaultdict, namedtuple import bb, bb.exceptions, bb.command from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build import queue import signal -import subprocess -import errno import prserv.serv import pyinotify import json import pickle import codecs +import hashserv logger = logging.getLogger("BitBake") collectlog = logging.getLogger("BitBake.Collection") @@ -182,8 +165,8 @@ class BBCooker: bb.debug(1, "BBCooker pyinotify1 %s" % time.time()) sys.stdout.flush() - self.configwatcher.bbseen = [] - self.configwatcher.bbwatchedfiles = [] + self.configwatcher.bbseen = set() + self.configwatcher.bbwatchedfiles = set() self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) bb.debug(1, "BBCooker pyinotify2 %s" % time.time()) sys.stdout.flush() @@ -193,8 +176,8 @@ class BBCooker: self.watcher = pyinotify.WatchManager() bb.debug(1, "BBCooker pyinotify3 %s" % time.time()) sys.stdout.flush() - self.watcher.bbseen = [] - self.watcher.bbwatchedfiles = [] + self.watcher.bbseen = set() + self.watcher.bbwatchedfiles = set() self.notifier = pyinotify.Notifier(self.watcher, self.notifications) bb.debug(1, "BBCooker pyinotify complete %s" % time.time()) @@ -206,6 +189,8 @@ class BBCooker: bb.parse.BBHandler.cached_statements = {} self.ui_cmdline = None + self.hashserv = None + self.hashservaddr = None self.initConfigurationData() @@ -289,14 +274,14 @@ class BBCooker: if not watcher: watcher = self.watcher for i in deps: - watcher.bbwatchedfiles.append(i[0]) + watcher.bbwatchedfiles.add(i[0]) if dirs: f = i[0] else: f = os.path.dirname(i[0]) if f in watcher.bbseen: continue - watcher.bbseen.append(f) + watcher.bbseen.add(f) watchtarget = None while True: # We try and add watches for files that don't exist but if they did, would influence @@ -305,7 +290,7 @@ class BBCooker: try: watcher.add_watch(f, self.watchmask, quiet=False) if watchtarget: - watcher.bbwatchedfiles.append(watchtarget) + watcher.bbwatchedfiles.add(watchtarget) break except pyinotify.WatchManagerError as e: if 'ENOENT' in str(e): @@ -313,7 +298,7 @@ class BBCooker: f = os.path.dirname(f) if f in watcher.bbseen: break - watcher.bbseen.append(f) + watcher.bbseen.add(f) continue if 'ENOSPC' in str(e): providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?") @@ -382,12 +367,6 @@ class BBCooker: self.data.setVar('BB_CMDLINE', self.ui_cmdline) - # - # Copy of the data store which has been expanded. - # Used for firing events and accessing variables where expansion needs to be accounted for - # - bb.parse.init_parser(self.data) - if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: self.disableDataTracking() @@ -405,6 +384,22 @@ class BBCooker: except prserv.serv.PRServiceConfigError as e: bb.fatal("Unable to start PR Server, exitting") + if self.data.getVar("BB_HASHSERVE") == "auto": + # Create a new hash server bound to a unix domain socket + if not self.hashserv: + dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" + self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") + self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False) + self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever) + self.hashserv.process.start() + self.data.setVar("BB_HASHSERVE", self.hashservaddr) + self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) + self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr) + for mc in self.databuilder.mcdata: + self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) + + bb.parse.init_parser(self.data) + def enableDataTracking(self): self.configuration.tracking = True if hasattr(self, "data"): @@ -508,6 +503,7 @@ class BBCooker: """ fn = None envdata = None + mc = '' if not pkgs_to_build: pkgs_to_build = [] @@ -516,6 +512,12 @@ class BBCooker: self.enableDataTracking() self.reset() + def mc_base(p): + if p.startswith('mc:'): + s = p.split(':') + if len(s) == 2: + return s[1] + return None if buildfile: # Parse the configuration here. We need to do it explicitly here since @@ -526,18 +528,16 @@ class BBCooker: fn = self.matchFile(fn) fn = bb.cache.realfn2virtual(fn, cls, mc) elif len(pkgs_to_build) == 1: - ignore = self.data.getVar("ASSUME_PROVIDED") or "" - if pkgs_to_build[0] in set(ignore.split()): - bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) + mc = mc_base(pkgs_to_build[0]) + if not mc: + ignore = self.data.getVar("ASSUME_PROVIDED") or "" + if pkgs_to_build[0] in set(ignore.split()): + bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) - taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) + taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) - mc = runlist[0][0] - fn = runlist[0][3] - else: - envdata = self.data - data.expandKeys(envdata) - parse.ast.runAnonFuncs(envdata) + mc = runlist[0][0] + fn = runlist[0][3] if fn: try: @@ -546,6 +546,12 @@ class BBCooker: except Exception as e: parselog.exception("Unable to read %s", fn) raise + else: + if not mc in self.databuilder.mcdata: + bb.fatal('Not multiconfig named "%s" found' % mc) + envdata = self.databuilder.mcdata[mc] + data.expandKeys(envdata) + parse.ast.runAnonFuncs(envdata) # Display history with closing(StringIO()) as env: @@ -585,10 +591,10 @@ class BBCooker: wildcard = False # Wild card expansion: - # Replace string such as "multiconfig:*:bash" - # into "multiconfig:A:bash multiconfig:B:bash bash" + # Replace string such as "mc:*:bash" + # into "mc:A:bash mc:B:bash bash" for k in targetlist: - if k.startswith("multiconfig:"): + if k.startswith("mc:"): if wildcard: bb.fatal('multiconfig conflict') if k.split(":")[1] == "*": @@ -621,7 +627,7 @@ class BBCooker: runlist = [] for k in fulltargetlist: mc = "" - if k.startswith("multiconfig:"): + if k.startswith("mc:"): mc = k.split(":")[1] k = ":".join(k.split(":")[2:]) ktask = task @@ -640,9 +646,13 @@ class BBCooker: runlist.append([mc, k, ktask, fn]) bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) + havemc = False + for mc in self.multiconfigs: + if taskdata[mc].get_mcdepends(): + havemc = True # No need to do check providers if there are no mcdeps or not an mc build - if len(self.multiconfigs) > 1: + if havemc or len(self.multiconfigs) > 1: seen = set() new = True # Make sure we can provide the multiconfig dependency @@ -702,7 +712,7 @@ class BBCooker: @staticmethod def add_mc_prefix(mc, pn): if mc: - return "multiconfig:%s:%s" % (mc, pn) + return "mc:%s:%s" % (mc, pn) return pn def buildDependTree(self, rq, taskdata): @@ -901,6 +911,10 @@ class BBCooker: os.unlink('package-depends.dot') except FileNotFoundError: pass + try: + os.unlink('recipe-depends.dot') + except FileNotFoundError: + pass with open('task-depends.dot', 'w') as f: f.write("digraph depends {\n") @@ -914,27 +928,6 @@ class BBCooker: f.write("}\n") logger.info("Task dependencies saved to 'task-depends.dot'") - with open('recipe-depends.dot', 'w') as f: - f.write("digraph depends {\n") - pndeps = {} - for task in sorted(depgraph["tdepends"]): - (pn, taskname) = task.rsplit(".", 1) - if pn not in pndeps: - pndeps[pn] = set() - for dep in sorted(depgraph["tdepends"][task]): - (deppn, deptaskname) = dep.rsplit(".", 1) - pndeps[pn].add(deppn) - for pn in sorted(pndeps): - fn = depgraph["pn"][pn]["filename"] - version = depgraph["pn"][pn]["version"] - f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn)) - for dep in sorted(pndeps[pn]): - if dep == pn: - continue - f.write('"%s" -> "%s"\n' % (pn, dep)) - f.write("}\n") - logger.info("Flattened recipe dependencies saved to 'recipe-depends.dot'") - def show_appends_with_no_recipes(self): # Determine which bbappends haven't been applied @@ -1030,16 +1023,16 @@ class BBCooker: bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) def findProviders(self, mc=''): - return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) def findBestProvider(self, pn, mc=''): if pn in self.recipecaches[mc].providers: filenames = self.recipecaches[mc].providers[pn] - eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc]) + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc]) filename = eligible[0] return None, None, None, filename elif pn in self.recipecaches[mc].pkg_pn: - return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) else: return None, None, None, None @@ -1211,14 +1204,14 @@ class BBCooker: for c in collection_list: calc_layer_priority(c) regex = self.data.getVar("BBFILE_PATTERN_%s" % c) - if regex == None: + if regex is None: parselog.error("BBFILE_PATTERN_%s not defined" % c) errors = True continue elif regex == "": parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c) + cre = re.compile('^NULL$') errors = False - continue else: try: cre = re.compile(regex) @@ -1317,7 +1310,7 @@ class BBCooker: self.parseConfiguration() # If we are told to do the None task then query the default task - if (task == None): + if task is None: task = self.configuration.cmd if not task.startswith("do_"): task = "do_%s" % task @@ -1461,7 +1454,7 @@ class BBCooker: self.buildSetVars() # If we are told to do the None task then query the default task - if (task == None): + if task is None: task = self.configuration.cmd if not task.startswith("do_"): @@ -1479,7 +1472,7 @@ class BBCooker: ntargets = [] for target in runlist: if target[0]: - ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2])) + ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2])) ntargets.append("%s:%s" % (target[1], target[2])) for mc in self.multiconfigs: @@ -1599,9 +1592,12 @@ class BBCooker: raise NothingToBuild ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() - for pkg in pkgs_to_build: + for pkg in pkgs_to_build.copy(): if pkg in ignore: parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) + if pkg.startswith("multiconfig:"): + pkgs_to_build.remove(pkg) + pkgs_to_build.append(pkg.replace("multiconfig:", "mc:")) if 'world' in pkgs_to_build: pkgs_to_build.remove('world') @@ -1609,7 +1605,7 @@ class BBCooker: bb.providers.buildWorldTargetList(self.recipecaches[mc], task) for t in self.recipecaches[mc].world_target: if mc: - t = "multiconfig:" + mc + ":" + t + t = "mc:" + mc + ":" + t pkgs_to_build.append(t) if 'universe' in pkgs_to_build: @@ -1628,7 +1624,7 @@ class BBCooker: bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task)) continue if mc: - t = "multiconfig:" + mc + ":" + t + t = "mc:" + mc + ":" + t pkgs_to_build.append(t) return pkgs_to_build @@ -1641,9 +1637,11 @@ class BBCooker: def post_serve(self): prserv.serv.auto_shutdown() + if self.hashserv: + self.hashserv.process.terminate() + self.hashserv.process.join() bb.event.fire(CookerExit(), self.data) - def shutdown(self, force = False): if force: self.state = state.forceshutdown @@ -1658,6 +1656,7 @@ class BBCooker: def reset(self): self.initConfigurationData() + self.handlePRServ() def clientComplete(self): """Called when the client is done using the server""" @@ -1666,6 +1665,8 @@ class BBCooker: self.command.reset() self.databuilder.reset() self.data = self.databuilder.data + self.parsecache_valid = False + self.baseconfig_valid = False class CookerExit(bb.event.Event): @@ -1688,7 +1689,7 @@ class CookerCollectFiles(object): def calc_bbfile_priority( self, filename, matched = None ): for _, _, regex, pri in self.bbfile_config_priorities: if regex.match(filename): - if matched != None: + if matched is not None: if not regex in matched: matched.add(regex) return pri @@ -1785,7 +1786,7 @@ class CookerCollectFiles(object): # When constructing an older style single regex, it's possible for BBMASK # to end up beginning with '|', which matches and masks _everything_. if mask.startswith("|"): - collectlog.warn("BBMASK contains regular expression beginning with '|', fixing: %s" % mask) + collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask) mask = mask[1:] try: re.compile(mask) @@ -1891,35 +1892,6 @@ class ParsingFailure(Exception): self.recipe = recipe Exception.__init__(self, realexception, recipe) -class Feeder(multiprocessing.Process): - def __init__(self, jobs, to_parsers, quit): - self.quit = quit - self.jobs = jobs - self.to_parsers = to_parsers - multiprocessing.Process.__init__(self) - - def run(self): - while True: - try: - quit = self.quit.get_nowait() - except queue.Empty: - pass - else: - if quit == 'cancel': - self.to_parsers.cancel_join_thread() - break - - try: - job = self.jobs.pop() - except IndexError: - break - - try: - self.to_parsers.put(job, timeout=0.5) - except queue.Full: - self.jobs.insert(0, job) - continue - class Parser(multiprocessing.Process): def __init__(self, jobs, results, quit, init, profile): self.jobs = jobs @@ -1966,14 +1938,12 @@ class Parser(multiprocessing.Process): result = pending.pop() else: try: - job = self.jobs.get(timeout=0.25) - except queue.Empty: - continue - - if job is None: + job = self.jobs.pop() + except IndexError: break result = self.parse(*job) - + # Clear the siggen cache after parsing to control memory usage, its huge + bb.parse.siggen.postparsing_clean_cache() try: self.results.put(result, timeout=0.25) except queue.Full: @@ -1981,6 +1951,7 @@ class Parser(multiprocessing.Process): def parse(self, filename, appends): try: + origfilter = bb.event.LogHandler.filter # Record the filename we're parsing into any events generated def parse_filter(self, record): record.taskpid = bb.event.worker_pid @@ -2003,6 +1974,8 @@ class Parser(multiprocessing.Process): # a SystemExit event for example. except BaseException as exc: return True, ParsingFailure(exc, filename) + finally: + bb.event.LogHandler.filter = origfilter class CookerParser(object): def __init__(self, cooker, filelist, masked): @@ -2054,14 +2027,15 @@ class CookerParser(object): multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) - self.feeder_quit = multiprocessing.Queue(maxsize=1) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) - self.jobs = multiprocessing.Queue(maxsize=self.num_processes) self.result_queue = multiprocessing.Queue() - self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit) - self.feeder.start() + + def chunkify(lst,n): + return [lst[i::n] for i in range(n)] + self.jobs = chunkify(self.willparse, self.num_processes) + for i in range(0, self.num_processes): - parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) parser.start() self.process_names.append(parser.name) self.processes.append(parser) @@ -2082,17 +2056,20 @@ class CookerParser(object): self.total) bb.event.fire(event, self.cfgdata) - self.feeder_quit.put(None) for process in self.processes: self.parser_quit.put(None) else: - self.feeder_quit.put('cancel') - self.parser_quit.cancel_join_thread() for process in self.processes: self.parser_quit.put(None) - self.jobs.cancel_join_thread() + # Cleanup the queue before call process.join(), otherwise there might be + # deadlocks. + while True: + try: + self.result_queue.get(timeout=0.25) + except queue.Empty: + break for process in self.processes: if force: @@ -2100,7 +2077,6 @@ class CookerParser(object): process.terminate() else: process.join() - self.feeder.join() sync = threading.Thread(target=self.bb_cache.sync) sync.start() |