diff options
Diffstat (limited to 'external/poky/bitbake/lib/toaster/bldcontrol/management/commands')
3 files changed, 360 insertions, 0 deletions
diff --git a/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py diff --git a/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 00000000..14298d9d --- /dev/null +++ b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -0,0 +1,168 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from django.core.management import call_command +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting, Build, Layer + +import os +import traceback +import warnings + + +def DN(path): + if path is None: + return "" + else: + return os.path.dirname(path) + + +class Command(BaseCommand): + args = "" + help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + + def _verify_build_environment(self): + # provide a local build env. This will be extended later to include non local + if BuildEnvironment.objects.count() == 0: + BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL) + + # we make sure we have builddir and sourcedir for all defined build envionments + for be in BuildEnvironment.objects.all(): + be.needs_import = False + def _verify_be(): + is_changed = False + + def _update_sourcedir(): + be.sourcedir = os.environ.get('TOASTER_DIR') + return True + + if len(be.sourcedir) == 0: + is_changed = _update_sourcedir() + + if not be.sourcedir.startswith("/"): + print("\n -- Validation: The layers checkout directory must be set to an absolute path.") + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): + be.builddir = os.environ.get('TOASTER_DIR')+"/build" + return True + + if len(be.builddir) == 0: + is_changed = _update_builddir() + + if not be.builddir.startswith("/"): + print("\n -- Validation: The build directory must to be set to an absolute path.") + is_changed = _update_builddir() + + if is_changed: + print("\nBuild configuration saved") + be.save() + return True + + if be.needs_import: + try: + print("Loading default settings") + call_command("loaddata", "settings") + template_conf = os.environ.get("TEMPLATECONF", "") + custom_xml_only = os.environ.get("CUSTOM_XML_ONLY") + + if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or (not custom_xml_only == None): + # only use the custom settings + pass + elif "poky" in template_conf: + print("Loading poky configuration") + call_command("loaddata", "poky") + else: + print("Loading OE-Core configuration") + call_command("loaddata", "oe-core") + if template_conf: + oe_core_path = os.path.realpath( + template_conf + + "/../") + else: + print("TEMPLATECONF not found. You may have to" + " manually configure layer paths") + oe_core_path = input("Please enter the path of" + " your openembedded-core " + "layer: ") + # Update the layer instances of openemebedded-core + for layer in Layer.objects.filter( + name="openembedded-core", + local_source_dir="OE-CORE-LAYER-DIR"): + layer.local_path = oe_core_path + layer.save() + + # Import the custom fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing custom settings if present") + try: + call_command("loaddata", "custom") + except: + print("NOTE: optional fixture 'custom' not found") + + # we run lsupdates after config update + print("\nFetching information from the layer index, " + "please wait.\nYou can re-update any time later " + "by running bitbake/lib/toaster/manage.py " + "lsupdates\n") + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print("Failure while trying to setup toaster: %s" + % e) + traceback.print_exc() + + return is_changed + + while _verify_be(): + pass + return 0 + + def _verify_default_settings(self): + # verify that default settings are there + if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1: + ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete() + ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') + return 0 + + def _verify_builds_in_progress(self): + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req=b, errtype="toaster", + errmsg= + "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE) + + # also mark "In Progress builds as failures" + from django.utils import timezone + Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now()) + + return 0 + + + + def handle(self, **options): + retval = 0 + retval += self._verify_build_environment() + retval += self._verify_default_settings() + retval += self._verify_builds_in_progress() + + return retval diff --git a/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 00000000..6a55dd46 --- /dev/null +++ b/external/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -0,0 +1,192 @@ +from django.core.management.base import BaseCommand +from django.db import transaction +from django.db.models import Q + +from bldcontrol.bbcontroller import getBuildEnvironmentController +from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.models import BRError, BRVariable + +from orm.models import Build, LogMessage, Target + +import logging +import traceback +import signal +import os + +logger = logging.getLogger("toaster") + + +class Command(BaseCommand): + args = "" + help = "Schedules and executes build requests as possible. "\ + "Does not return (interrupt with Ctrl-C)" + + @transaction.atomic + def _selectBuildEnvironment(self): + bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) + bec.be.lock = BuildEnvironment.LOCK_LOCK + bec.be.save() + return bec + + @transaction.atomic + def _selectBuildRequest(self): + br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() + return br + + def schedule(self): + try: + # select the build environment and the request to build + br = self._selectBuildRequest() + if br: + br.state = BuildRequest.REQ_INPROGRESS + br.save() + else: + return + + try: + bec = self._selectBuildEnvironment() + except IndexError as e: + # we could not find a BEC; postpone the BR + br.state = BuildRequest.REQ_QUEUED + br.save() + logger.debug("runbuilds: No build env (%s)" % e) + return + + logger.info("runbuilds: starting build %s, environment %s" % + (br, bec.be)) + + # let the build request know where it is being executed + br.environment = bec.be + br.save() + + # this triggers an async build + bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), + br.brvariable_set.all(), br.brtarget_set.all(), + "%d:%d" % (br.pk, bec.be.pk)) + + except Exception as e: + logger.error("runbuilds: Error launching build %s" % e) + traceback.print_exc() + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + + BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, + traceback=traceback.format_exc()) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + # Cancel the pending build and report the exception to the UI + log_object = LogMessage.objects.create( + build = br.build, + level = LogMessage.EXCEPTION, + message = errmsg) + log_object.save() + br.build.outcome = Build.FAILED + br.build.save() + + def archive(self): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): + if br.build is None: + br.state = BuildRequest.REQ_FAILED + else: + br.state = BuildRequest.REQ_COMPLETED + br.save() + + def cleanup(self): + from django.utils import timezone + from datetime import timedelta + # environments locked for more than 30 seconds + # they should be unlocked + BuildEnvironment.objects.filter( + Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, + BuildRequest.REQ_COMPLETED, + BuildRequest.REQ_CANCELLING]) & + Q(lock=BuildEnvironment.LOCK_LOCK) & + Q(updated__lt=timezone.now() - timedelta(seconds=30)) + ).update(lock=BuildEnvironment.LOCK_FREE) + + # update all Builds that were in progress and failed to start + for br in BuildRequest.objects.filter( + state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): + # transpose the launch errors in ToasterExceptions + br.build.outcome = Build.FAILED + for brerror in br.brerror_set.all(): + logger.debug("Saving error %s" % brerror) + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + br.build.save() + + # we don't have a true build object here; hence, toasterui + # didn't have a change to release the BE lock + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + # update all BuildRequests without a build created + for br in BuildRequest.objects.filter(build=None): + br.build = Build.objects.create(project=br.project, + completed_on=br.updated, + started_on=br.created) + br.build.outcome = Build.FAILED + try: + br.build.machine = br.brvariable_set.get(name='MACHINE').value + except BRVariable.DoesNotExist: + pass + br.save() + # transpose target information + for brtarget in br.brtarget_set.all(): + Target.objects.create(build=br.build, + target=brtarget.target, + task=brtarget.task) + # transpose the launch errors in ToasterExceptions + for brerror in br.brerror_set.all(): + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + + br.build.save() + + # Make sure the LOCK is removed for builds which have been fully + # cancelled + for br in BuildRequest.objects.filter( + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + def runbuild(self): + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) + + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) + + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) + + def handle(self, **options): + pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), + ".runbuilds.pid") + + with open(pidfile_path, 'w') as pidfile: + pidfile.write("%s" % os.getpid()) + + self.runbuild() + + signal.signal(signal.SIGUSR1, lambda sig, frame: None) + + while True: + signal.pause() + self.runbuild() |