diff options
Diffstat (limited to 'roms/u-boot/tools/buildman/toolchain.py')
| -rw-r--r-- | roms/u-boot/tools/buildman/toolchain.py | 645 |
1 files changed, 645 insertions, 0 deletions
diff --git a/roms/u-boot/tools/buildman/toolchain.py b/roms/u-boot/tools/buildman/toolchain.py new file mode 100644 index 000000000..fd137f730 --- /dev/null +++ b/roms/u-boot/tools/buildman/toolchain.py @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. +# + +import re +import glob +from html.parser import HTMLParser +import os +import sys +import tempfile +import urllib.request, urllib.error, urllib.parse + +from buildman import bsettings +from patman import command +from patman import terminal +from patman import tools + +(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, + PRIORITY_CALC) = list(range(4)) + +(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) + +# Simple class to collect links from a page +class MyHTMLParser(HTMLParser): + def __init__(self, arch): + """Create a new parser + + After the parser runs, self.links will be set to a list of the links + to .xz archives found in the page, and self.arch_link will be set to + the one for the given architecture (or None if not found). + + Args: + arch: Architecture to search for + """ + HTMLParser.__init__(self) + self.arch_link = None + self.links = [] + self.re_arch = re.compile('[-_]%s-' % arch) + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for tag, value in attrs: + if tag == 'href': + if value and value.endswith('.xz'): + self.links.append(value) + if self.re_arch.search(value): + self.arch_link = value + + +class Toolchain: + """A single toolchain + + Public members: + gcc: Full path to C compiler + path: Directory path containing C compiler + cross: Cross compile string, e.g. 'arm-linux-' + arch: Architecture of toolchain as determined from the first + component of the filename. E.g. arm-linux-gcc becomes arm + priority: Toolchain priority (0=highest, 20=lowest) + override_toolchain: Toolchain to use for sandbox, overriding the normal + one + """ + def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, + arch=None, override_toolchain=None): + """Create a new toolchain object. + + Args: + fname: Filename of the gcc component + test: True to run the toolchain to test it + verbose: True to print out the information + priority: Priority to use for this toolchain, or PRIORITY_CALC to + calculate it + """ + self.gcc = fname + self.path = os.path.dirname(fname) + self.override_toolchain = override_toolchain + + # Find the CROSS_COMPILE prefix to use for U-Boot. For example, + # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. + basename = os.path.basename(fname) + pos = basename.rfind('-') + self.cross = basename[:pos + 1] if pos != -1 else '' + + # The architecture is the first part of the name + pos = self.cross.find('-') + if arch: + self.arch = arch + else: + self.arch = self.cross[:pos] if pos != -1 else 'sandbox' + if self.arch == 'sandbox' and override_toolchain: + self.gcc = override_toolchain + + env = self.MakeEnvironment(False) + + # As a basic sanity check, run the C compiler with --version + cmd = [fname, '--version'] + if priority == PRIORITY_CALC: + self.priority = self.GetPriority(fname) + else: + self.priority = priority + if test: + result = command.RunPipe([cmd], capture=True, env=env, + raise_on_error=False) + self.ok = result.return_code == 0 + if verbose: + print('Tool chain test: ', end=' ') + if self.ok: + print("OK, arch='%s', priority %d" % (self.arch, + self.priority)) + else: + print('BAD') + print('Command: ', cmd) + print(result.stdout) + print(result.stderr) + else: + self.ok = True + + def GetPriority(self, fname): + """Return the priority of the toolchain. + + Toolchains are ranked according to their suitability by their + filename prefix. + + Args: + fname: Filename of toolchain + Returns: + Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. + """ + priority_list = ['-elf', '-unknown-linux-gnu', '-linux', + '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', + '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', + '-linux-gnueabihf', '-le-linux', '-uclinux'] + for prio in range(len(priority_list)): + if priority_list[prio] in fname: + return PRIORITY_CALC + prio + return PRIORITY_CALC + prio + + def GetWrapper(self, show_warning=True): + """Get toolchain wrapper from the setting file. + """ + value = '' + for name, value in bsettings.GetItems('toolchain-wrapper'): + if not value: + print("Warning: Wrapper not found") + if value: + value = value + ' ' + + return value + + def GetEnvArgs(self, which): + """Get an environment variable/args value based on the the toolchain + + Args: + which: VAR_... value to get + + Returns: + Value of that environment variable or arguments + """ + wrapper = self.GetWrapper() + if which == VAR_CROSS_COMPILE: + return wrapper + os.path.join(self.path, self.cross) + elif which == VAR_PATH: + return self.path + elif which == VAR_ARCH: + return self.arch + elif which == VAR_MAKE_ARGS: + args = self.MakeArgs() + if args: + return ' '.join(args) + return '' + else: + raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) + + def MakeEnvironment(self, full_path): + """Returns an environment for using the toolchain. + + Thie takes the current environment and adds CROSS_COMPILE so that + the tool chain will operate correctly. This also disables localized + output and possibly unicode encoded output of all build tools by + adding LC_ALL=C. + + Note that os.environb is used to obtain the environment, since in some + cases the environment many contain non-ASCII characters and we see + errors like: + + UnicodeEncodeError: 'utf-8' codec can't encode characters in position + 569-570: surrogates not allowed + + Args: + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + Returns: + Dict containing the (bytes) environment to use. This is based on the + current environment, with changes as needed to CROSS_COMPILE, PATH + and LC_ALL. + """ + env = dict(os.environb) + wrapper = self.GetWrapper() + + if self.override_toolchain: + # We'll use MakeArgs() to provide this + pass + elif full_path: + env[b'CROSS_COMPILE'] = tools.ToBytes( + wrapper + os.path.join(self.path, self.cross)) + else: + env[b'CROSS_COMPILE'] = tools.ToBytes(wrapper + self.cross) + env[b'PATH'] = tools.ToBytes(self.path) + b':' + env[b'PATH'] + + env[b'LC_ALL'] = b'C' + + return env + + def MakeArgs(self): + """Create the 'make' arguments for a toolchain + + This is only used when the toolchain is being overridden. Since the + U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the + environment (and MakeEnvironment()) to override these values. This + function returns the arguments to accomplish this. + + Returns: + List of arguments to pass to 'make' + """ + if self.override_toolchain: + return ['HOSTCC=%s' % self.override_toolchain, + 'CC=%s' % self.override_toolchain] + return [] + + +class Toolchains: + """Manage a list of toolchains for building U-Boot + + We select one toolchain for each architecture type + + Public members: + toolchains: Dict of Toolchain objects, keyed by architecture name + prefixes: Dict of prefixes to check, keyed by architecture. This can + be a full path and toolchain prefix, for example + {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of + something on the search path, for example + {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. + paths: List of paths to check for toolchains (may contain wildcards) + """ + + def __init__(self, override_toolchain=None): + self.toolchains = {} + self.prefixes = {} + self.paths = [] + self.override_toolchain = override_toolchain + self._make_flags = dict(bsettings.GetItems('make-flags')) + + def GetPathList(self, show_warning=True): + """Get a list of available toolchain paths + + Args: + show_warning: True to show a warning if there are no tool chains. + + Returns: + List of strings, each a path to a toolchain mentioned in the + [toolchain] section of the settings file. + """ + toolchains = bsettings.GetItems('toolchain') + if show_warning and not toolchains: + print(("Warning: No tool chains. Please run 'buildman " + "--fetch-arch all' to download all available toolchains, or " + "add a [toolchain] section to your buildman config file " + "%s. See README for details" % + bsettings.config_fname)) + + paths = [] + for name, value in toolchains: + if '*' in value: + paths += glob.glob(value) + else: + paths.append(value) + return paths + + def GetSettings(self, show_warning=True): + """Get toolchain settings from the settings file. + + Args: + show_warning: True to show a warning if there are no tool chains. + """ + self.prefixes = bsettings.GetItems('toolchain-prefix') + self.paths += self.GetPathList(show_warning) + + def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, + arch=None): + """Add a toolchain to our list + + We select the given toolchain as our preferred one for its + architecture if it is a higher priority than the others. + + Args: + fname: Filename of toolchain's gcc driver + test: True to run the toolchain to test it + priority: Priority to use for this toolchain + arch: Toolchain architecture, or None if not known + """ + toolchain = Toolchain(fname, test, verbose, priority, arch, + self.override_toolchain) + add_it = toolchain.ok + if toolchain.arch in self.toolchains: + add_it = (toolchain.priority < + self.toolchains[toolchain.arch].priority) + if add_it: + self.toolchains[toolchain.arch] = toolchain + elif verbose: + print(("Toolchain '%s' at priority %d will be ignored because " + "another toolchain for arch '%s' has priority %d" % + (toolchain.gcc, toolchain.priority, toolchain.arch, + self.toolchains[toolchain.arch].priority))) + + def ScanPath(self, path, verbose): + """Scan a path for a valid toolchain + + Args: + path: Path to scan + verbose: True to print out progress information + Returns: + Filename of C compiler if found, else None + """ + fnames = [] + for subdir in ['.', 'bin', 'usr/bin']: + dirname = os.path.join(path, subdir) + if verbose: print(" - looking in '%s'" % dirname) + for fname in glob.glob(dirname + '/*gcc'): + if verbose: print(" - found '%s'" % fname) + fnames.append(fname) + return fnames + + def ScanPathEnv(self, fname): + """Scan the PATH environment variable for a given filename. + + Args: + fname: Filename to scan for + Returns: + List of matching pathanames, or [] if none + """ + pathname_list = [] + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + pathname = os.path.join(path, fname) + if os.path.exists(pathname): + pathname_list.append(pathname) + return pathname_list + + def Scan(self, verbose): + """Scan for available toolchains and select the best for each arch. + + We look for all the toolchains we can file, figure out the + architecture for each, and whether it works. Then we select the + highest priority toolchain for each arch. + + Args: + verbose: True to print out progress information + """ + if verbose: print('Scanning for tool chains') + for name, value in self.prefixes: + if verbose: print(" - scanning prefix '%s'" % value) + if os.path.exists(value): + self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) + continue + fname = value + 'gcc' + if os.path.exists(fname): + self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) + continue + fname_list = self.ScanPathEnv(fname) + for f in fname_list: + self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) + if not fname_list: + raise ValueError("No tool chain found for prefix '%s'" % + value) + for path in self.paths: + if verbose: print(" - scanning path '%s'" % path) + fnames = self.ScanPath(path, verbose) + for fname in fnames: + self.Add(fname, True, verbose) + + def List(self): + """List out the selected toolchains for each architecture""" + col = terminal.Color() + print(col.Color(col.BLUE, 'List of available toolchains (%d):' % + len(self.toolchains))) + if len(self.toolchains): + for key, value in sorted(self.toolchains.items()): + print('%-10s: %s' % (key, value.gcc)) + else: + print('None') + + def Select(self, arch): + """Returns the toolchain for a given architecture + + Args: + args: Name of architecture (e.g. 'arm', 'ppc_8xx') + + returns: + toolchain object, or None if none found + """ + for tag, value in bsettings.GetItems('toolchain-alias'): + if arch == tag: + for alias in value.split(): + if alias in self.toolchains: + return self.toolchains[alias] + + if not arch in self.toolchains: + raise ValueError("No tool chain found for arch '%s'" % arch) + return self.toolchains[arch] + + def ResolveReferences(self, var_dict, args): + """Resolve variable references in a string + + This converts ${blah} within the string to the value of blah. + This function works recursively. + + Args: + var_dict: Dictionary containing variables and their values + args: String containing make arguments + Returns: + Resolved string + + >>> bsettings.Setup() + >>> tcs = Toolchains() + >>> tcs.Add('fred', False) + >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ + 'second' : '2nd'} + >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') + 'this=OBLIQUE_set' + >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') + 'this=OBLIQUE_setfi2ndrstnd' + """ + re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') + + while True: + m = re_var.search(args) + if not m: + break + lookup = m.group(0)[2:-1] + value = var_dict.get(lookup, '') + args = args[:m.start(0)] + value + args[m.end(0):] + return args + + def GetMakeArguments(self, board): + """Returns 'make' arguments for a given board + + The flags are in a section called 'make-flags'. Flags are named + after the target they represent, for example snapper9260=TESTING=1 + will pass TESTING=1 to make when building the snapper9260 board. + + References to other boards can be added in the string also. For + example: + + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443 + + This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. + + A special 'target' variable is set to the board target. + + Args: + board: Board object for the board to check. + Returns: + 'make' flags for that board, or '' if none + """ + self._make_flags['target'] = board.target + arg_str = self.ResolveReferences(self._make_flags, + self._make_flags.get(board.target, '')) + args = re.findall("(?:\".*?\"|\S)+", arg_str) + i = 0 + while i < len(args): + args[i] = args[i].replace('"', '') + if not args[i]: + del args[i] + else: + i += 1 + return args + + def LocateArchUrl(self, fetch_arch): + """Find a toolchain available online + + Look in standard places for available toolchains. At present the + only standard place is at kernel.org. + + Args: + arch: Architecture to look for, or 'list' for all + Returns: + If fetch_arch is 'list', a tuple: + Machine architecture (e.g. x86_64) + List of toolchains + else + URL containing this toolchain, if avaialble, else None + """ + arch = command.OutputOneLine('uname', '-m') + if arch == 'aarch64': + arch = 'arm64' + base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' + versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4'] + links = [] + for version in versions: + url = '%s/%s/%s/' % (base, arch, version) + print('Checking: %s' % url) + response = urllib.request.urlopen(url) + html = tools.ToString(response.read()) + parser = MyHTMLParser(fetch_arch) + parser.feed(html) + if fetch_arch == 'list': + links += parser.links + elif parser.arch_link: + return url + parser.arch_link + if fetch_arch == 'list': + return arch, links + return None + + def Download(self, url): + """Download a file to a temporary directory + + Args: + url: URL to download + Returns: + Tuple: + Temporary directory name + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + """ + print('Downloading: %s' % url) + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp('.buildman') + response = urllib.request.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.get('Content-Length')) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print(chr(8) * (len(status) + 1), '\r', end=' ') + break + + done += len(buffer) + fd.write(buffer) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) + status = status + chr(8) * (len(status) + 1) + print(status, end=' ') + sys.stdout.flush() + fd.close() + if done != size: + print('Error, failed to download') + os.remove(fname) + fname = None + return tmpdir, fname + + def Unpack(self, fname, dest): + """Unpack a tar file + + Args: + fname: Filename to unpack + dest: Destination directory + Returns: + Directory name of the first entry in the archive, without the + trailing / + """ + stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) + dirs = stdout.splitlines()[1].split('/')[:2] + return '/'.join(dirs) + + def TestSettingsHasPath(self, path): + """Check if buildman will find this toolchain + + Returns: + True if the path is in settings, False if not + """ + paths = self.GetPathList(False) + return path in paths + + def ListArchs(self): + """List architectures with available toolchains to download""" + host_arch, archives = self.LocateArchUrl('list') + re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') + arch_set = set() + for archive in archives: + # Remove the host architecture from the start + arch = re_arch.match(archive[len(host_arch):]) + if arch: + if arch.group(1) != '2.0' and arch.group(1) != '64': + arch_set.add(arch.group(1)) + return sorted(arch_set) + + def FetchAndInstall(self, arch): + """Fetch and install a new toolchain + + arch: + Architecture to fetch, or 'list' to list + """ + # Fist get the URL for this architecture + col = terminal.Color() + print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) + url = self.LocateArchUrl(arch) + if not url: + print(("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch)) + return 2 + home = os.environ['HOME'] + dest = os.path.join(home, '.buildman-toolchains') + if not os.path.exists(dest): + os.mkdir(dest) + + # Download the tar file for this toolchain and unpack it + tmpdir, tarfile = self.Download(url) + if not tarfile: + return 1 + print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ') + sys.stdout.flush() + path = self.Unpack(tarfile, dest) + os.remove(tarfile) + os.rmdir(tmpdir) + print() + + # Check that the toolchain works + print(col.Color(col.GREEN, 'Testing')) + dirpath = os.path.join(dest, path) + compiler_fname_list = self.ScanPath(dirpath, True) + if not compiler_fname_list: + print('Could not locate C compiler - fetch failed.') + return 1 + if len(compiler_fname_list) != 1: + print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % + ', '.join(compiler_fname_list))) + toolchain = Toolchain(compiler_fname_list[0], True, True) + + # Make sure that it will be found by buildman + if not self.TestSettingsHasPath(dirpath): + print(("Adding 'download' to config file '%s'" % + bsettings.config_fname)) + bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) + return 0 |