diff options
Diffstat (limited to 'roms/u-boot/tools/buildman')
| -rw-r--r-- | roms/u-boot/tools/buildman/.gitignore | 1 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/README | 1279 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/board.py | 310 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/bsettings.py | 97 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/builder.py | 1744 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/builderthread.py | 562 | ||||
| l--------- | roms/u-boot/tools/buildman/buildman | 1 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/cmdline.py | 128 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/control.py | 385 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/func_test.py | 626 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/kconfiglib.py | 7160 | ||||
| -rwxr-xr-x | roms/u-boot/tools/buildman/main.py | 65 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/test.py | 628 | ||||
| -rw-r--r-- | roms/u-boot/tools/buildman/toolchain.py | 645 |
14 files changed, 13631 insertions, 0 deletions
diff --git a/roms/u-boot/tools/buildman/.gitignore b/roms/u-boot/tools/buildman/.gitignore new file mode 100644 index 000000000..0d20b6487 --- /dev/null +++ b/roms/u-boot/tools/buildman/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/roms/u-boot/tools/buildman/README b/roms/u-boot/tools/buildman/README new file mode 100644 index 000000000..600794790 --- /dev/null +++ b/roms/u-boot/tools/buildman/README @@ -0,0 +1,1279 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2013 The Chromium OS Authors. + +(Please read 'How to change from MAKEALL' if you are used to that tool) + +Quick-start +=========== + +If you just want to quickly set up buildman so you can build something (for +example Raspberry Pi 2): + + cd /path/to/u-boot + PATH=$PATH:`pwd`/tools/buildman + buildman --fetch-arch arm + buildman -k rpi_2 + ls ../current/rpi_2 + # u-boot.bin is the output image + + +What is this? +============= + +This tool handles building U-Boot to check that you have not broken it +with your patch series. It can build each individual commit and report +which boards fail on which commits, and which errors come up. It aims +to make full use of multi-processor machines. + +A key feature of buildman is its output summary, which allows warnings, +errors or image size increases in a particular commit or board to be +quickly identified and the offending commit pinpointed. This can be a big +help for anyone working with >10 patches at a time. + + +Caveats +======= + +Buildman can be stopped and restarted, in which case it will continue +where it left off. This should happen cleanly and without side-effects. +If not, it is a bug, for which a patch would be welcome. + +Buildman gets so tied up in its work that it can ignore the outside world. +You may need to press Ctrl-C several times to quit it. Also it will print +out various exceptions when stopped. You may have to kill it since the +Ctrl-C handling is somewhat broken. + + +Theory of Operation +=================== + +(please read this section in full twice or you will be perpetually confused) + +Buildman is a builder. It is not make, although it runs make. It does not +produce any useful output on the terminal while building, except for +progress information (but see -v below). All the output (errors, warnings and +binaries if you ask for them) is stored in output directories, which you can +look at from a separate 'buildman -s' instance while the build is progressing, +or when it is finished. + +Buildman is designed to build entire git branches, i.e. muliple commits. It +can be run repeatedly on the same branch after making changes to commits on +that branch. In this case it will automatically rebuild commits which have +changed (and remove its old results for that commit). It is possible to build +a branch for one board, then later build it for another board. This adds to +the output, so now you have results for two boards. If you want buildman to +re-build a commit it has already built (e.g. because of a toolchain update), +use the -f flag. + +Buildman produces a concise summary of which boards succeeded and failed. +It shows which commit introduced which board failure using a simple +red/green colour coding (with yellow/cyan for warnings). Full error +information can be requested, in which case it is de-duped and displayed +against the commit that introduced the error. An example workflow is below. + +Buildman stores image size information and can report changes in image size +from commit to commit. An example of this is below. + +Buildman starts multiple threads, and each thread builds for one board at +a time. A thread starts at the first commit, configures the source for your +board and builds it. Then it checks out the next commit and does an +incremental build (i.e. not using 'make xxx_defconfig' unless you use -C). +Eventually the thread reaches the last commit and stops. If a commit causes +an error or warning, buildman will try it again after reconfiguring (but see +-Q). Thus some commits may be built twice, with the first result silently +discarded. Lots of errors and warnings will causes lots of reconfigures and your +build will be very slow. This is because a file that produces just a warning +would not normally be rebuilt in an incremental build. Once a thread finishes +building all the commits for a board, it starts on the commits for another +board. + +Buildman works in an entirely separate place from your U-Boot repository. +It creates a separate working directory for each thread, and puts the +output files in the working directory, organised by commit name and board +name, in a two-level hierarchy (but see -P). + +Buildman is invoked in your U-Boot directory, the one with the .git +directory. It clones this repository into a copy for each thread, and the +threads do not affect the state of your git repository. Any checkouts done +by the thread affect only the working directory for that thread. + +Buildman automatically selects the correct tool chain for each board. You +must supply suitable tool chains (see --fetch-arch), but buildman takes care +of selecting the right one. + +Buildman generally builds a branch (with the -b flag), and in this case +builds the upstream commit as well, for comparison. So even if you have one +commit in your branch, two commits will be built. Put all your commits in a +branch, set the branch's upstream to a valid value, and all will be well. +Otherwise buildman will perform random actions. Use -n to check what the +random actions might be. + +Buildman effectively has two modes: without -s it builds, with -s it +summarises the results of previous (or active) builds. + +If you just want to build the current source tree, leave off the -b flag. +This will display results and errors as they happen. You can still look at +them later using -se. Note that buildman will assume that the source has +changed, and will build all specified boards in this case. + +Buildman is optimised for building many commits at once, for many boards. +On multi-core machines, Buildman is fast because it uses most of the +available CPU power. When it gets to the end, or if you are building just +a few commits or boards, it will be pretty slow. As a tip, if you don't +plan to use your machine for anything else, you can use -T to increase the +number of threads beyond the default. + + +Selecting which boards to build +=============================== + +Buildman lets you build all boards, or a subset. Specify the subset by passing +command-line arguments that list the desired board name, architecture name, +SOC name, or anything else in the boards.cfg file. Multiple arguments are +allowed. Each argument will be interpreted as a regular expression, so +behaviour is a superset of exact or substring matching. Examples are: + +* 'tegra20' All boards with a Tegra20 SoC +* 'tegra' All boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...) +* '^tegra[23]0$' All boards with either Tegra20 or Tegra30 SoC +* 'powerpc' All PowerPC boards + +While the default is to OR the terms together, you can also make use of +the '&' operator to limit the selection: + +* 'freescale & arm sandbox' All Freescale boards with ARM architecture, + plus sandbox + +You can also use -x to specifically exclude some boards. For example: + + buildman arm -x nvidia,freescale,.*ball$ + +means to build all arm boards except nvidia, freescale and anything ending +with 'ball'. + +For building specific boards you can use the --boards (or --bo) option, which +takes a comma-separated list of board target names and be used multiple times +on the command line: + + buildman --boards sandbox,snow --boards + +It is convenient to use the -n option to see what will be built based on +the subset given. Use -v as well to get an actual list of boards. + +Buildman does not store intermediate object files. It optionally copies +the binary output into a directory when a build is successful (-k). Size +information is always recorded. It needs a fair bit of disk space to work, +typically 250MB per thread. + + +Setting up +========== + +1. Get the U-Boot source. You probably already have it, but if not these +steps should get you started with a repo and some commits for testing. + +$ cd /path/to/u-boot +$ git clone git://git.denx.de/u-boot.git . +$ git checkout -b my-branch origin/master +$ # Add some commits to the branch, reading for testing + +2. Create ~/.buildman to tell buildman where to find tool chains (see 'The +.buildman file' later for details). As an example: + +# Buildman settings file + +[toolchain] +root: / +rest: /toolchains/* +eldk: /opt/eldk-4.2 +arm: /opt/linaro/gcc-linaro-arm-linux-gnueabihf-4.8-2013.08_linux +aarch64: /opt/linaro/gcc-linaro-aarch64-none-elf-4.8-2013.10_linux + +[toolchain-alias] +x86: i386 +blackfin: bfin +nds32: nds32le +openrisc: or1k + + +This selects the available toolchain paths. Add the base directory for +each of your toolchains here. Buildman will search inside these directories +and also in any '/usr' and '/usr/bin' subdirectories. + +Make sure the tags (here root: rest: and eldk:) are unique. + +The toolchain-alias section indicates that the i386 toolchain should be used +to build x86 commits. + +Note that you can also specific exactly toolchain prefixes if you like: + +[toolchain-prefix] +arm: /opt/arm-eabi-4.6/bin/arm-eabi- + +or even: + +[toolchain-prefix] +arm: /opt/arm-eabi-4.6/bin/arm-eabi-gcc + +This tells buildman that you want to use this exact toolchain for the arm +architecture. This will override any toolchains found by searching using the +[toolchain] settings. + +Since the toolchain prefix is an explicit request, buildman will report an +error if a toolchain is not found with that prefix. The current PATH will be +searched, so it is possible to use: + +[toolchain-prefix] +arm: arm-none-eabi- + +and buildman will find arm-none-eabi-gcc in /usr/bin if you have it installed. + +[toolchain-wrapper] +wrapper: ccache + +This tells buildman to use a compiler wrapper in front of CROSS_COMPILE. In +this example, ccache. It doesn't affect the toolchain scan. The wrapper is +added when CROSS_COMPILE environtal variable is set. The name in this +section is ignored. If more than one line is provided, only the last one +is taken. + +3. Make sure you have the require Python pre-requisites + +Buildman uses multiprocessing, Queue, shutil, StringIO, ConfigParser and +urllib2. These should normally be available, but if you get an error like +this then you will need to obtain those modules: + + ImportError: No module named multiprocessing + + +4. Check the available toolchains + +Run this check to make sure that you have a toolchain for every architecture. + +$ ./tools/buildman/buildman --list-tool-chains +Scanning for tool chains + - scanning prefix '/opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-' +Tool chain test: OK, arch='x86', priority 1 + - scanning prefix '/opt/arm-eabi-4.6/bin/arm-eabi-' +Tool chain test: OK, arch='arm', priority 1 + - scanning path '/toolchains/gcc-4.9.0-nolibc/i386-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/usr/bin' +Tool chain test: OK, arch='i386', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/aarch64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/usr/bin' +Tool chain test: OK, arch='aarch64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/microblaze-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/usr/bin' +Tool chain test: OK, arch='microblaze', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/mips64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/usr/bin' +Tool chain test: OK, arch='mips64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/usr/bin' +Tool chain test: OK, arch='sparc64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin' + - found '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/usr/bin' +Tool chain test: OK, arch='arm', priority 3 +Toolchain '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' at priority 3 will be ignored because another toolchain for arch 'arm' has priority 1 + - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/usr/bin' +Tool chain test: OK, arch='sparc', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/mips-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/usr/bin' +Tool chain test: OK, arch='mips', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/x86_64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc' + - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/usr/bin' +Tool chain test: OK, arch='x86_64', priority 4 +Tool chain test: OK, arch='x86_64', priority 4 +Toolchain '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/m68k-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/usr/bin' +Tool chain test: OK, arch='m68k', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/powerpc-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/usr/bin' +Tool chain test: OK, arch='powerpc', priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/usr/bin' +Tool chain test: OK, arch='bfin', priority 6 + - scanning path '/toolchains/gcc-4.6.3-nolibc/sparc-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/usr/bin' +Tool chain test: OK, arch='sparc', priority 4 +Toolchain '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'sparc' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/mips-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/usr/bin' +Tool chain test: OK, arch='mips', priority 4 +Toolchain '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'mips' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/m68k-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/usr/bin' +Tool chain test: OK, arch='m68k', priority 4 +Toolchain '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'm68k' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/powerpc-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin/powerpc-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/usr/bin' +Tool chain test: OK, arch='powerpc', priority 4 +Tool chain test: OK, arch='or32', priority 4 + - scanning path '/' + - looking in '/.' + - looking in '/bin' + - looking in '/usr/bin' + - found '/usr/bin/i586-mingw32msvc-gcc' + - found '/usr/bin/c89-gcc' + - found '/usr/bin/x86_64-linux-gnu-gcc' + - found '/usr/bin/gcc' + - found '/usr/bin/c99-gcc' + - found '/usr/bin/arm-linux-gnueabi-gcc' + - found '/usr/bin/aarch64-linux-gnu-gcc' + - found '/usr/bin/winegcc' + - found '/usr/bin/arm-linux-gnueabihf-gcc' +Tool chain test: OK, arch='i586', priority 11 +Tool chain test: OK, arch='c89', priority 11 +Tool chain test: OK, arch='x86_64', priority 4 +Toolchain '/usr/bin/x86_64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 +Tool chain test: OK, arch='sandbox', priority 11 +Tool chain test: OK, arch='c99', priority 11 +Tool chain test: OK, arch='arm', priority 4 +Toolchain '/usr/bin/arm-linux-gnueabi-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 +Tool chain test: OK, arch='aarch64', priority 4 +Toolchain '/usr/bin/aarch64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'aarch64' has priority 4 +Tool chain test: OK, arch='sandbox', priority 11 +Toolchain '/usr/bin/winegcc' at priority 11 will be ignored because another toolchain for arch 'sandbox' has priority 11 +Tool chain test: OK, arch='arm', priority 4 +Toolchain '/usr/bin/arm-linux-gnueabihf-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 +List of available toolchains (34): +aarch64 : /toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc +alpha : /toolchains/gcc-4.9.0-nolibc/alpha-linux/bin/alpha-linux-gcc +am33_2.0 : /toolchains/gcc-4.9.0-nolibc/am33_2.0-linux/bin/am33_2.0-linux-gcc +arm : /opt/arm-eabi-4.6/bin/arm-eabi-gcc +bfin : /toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc +c89 : /usr/bin/c89-gcc +c99 : /usr/bin/c99-gcc +frv : /toolchains/gcc-4.9.0-nolibc/frv-linux/bin/frv-linux-gcc +h8300 : /toolchains/gcc-4.9.0-nolibc/h8300-elf/bin/h8300-elf-gcc +hppa : /toolchains/gcc-4.9.0-nolibc/hppa-linux/bin/hppa-linux-gcc +hppa64 : /toolchains/gcc-4.9.0-nolibc/hppa64-linux/bin/hppa64-linux-gcc +i386 : /toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc +i586 : /usr/bin/i586-mingw32msvc-gcc +ia64 : /toolchains/gcc-4.9.0-nolibc/ia64-linux/bin/ia64-linux-gcc +m32r : /toolchains/gcc-4.9.0-nolibc/m32r-linux/bin/m32r-linux-gcc +m68k : /toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc +microblaze: /toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc +mips : /toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc +mips64 : /toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc +or32 : /toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc +powerpc : /toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +powerpc64 : /toolchains/gcc-4.9.0-nolibc/powerpc64-linux/bin/powerpc64-linux-gcc +ppc64le : /toolchains/gcc-4.9.0-nolibc/ppc64le-linux/bin/ppc64le-linux-gcc +s390x : /toolchains/gcc-4.9.0-nolibc/s390x-linux/bin/s390x-linux-gcc +sandbox : /usr/bin/gcc +sh4 : /toolchains/gcc-4.6.3-nolibc/sh4-linux/bin/sh4-linux-gcc +sparc : /toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc +sparc64 : /toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc +tilegx : /toolchains/gcc-4.6.2-nolibc/tilegx-linux/bin/tilegx-linux-gcc +x86 : /opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-gcc +x86_64 : /toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc + + +You can see that everything is covered, even some strange ones that won't +be used (c88 and c99). This is a feature. + + +5. Install new toolchains if needed + +You can download toolchains and update the [toolchain] section of the +settings file to find them. + +To make this easier, buildman can automatically download and install +toolchains from kernel.org. First list the available architectures: + +$ ./tools/buildman/buildman --fetch-arch list +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.2.4/ +Available architectures: alpha am33_2.0 arm bfin cris crisv32 frv h8300 +hppa hppa64 i386 ia64 m32r m68k mips mips64 or32 powerpc powerpc64 s390x sh4 +sparc sparc64 tilegx x86_64 xtensa + +Then pick one and download it: + +$ ./tools/buildman/buildman --fetch-arch or32 +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Downloading: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1//x86_64-gcc-4.5.1-nolibc_or32-linux.tar.xz +Unpacking to: /home/sjg/.buildman-toolchains +Testing + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/.' + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin' + - found '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc' +Tool chain test: OK + +Or download them all from kernel.org and move them to /toolchains directory, + +$ ./tools/buildman/buildman --fetch-arch all +$ sudo mkdir -p /toolchains +$ sudo mv ~/.buildman-toolchains/*/* /toolchains/ + +For those not available from kernel.org, download from the following links. + +arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases/ + download/arc-2016.09-release/arc_gnu_2016.09_prebuilt_uclibc_le_archs_linux_install.tar.gz +blackfin: http://sourceforge.net/projects/adi-toolchain/files/ + blackfin-toolchain-elf-gcc-4.5-2014R1_45-RC2.x86_64.tar.bz2 +nds32: http://osdk.andestech.com/packages/ + nds32le-linux-glibc-v1.tgz +nios2: http://sourcery.mentor.com/public/gnu_toolchain/nios2-linux-gnu/ + sourceryg++-2015.11-27-nios2-linux-gnu-i686-pc-linux-gnu.tar.bz2 +sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu/ + renesas-4.4-200-sh-linux-gnu-i686-pc-linux-gnu.tar.bz2 + +Note openrisc kernel.org toolchain is out of date. Download the latest one from +http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions - eg: +ftp://ocuser:ocuser@openrisc.opencores.org/toolchain/gcc-or1k-elf-4.8.1-x86.tar.bz2. + +Buildman should now be set up to use your new toolchain. + +At the time of writing, U-Boot has these architectures: + + arc, arm, blackfin, m68k, microblaze, mips, nds32, nios2, openrisc + powerpc, sandbox, sh, sparc, x86 + +Of these, only arc and nds32 are not available at kernel.org.. + + +How to run it +============= + +First do a dry run using the -n flag: (replace <branch> with a real, local +branch with a valid upstream) + +$ ./tools/buildman/buildman -b <branch> -n + +If it can't detect the upstream branch, try checking out the branch, and +doing something like 'git branch --set-upstream-to upstream/master' +or something similar. Buildman will try to guess a suitable upstream branch +if it can't find one (you will see a message like" Guessing upstream as ...). +You can also use the -c option to manually specify the number of commits to +build. + +As an example: + +Dry run, so not doing much. But I would do this: + +Building 18 commits for 1059 boards (4 threads, 1 job per thread) +Build directory: ../lcd9b + 5bb3505 Merge branch 'master' of git://git.denx.de/u-boot-arm + c18f1b4 tegra: Use const for pinmux_config_pingroup/table() + 2f043ae tegra: Add display support to funcmux + e349900 tegra: fdt: Add pwm binding and node + 424a5f0 tegra: fdt: Add LCD definitions for Tegra + 0636ccf tegra: Add support for PWM + a994fe7 tegra: Add SOC support for display/lcd + fcd7350 tegra: Add LCD driver + 4d46e9d tegra: Add LCD support to Nvidia boards + 991bd48 arm: Add control over cachability of memory regions + 54e8019 lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment + d92aff7 lcd: Add support for flushing LCD fb from dcache after update + dbd0677 tegra: Align LCD frame buffer to section boundary + 0cff9b8 tegra: Support control of cache settings for LCD + 9c56900 tegra: fdt: Add LCD definitions for Seaboard + 5cc29db lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console + cac5a23 tegra: Enable display/lcd support on Seaboard + 49ff541 wip + +Total boards to build for each commit: 1059 + +This shows that it will build all 1059 boards, using 4 threads (because +we have a 4-core CPU). Each thread will run with -j1, meaning that each +make job will use a single CPU. The list of commits to be built helps you +confirm that things look about right. Notice that buildman has chosen a +'base' directory for you, immediately above your source tree. + +Buildman works entirely inside the base directory, here ../lcd9b, +creating a working directory for each thread, and creating output +directories for each commit and board. + + +Suggested Workflow +================== + +To run the build for real, take off the -n: + +$ ./tools/buildman/buildman -b <branch> + +Buildman will set up some working directories, and get started. After a +minute or so it will settle down to a steady pace, with a display like this: + +Building 18 commits for 1059 boards (4 threads, 1 job per thread) + 528 36 124 /19062 -18374 1:13:30 : SIMPC8313_SP + +This means that it is building 19062 board/commit combinations. So far it +has managed to successfully build 528. Another 36 have built with warnings, +and 124 more didn't build at all. It has 18374 builds left to complete. +Buildman expects to complete the process in around an hour and a quarter. +Use this time to buy a faster computer. + + +To find out how the build went, ask for a summary with -s. You can do this +either before the build completes (presumably in another terminal) or +afterwards. Let's work through an example of how this is used: + +$ ./tools/buildman/buildman -b lcd9b -s +... +01: Merge branch 'master' of git://git.denx.de/u-boot-arm + powerpc: + galaxy5200_LOWBOOT +02: tegra: Use const for pinmux_config_pingroup/table() +03: tegra: Add display support to funcmux +04: tegra: fdt: Add pwm binding and node +05: tegra: fdt: Add LCD definitions for Tegra +06: tegra: Add support for PWM +07: tegra: Add SOC support for display/lcd +08: tegra: Add LCD driver +09: tegra: Add LCD support to Nvidia boards +10: arm: Add control over cachability of memory regions +11: lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment +12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock +13: tegra: Align LCD frame buffer to section boundary +14: tegra: Support control of cache settings for LCD +15: tegra: fdt: Add LCD definitions for Seaboard +16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console +17: tegra: Enable display/lcd support on Seaboard +18: wip + +This shows which commits have succeeded and which have failed. In this case +the build is still in progress so many boards are not built yet (use -u to +see which ones). But already we can see a few failures. The galaxy5200_LOWBOOT +never builds correctly. This could be a problem with our toolchain, or it +could be a bug in the upstream. The good news is that we probably don't need +to blame our commits. The bad news is that our commits are not tested on that +board. + +Commit 12 broke lubbock. That's what the '+ lubbock', in red, means. The +failure is never fixed by a later commit, or you would see lubbock again, in +green, without the +. + +To see the actual error: + +$ ./tools/buildman/buildman -b <branch> -se +... +12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock ++common/libcommon.o: In function `lcd_sync': ++common/lcd.c:120: undefined reference to `flush_dcache_range' ++arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 ++make: *** [build/u-boot] Error 139 +13: tegra: Align LCD frame buffer to section boundary +14: tegra: Support control of cache settings for LCD +15: tegra: fdt: Add LCD definitions for Seaboard +16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console +-common/lcd.c:120: undefined reference to `flush_dcache_range' ++common/lcd.c:125: undefined reference to `flush_dcache_range' +17: tegra: Enable display/lcd support on Seaboard +18: wip + +So the problem is in lcd.c, due to missing cache operations. This information +should be enough to work out what that commit is doing to break these +boards. (In this case pxa did not have cache operations defined). + +Note that if there were other boards with errors, the above command would +show their errors also. Each line is shown only once. So if lubbock and snow +produce the same error, we just see: + +12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock snow ++common/libcommon.o: In function `lcd_sync': ++common/lcd.c:120: undefined reference to `flush_dcache_range' ++arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 ++make: *** [build/u-boot] Error 139 + +But if you did want to see just the errors for lubbock, use: + +$ ./tools/buildman/buildman -b <branch> -se lubbock + +If you see error lines marked with '-', that means that the errors were fixed +by that commit. Sometimes commits can be in the wrong order, so that a +breakage is introduced for a few commits and fixed by later commits. This +shows up clearly with buildman. You can then reorder the commits and try +again. + +At commit 16, the error moves: you can see that the old error at line 120 +is fixed, but there is a new one at line 126. This is probably only because +we added some code and moved the broken line further down the file. + +As mentioned, if many boards have the same error, then -e will display the +error only once. This makes the output as concise as possible. To see which +boards have each error, use -l. So it is safe to omit the board name - you +will not get lots of repeated output for every board. + +Buildman tries to distinguish warnings from errors, and shows warning lines +separately with a 'w' prefix. Warnings introduced show as yellow. Warnings +fixed show as cyan. + +The full build output in this case is available in: + +../lcd9b/12_of_18_gd92aff7_lcd--Add-support-for/lubbock/ + + done: Indicates the build was done, and holds the return code from make. + This is 0 for a good build, typically 2 for a failure. + + err: Output from stderr, if any. Errors and warnings appear here. + + log: Output from stdout. Normally there isn't any since buildman runs + in silent mode. Use -V to force a verbose build (this passes V=1 + to 'make') + + toolchain: Shows information about the toolchain used for the build. + + sizes: Shows image size information. + +It is possible to get the build binary output there also. Use the -k option +for this. In that case you will also see some output files, like: + + System.map toolchain u-boot u-boot.bin u-boot.map autoconf.mk + (also SPL versions u-boot-spl and u-boot-spl.bin if available) + + +Checking Image Sizes +==================== + +A key requirement for U-Boot is that you keep code/data size to a minimum. +Where a new feature increases this noticeably it should normally be put +behind a CONFIG flag so that boards can leave it disabled and keep the image +size more or less the same with each new release. + +To check the impact of your commits on image size, use -S. For example: + +$ ./tools/buildman/buildman -b us-x86 -sS +Summary of 10 commits for 1066 boards (4 threads, 1 job per thread) +01: MAKEALL: add support for per architecture toolchains +02: x86: Add function to get top of usable ram + x86: (for 1/3 boards) text -272.0 rodata +41.0 +03: x86: Add basic cache operations +04: x86: Permit bootstage and timer data to be used prior to relocation + x86: (for 1/3 boards) data +16.0 +05: x86: Add an __end symbol to signal the end of the U-Boot binary + x86: (for 1/3 boards) text +76.0 +06: x86: Rearrange the output input to remove BSS + x86: (for 1/3 boards) bss -2140.0 +07: x86: Support relocation of FDT on start-up + x86: + coreboot-x86 +08: x86: Add error checking to x86 relocation code +09: x86: Adjust link device tree include file +10: x86: Enable CONFIG_OF_CONTROL on coreboot + + +You can see that image size only changed on x86, which is good because this +series is not supposed to change any other board. From commit 7 onwards the +build fails so we don't get code size numbers. The numbers are fractional +because they are an average of all boards for that architecture. The +intention is to allow you to quickly find image size problems introduced by +your commits. + +Note that the 'text' region and 'rodata' are split out. You should add the +two together to get the total read-only size (reported as the first column +in the output from binutil's 'size' utility). + +A useful option is --step which lets you skip some commits. For example +--step 2 will show the image sizes for only every 2nd commit (so it will +compare the image sizes of the 1st, 3rd, 5th... commits). You can also use +--step 0 which will compare only the first and last commits. This is useful +for an overview of how your entire series affects code size. It will build +only the upstream commit and your final branch commit. + +You can also use -d to see a detailed size breakdown for each board. This +list is sorted in order from largest growth to largest reduction. + +It is even possible to go a little further with the -B option (--bloat). This +shows where U-Boot has bloated, breaking the size change down to the function +level. Example output is below: + +$ ./tools/buildman/buildman -b us-mem4 -sSdB +... +19: Roll crc32 into hash infrastructure + arm: (for 10/10 boards) all -143.4 bss +1.2 data -4.8 rodata -48.2 text -91.6 + paz00 : all +23 bss -4 rodata -29 text +56 + u-boot: add: 1/0, grow: 3/-2 bytes: 168/-104 (64) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_read_file 540 568 +28 + insert_var_value_sub 688 692 +4 + run_list_real 1996 1992 -4 + do_mem_crc 168 68 -100 + trimslice : all -9 bss +16 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + whistler : all -9 bss +16 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + seaboard : all -9 bss -28 rodata -29 text +48 + u-boot: add: 1/0, grow: 3/-2 bytes: 160/-104 (56) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_read_file 548 568 +20 + run_list_real 1996 2000 +4 + do_nandboot 760 756 -4 + do_mem_crc 168 68 -100 + colibri_t20 : all -9 rodata -29 text +20 + u-boot: add: 1/0, grow: 2/-3 bytes: 140/-112 (28) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + read_abs_bbt 204 208 +4 + do_nandboot 760 756 -4 + ext4fs_read_file 576 568 -8 + do_mem_crc 168 68 -100 + ventana : all -37 bss -12 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + harmony : all -37 bss -16 rodata -29 text +8 + u-boot: add: 1/0, grow: 2/-3 bytes: 140/-124 (16) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + nand_write_oob_syndrome 428 432 +4 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + medcom-wide : all -417 bss +28 data -16 rodata -93 text -336 + u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) + function old new delta + crc32_wd_buf - 56 +56 + do_fat_read_at 2872 2904 +32 + hash_algo 16 - -16 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + tec : all -449 bss -4 data -16 rodata -93 text -336 + u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) + function old new delta + crc32_wd_buf - 56 +56 + do_fat_read_at 2872 2904 +32 + hash_algo 16 - -16 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + plutux : all -481 bss +16 data -16 rodata -93 text -388 + u-boot: add: 1/-1, grow: 1/-3 bytes: 68/-408 (-340) + function old new delta + crc32_wd_buf - 56 +56 + do_load_serial_bin 1688 1700 +12 + hash_algo 16 - -16 + do_fat_read_at 2904 2872 -32 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + powerpc: (for 5/5 boards) all +37.4 data -3.2 rodata -41.8 text +82.4 + MPC8610HPCD : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + MPC8641HPCN : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + MPC8641HPCN_36BIT: all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + sbc8641d : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + xpedite517x : all -33 data -16 rodata -93 text +76 + u-boot: add: 1/-1, grow: 0/-1 bytes: 176/-112 (64) + function old new delta + hash_command - 176 +176 + hash_algo 16 - -16 + do_mem_crc 184 88 -96 +... + + +This shows that commit 19 has reduced codesize for arm slightly and increased +it for powerpc. This increase was offset in by reductions in rodata and +data/bss. + +Shown below the summary lines are the sizes for each board. Below each board +are the sizes for each function. This information starts with: + + add - number of functions added / removed + grow - number of functions which grew / shrunk + bytes - number of bytes of code added to / removed from all functions, + plus the total byte change in brackets + +The change seems to be that hash_command() has increased by more than the +do_mem_crc() function has decreased. The function sizes typically add up to +roughly the text area size, but note that every read-only section except +rodata is included in 'text', so the function total does not exactly +correspond. + +It is common when refactoring code for the rodata to decrease as the text size +increases, and vice versa. + + +The .buildman file +================== + +The .buildman file provides information about the available toolchains and +also allows build flags to be passed to 'make'. It consists of several +sections, with the section name in square brackets. Within each section are +a set of (tag, value) pairs. + +'[toolchain]' section + + This lists the available toolchains. The tag here doesn't matter, but + make sure it is unique. The value is the path to the toolchain. Buildman + will look in that path for a file ending in 'gcc'. It will then execute + it to check that it is a C compiler, passing only the --version flag to + it. If the return code is 0, buildman assumes that it is a valid C + compiler. It uses the first part of the name as the architecture and + strips off the last part when setting the CROSS_COMPILE environment + variable (parts are delimited with a hyphen). + + For example powerpc-linux-gcc will be noted as a toolchain for 'powerpc' + and CROSS_COMPILE will be set to powerpc-linux- when using it. + +'[toolchain-alias]' section + + This converts toolchain architecture names to U-Boot names. For example, + if an x86 toolchains is called i386-linux-gcc it will not normally be + used for architecture 'x86'. Adding 'x86: i386 x86_64' to this section + will tell buildman that the i386 and x86_64 toolchains can be used for + the x86 architecture. + +'[make-flags]' section + + U-Boot's build system supports a few flags (such as BUILD_TAG) which + affect the build product. These flags can be specified in the buildman + settings file. They can also be useful when building U-Boot against other + open source software. + + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443 + + This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special + variable ${target} is available to access the target name (snapper9260 + and snapper9g20 in this case). Variables are resolved recursively. Note + that variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) + and underscore (_). + + It is expected that any variables added are dealt with in U-Boot's + config.mk file and documented in the README. + + Note that you can pass ad-hoc options to the build using environment + variables, for example: + + SOME_OPTION=1234 ./tools/buildman/buildman my_board + + +Quick Sanity Check +================== + +If you have made changes and want to do a quick sanity check of the +currently checked-out source, run buildman without the -b flag. This will +build the selected boards and display build status as it runs (i.e. -v is +enabled automatically). Use -e to see errors/warnings as well. + + +Building Ranges +=============== + +You can build a range of commits by specifying a range instead of a branch +when using the -b flag. For example: + + upstream/master..us-buildman + +will build commits in us-buildman that are not in upstream/master. + + +Building Faster +=============== + +By default, buildman doesn't execute 'make mrproper' prior to building the +first commit for each board. This reduces the amount of work 'make' does, and +hence speeds up the build. To force use of 'make mrproper', use -the -m flag. +This flag will slow down any buildman invocation, since it increases the amount +of work done on any build. + +One possible application of buildman is as part of a continual edit, build, +edit, build, ... cycle; repeatedly applying buildman to the same change or +series of changes while making small incremental modifications to the source +each time. This provides quick feedback regarding the correctness of recent +modifications. In this scenario, buildman's default choice of build directory +causes more build work to be performed than strictly necessary. + +By default, each buildman thread uses a single directory for all builds. When a +thread builds multiple boards, the configuration built in this directory will +cycle through various different configurations, one per board built by the +thread. Variations in the configuration will force a rebuild of affected source +files when a thread switches between boards. Ideally, such buildman-induced +rebuilds would not happen, thus allowing the build to operate as efficiently as +the build system and source changes allow. buildman's -P flag may be used to +enable this; -P causes each board to be built in a separate (board-specific) +directory, thus avoiding any buildman-induced configuration changes in any +build directory. + +U-Boot's build system embeds information such as a build timestamp into the +final binary. This information varies each time U-Boot is built. This causes +various files to be rebuilt even if no source changes are made, which in turn +requires that the final U-Boot binary be re-linked. This unnecessary work can +be avoided by turning off the timestamp feature. This can be achieved by +setting the SOURCE_DATE_EPOCH environment variable to 0. + +Combining all of these options together yields the command-line shown below. +This will provide the quickest possible feedback regarding the current content +of the source tree, thus allowing rapid tested evolution of the code. + + SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + + +Checking configuration +====================== + +A common requirement when converting CONFIG options to Kconfig is to check +that the effective configuration has not changed due to the conversion. +Buildman supports this with the -K option, used after a build. This shows +differences in effective configuration between one commit and the next. + +For example: + + $ buildman -b kc4 -sK + ... + 43: Convert CONFIG_SPL_USBETH_SUPPORT to Kconfig + arm: + + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET_SUPPORT=1 + + u-boot-spl.cfg: CONFIG_SPL_MMC_SUPPORT=1 CONFIG_SPL_NAND_SUPPORT=1 + + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC_SUPPORT=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET_SUPPORT=1 + am335x_evm_usbspl : + + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET_SUPPORT=1 + + u-boot-spl.cfg: CONFIG_SPL_MMC_SUPPORT=1 CONFIG_SPL_NAND_SUPPORT=1 + + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC_SUPPORT=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET_SUPPORT=1 + 44: Convert CONFIG_SPL_USB_HOST_SUPPORT to Kconfig + ... + +This shows that commit 44 enabled three new options for the board +am335x_evm_usbspl which were not enabled in commit 43. There is also a +summary for 'arm' showing all the changes detected for that architecture. +In this case there is only one board with changes, so 'arm' output is the +same as 'am335x_evm_usbspl'/ + +The -K option uses the u-boot.cfg, spl/u-boot-spl.cfg and tpl/u-boot-tpl.cfg +files which are produced by a build. If all you want is to check the +configuration you can in fact avoid doing a full build, using -D. This tells +buildman to configuration U-Boot and create the .cfg files, but not actually +build the source. This is 5-10 times faster than doing a full build. + +By default buildman considers the follow two configuration methods +equivalent: + + #define CONFIG_SOME_OPTION + + CONFIG_SOME_OPTION=y + +The former would appear in a header filer and the latter in a defconfig +file. The achieve this, buildman considers 'y' to be '1' in configuration +variables. This avoids lots of useless output when converting a CONFIG +option to Kconfig. To disable this behaviour, use --squash-config-y. + + +Checking the environment +======================== + +When converting CONFIG options which manipulate the default environment, +a common requirement is to check that the default environment has not +changed due to the conversion. Buildman supports this with the -U option, +used after a build. This shows differences in the default environment +between one commit and the next. + +For example: + +$ buildman -b squash brppt1 -sU +boards.cfg is up to date. Nothing to do. +Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) +01: Migrate bootlimit to Kconfig +02: Squashed commit of the following: + c brppt1_mmc: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + c brppt1_spi: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + + brppt1_nand: altbootcmd=run usbscript + - brppt1_nand: altbootcmd=run usbscript +(no errors to report) + +This shows that commit 2 modified the value of 'altbootcmd' for 'brppt1_mmc' +and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a +value for 'altbootcmd', but lost one for ' altbootcmd'. + +The -U option uses the u-boot.env files which are produced by a build. + + +Building with clang +=================== + +To build with clang (sandbox only), use the -O option to override the +toolchain. For example: + + buildman -O clang-7 --board sandbox + + +Doing a simple build +==================== + +In some cases you just want to build a single board and get the full output, use +the -w option, for example: + + buildman -o /tmp/build --board sandbox -w + +This will write the full build into /tmp/build including object files. You must +specify the output directory with -o when using -w. + + +Other options +============= + +Buildman has various other command-line options. Try --help to see them. + +To find out what toolchain prefix buildman will use for a build, use the -A +option. + +To request that compiler warnings be promoted to errors, use -E. This passes the +-Werror flag to the compiler. Note that the build can still produce warnings +with -E, e.g. the migration warnings: + + ===================== WARNING ====================== + This board does not use CONFIG_DM_MMC. Please update + ... + ==================================================== + +When doing builds, Buildman's return code will reflect the overall result: + + 0 (success) No errors or warnings found + 100 Errors found + 101 Warnings found (only if no -W) + +You can use -W to tell Buildman to return 0 (success) instead of 101 when +warnings are found. Note that it can be useful to combine -E and -W. This means +that all compiler warnings will produce failures (code 100) and all other +warnings will produce success (since 101 is changed to 0). + +If there are both warnings and errors, errors win, so buildman returns 100. + +The -y option is provided (for use with -s) to ignore the bountiful device-tree +warnings. Similarly, -Y tells buildman to ignore the migration warnings. + +Sometimes you might get an error in a thread that is not handled by buildman, +perhaps due to a failure of a tool that it calls. You might see the output, but +then buildman hangs. Failing to handle any eventuality is a bug in buildman and +should be reported. But you can use -T0 to disable threading and hopefully +figure out the root cause of the build failure. + +Build summary +============= + +When buildman finishes it shows a summary, something like this: + + Completed: 5 total built, duration 0:00:21, rate 0.24 + +This shows that a total of 5 builds were done across all selected boards, it +took 21 seconds and the builds happened at the rate of 0.24 per second. The +latter number depends on the speed of your machine and the efficiency of the +U-Boot build. + + +How to change from MAKEALL +========================== + +Buildman includes most of the features of MAKEALL and is generally faster +and easier to use. In particular it builds entire branches: if a particular +commit introduces an error in a particular board, buildman can easily show +you this, even if a later commit fixes that error. + +The reasons to deprecate MAKEALL are: +- We don't want to maintain two build systems +- Buildman is typically faster +- Buildman has a lot more features + +But still, many people will be sad to lose MAKEALL. If you are used to +MAKEALL, here are a few pointers. + +First you need to set up your tool chains - see the 'Setting up' section +for details. Once you have your required toolchain(s) detected then you are +ready to go. + +To build the current source tree, run buildman without a -b flag: + + ./tools/buildman/buildman <list of things to build> + +This will build the current source tree for the given boards and display +the results and errors. + +However buildman usually works on entire branches, and for that you must +specify a board flag: + + ./tools/buildman/buildman -b <branch_name> <list of things to build> + +followed by (afterwards, or perhaps concurrently in another terminal): + + ./tools/buildman/buildman -b <branch_name> -s <list of things to build> + +to see the results of the build. Rather than showing you all the output, +buildman just shows a summary, with red indicating that a commit introduced +an error and green indicating that a commit fixed an error. Use the -e +flag to see the full errors and -l to see which boards caused which errors. + +If you really want to see build results as they happen, use -v when doing a +build (and -e to see the errors/warnings too). + +You don't need to stick around on that branch while buildman is running. It +checks out its own copy of the source code, so you can change branches, +add commits, etc. without affecting the build in progress. + +The <list of things to build> can include board names, architectures or the +like. There are no flags to disambiguate since ambiguities are rare. Using +the examples from MAKEALL: + +Examples: + - build all Power Architecture boards: + MAKEALL -a powerpc + MAKEALL --arch powerpc + MAKEALL powerpc + ** buildman -b <branch> powerpc + - build all PowerPC boards manufactured by vendor "esd": + MAKEALL -a powerpc -v esd + ** buildman -b <branch> esd + - build all PowerPC boards manufactured either by "keymile" or "siemens": + MAKEALL -a powerpc -v keymile -v siemens + ** buildman -b <branch> keymile siemens + - build all Freescale boards with MPC83xx CPUs, plus all 4xx boards: + MAKEALL -c mpc83xx -v freescale 4xx + ** buildman -b <branch> mpc83xx freescale 4xx + +Buildman automatically tries to use all the CPUs in your machine. If you +are building a lot of boards it will use one thread for every CPU core +it detects in your machine. This is like MAKEALL's BUILD_NBUILDS option. +You can use the -T flag to change the number of threads. If you are only +building a few boards, buildman will automatically run make with the -j +flag to increase the number of concurrent make tasks. It isn't normally +that helpful to fiddle with this option, but if you use the BUILD_NCPUS +option in MAKEALL then -j is the equivalent in buildman. + +Buildman puts its output in ../<branch_name> by default but you can change +this with the -o option. Buildman normally does out-of-tree builds: use -i +to disable that if you really want to. But be careful that once you have +used -i you pollute buildman's copies of the source tree, and you will need +to remove the build directory (normally ../<branch_name>) to run buildman +in normal mode (without -i). + +Buildman doesn't keep the output result normally, but use the -k option to +do this. + +Please read 'Theory of Operation' a few times as it will make a lot of +things clearer. + +Some options you might like are: + + -B shows which functions are growing/shrinking in which commit - great + for finding code bloat. + -S shows image sizes for each commit (just an overall summary) + -u shows boards that you haven't built yet + --step 0 will build just the upstream commit and the last commit of your + branch. This is often a quick sanity check that your branch doesn't + break anything. But note this does not check bisectability! + + +TODO +==== + +Many improvements have been made over the years. There is still quite a bit of +scope for more though, e.g.: + +- easier access to log files +- 'hunting' for problems, perhaps by building a few boards for each arch, or + checking commits for changed files and building only boards which use those + files +- using the same git repo for all threads instead of cloning it. Currently + it uses about 500MB per thread, so on a 64-thread machine this is 32GB for + the build. + + +Credits +======= + +Thanks to Grant Grundler <grundler@chromium.org> for his ideas for improving +the build speed by building all commits for a board instead of the other +way around. + + +Simon Glass +sjg@chromium.org +Halloween 2012 +Updated 12-12-12 +Updated 23-02-13 +Updated 09-04-20 diff --git a/roms/u-boot/tools/buildman/board.py b/roms/u-boot/tools/buildman/board.py new file mode 100644 index 000000000..447aaabea --- /dev/null +++ b/roms/u-boot/tools/buildman/board.py @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. + +from collections import OrderedDict +import re + +class Expr: + """A single regular expression for matching boards to build""" + + def __init__(self, expr): + """Set up a new Expr object. + + Args: + expr: String cotaining regular expression to store + """ + self._expr = expr + self._re = re.compile(expr) + + def Matches(self, props): + """Check if any of the properties match the regular expression. + + Args: + props: List of properties to check + Returns: + True if any of the properties match the regular expression + """ + for prop in props: + if self._re.match(prop): + return True + return False + + def __str__(self): + return self._expr + +class Term: + """A list of expressions each of which must match with properties. + + This provides a list of 'AND' expressions, meaning that each must + match the board properties for that board to be built. + """ + def __init__(self): + self._expr_list = [] + self._board_count = 0 + + def AddExpr(self, expr): + """Add an Expr object to the list to check. + + Args: + expr: New Expr object to add to the list of those that must + match for a board to be built. + """ + self._expr_list.append(Expr(expr)) + + def __str__(self): + """Return some sort of useful string describing the term""" + return '&'.join([str(expr) for expr in self._expr_list]) + + def Matches(self, props): + """Check if any of the properties match this term + + Each of the expressions in the term is checked. All must match. + + Args: + props: List of properties to check + Returns: + True if all of the expressions in the Term match, else False + """ + for expr in self._expr_list: + if not expr.Matches(props): + return False + return True + +class Board: + """A particular board that we can build""" + def __init__(self, status, arch, cpu, soc, vendor, board_name, target, options): + """Create a new board type. + + Args: + status: define whether the board is 'Active' or 'Orphaned' + arch: Architecture name (e.g. arm) + cpu: Cpu name (e.g. arm1136) + soc: Name of SOC, or '' if none (e.g. mx31) + vendor: Name of vendor (e.g. armltd) + board_name: Name of board (e.g. integrator) + target: Target name (use make <target>_defconfig to configure) + options: board-specific options (e.g. integratorcp:CM1136) + """ + self.target = target + self.arch = arch + self.cpu = cpu + self.board_name = board_name + self.vendor = vendor + self.soc = soc + self.options = options + self.props = [self.target, self.arch, self.cpu, self.board_name, + self.vendor, self.soc, self.options] + self.build_it = False + + +class Boards: + """Manage a list of boards.""" + def __init__(self): + # Use a simple list here, sinc OrderedDict requires Python 2.7 + self._boards = [] + + def AddBoard(self, board): + """Add a new board to the list. + + The board's target member must not already exist in the board list. + + Args: + board: board to add + """ + self._boards.append(board) + + def ReadBoards(self, fname): + """Read a list of boards from a board file. + + Create a board object for each and add it to our _boards list. + + Args: + fname: Filename of boards.cfg file + """ + with open(fname, 'r', encoding='utf-8') as fd: + for line in fd: + if line[0] == '#': + continue + fields = line.split() + if not fields: + continue + for upto in range(len(fields)): + if fields[upto] == '-': + fields[upto] = '' + while len(fields) < 8: + fields.append('') + if len(fields) > 8: + fields = fields[:8] + + board = Board(*fields) + self.AddBoard(board) + + + def GetList(self): + """Return a list of available boards. + + Returns: + List of Board objects + """ + return self._boards + + def GetDict(self): + """Build a dictionary containing all the boards. + + Returns: + Dictionary: + key is board.target + value is board + """ + board_dict = OrderedDict() + for board in self._boards: + board_dict[board.target] = board + return board_dict + + def GetSelectedDict(self): + """Return a dictionary containing the selected boards + + Returns: + List of Board objects that are marked selected + """ + board_dict = OrderedDict() + for board in self._boards: + if board.build_it: + board_dict[board.target] = board + return board_dict + + def GetSelected(self): + """Return a list of selected boards + + Returns: + List of Board objects that are marked selected + """ + return [board for board in self._boards if board.build_it] + + def GetSelectedNames(self): + """Return a list of selected boards + + Returns: + List of board names that are marked selected + """ + return [board.target for board in self._boards if board.build_it] + + def _BuildTerms(self, args): + """Convert command line arguments to a list of terms. + + This deals with parsing of the arguments. It handles the '&' + operator, which joins several expressions into a single Term. + + For example: + ['arm & freescale sandbox', 'tegra'] + + will produce 3 Terms containing expressions as follows: + arm, freescale + sandbox + tegra + + The first Term has two expressions, both of which must match for + a board to be selected. + + Args: + args: List of command line arguments + Returns: + A list of Term objects + """ + syms = [] + for arg in args: + for word in arg.split(): + sym_build = [] + for term in word.split('&'): + if term: + sym_build.append(term) + sym_build.append('&') + syms += sym_build[:-1] + terms = [] + term = None + oper = None + for sym in syms: + if sym == '&': + oper = sym + elif oper: + term.AddExpr(sym) + oper = None + else: + if term: + terms.append(term) + term = Term() + term.AddExpr(sym) + if term: + terms.append(term) + return terms + + def SelectBoards(self, args, exclude=[], boards=None): + """Mark boards selected based on args + + Normally either boards (an explicit list of boards) or args (a list of + terms to match against) is used. It is possible to specify both, in + which case they are additive. + + If boards and args are both empty, all boards are selected. + + Args: + args: List of strings specifying boards to include, either named, + or by their target, architecture, cpu, vendor or soc. If + empty, all boards are selected. + exclude: List of boards to exclude, regardless of 'args' + boards: List of boards to build + + Returns: + Tuple + Dictionary which holds the list of boards which were selected + due to each argument, arranged by argument. + List of errors found + """ + result = OrderedDict() + warnings = [] + terms = self._BuildTerms(args) + + result['all'] = [] + for term in terms: + result[str(term)] = [] + + exclude_list = [] + for expr in exclude: + exclude_list.append(Expr(expr)) + + found = [] + for board in self._boards: + matching_term = None + build_it = False + if terms: + match = False + for term in terms: + if term.Matches(board.props): + matching_term = str(term) + build_it = True + break + elif boards: + if board.target in boards: + build_it = True + found.append(board.target) + else: + build_it = True + + # Check that it is not specifically excluded + for expr in exclude_list: + if expr.Matches(board.props): + build_it = False + break + + if build_it: + board.build_it = True + if matching_term: + result[matching_term].append(board.target) + result['all'].append(board.target) + + if boards: + remaining = set(boards) - set(found) + if remaining: + warnings.append('Boards not found: %s\n' % ', '.join(remaining)) + + return result, warnings diff --git a/roms/u-boot/tools/buildman/bsettings.py b/roms/u-boot/tools/buildman/bsettings.py new file mode 100644 index 000000000..0b7208da3 --- /dev/null +++ b/roms/u-boot/tools/buildman/bsettings.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. + +import configparser +import os +import io + + +def Setup(fname=''): + """Set up the buildman settings module by reading config files + + Args: + config_fname: Config filename to read ('' for default) + """ + global settings + global config_fname + + settings = configparser.SafeConfigParser() + if fname is not None: + config_fname = fname + if config_fname == '': + config_fname = '%s/.buildman' % os.getenv('HOME') + if not os.path.exists(config_fname): + print('No config file found ~/.buildman\nCreating one...\n') + CreateBuildmanConfigFile(config_fname) + print('To install tool chains, please use the --fetch-arch option') + if config_fname: + settings.read(config_fname) + +def AddFile(data): + settings.readfp(io.StringIO(data)) + +def GetItems(section): + """Get the items from a section of the config. + + Args: + section: name of section to retrieve + + Returns: + List of (name, value) tuples for the section + """ + try: + return settings.items(section) + except configparser.NoSectionError as e: + return [] + except: + raise + +def SetItem(section, tag, value): + """Set an item and write it back to the settings file""" + global settings + global config_fname + + settings.set(section, tag, value) + if config_fname is not None: + with open(config_fname, 'w') as fd: + settings.write(fd) + +def CreateBuildmanConfigFile(config_fname): + """Creates a new config file with no tool chain information. + + Args: + config_fname: Config filename to create + + Returns: + None + """ + try: + f = open(config_fname, 'w') + except IOError: + print("Couldn't create buildman config file '%s'\n" % config_fname) + raise + + print('''[toolchain] +# name = path +# e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux + +[toolchain-prefix] +# name = path to prefix +# e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux- + +[toolchain-alias] +# arch = alias +# Indicates which toolchain should be used to build for that arch +x86 = i386 +blackfin = bfin +nds32 = nds32le +openrisc = or1k + +[make-flags] +# Special flags to pass to 'make' for certain boards, e.g. to pass a test +# flag and build tag to snapper boards: +# snapper-boards=ENABLE_AT91_TEST=1 +# snapper9260=${snapper-boards} BUILD_TAG=442 +# snapper9g45=${snapper-boards} BUILD_TAG=443 +''', file=f) + f.close(); diff --git a/roms/u-boot/tools/buildman/builder.py b/roms/u-boot/tools/buildman/builder.py new file mode 100644 index 000000000..ce852eb03 --- /dev/null +++ b/roms/u-boot/tools/buildman/builder.py @@ -0,0 +1,1744 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2013 The Chromium OS Authors. +# +# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> +# + +import collections +from datetime import datetime, timedelta +import glob +import os +import re +import queue +import shutil +import signal +import string +import sys +import threading +import time + +from buildman import builderthread +from buildman import toolchain +from patman import command +from patman import gitutil +from patman import terminal +from patman.terminal import Print + +""" +Theory of Operation + +Please see README for user documentation, and you should be familiar with +that before trying to make sense of this. + +Buildman works by keeping the machine as busy as possible, building different +commits for different boards on multiple CPUs at once. + +The source repo (self.git_dir) contains all the commits to be built. Each +thread works on a single board at a time. It checks out the first commit, +configures it for that board, then builds it. Then it checks out the next +commit and builds it (typically without re-configuring). When it runs out +of commits, it gets another job from the builder and starts again with that +board. + +Clearly the builder threads could work either way - they could check out a +commit and then built it for all boards. Using separate directories for each +commit/board pair they could leave their build product around afterwards +also. + +The intent behind building a single board for multiple commits, is to make +use of incremental builds. Since each commit is built incrementally from +the previous one, builds are faster. Reconfiguring for a different board +removes all intermediate object files. + +Many threads can be working at once, but each has its own working directory. +When a thread finishes a build, it puts the output files into a result +directory. + +The base directory used by buildman is normally '../<branch>', i.e. +a directory higher than the source repository and named after the branch +being built. + +Within the base directory, we have one subdirectory for each commit. Within +that is one subdirectory for each board. Within that is the build output for +that commit/board combination. + +Buildman also create working directories for each thread, in a .bm-work/ +subdirectory in the base dir. + +As an example, say we are building branch 'us-net' for boards 'sandbox' and +'seaboard', and say that us-net has two commits. We will have directories +like this: + +us-net/ base directory + 01_g4ed4ebc_net--Add-tftp-speed-/ + sandbox/ + u-boot.bin + seaboard/ + u-boot.bin + 02_g4ed4ebc_net--Check-tftp-comp/ + sandbox/ + u-boot.bin + seaboard/ + u-boot.bin + .bm-work/ + 00/ working directory for thread 0 (contains source checkout) + build/ build output + 01/ working directory for thread 1 + build/ build output + ... +u-boot/ source directory + .git/ repository +""" + +"""Holds information about a particular error line we are outputing + + char: Character representation: '+': error, '-': fixed error, 'w+': warning, + 'w-' = fixed warning + boards: List of Board objects which have line in the error/warning output + errline: The text of the error line +""" +ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline') + +# Possible build outcomes +OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4)) + +# Translate a commit subject into a valid filename (and handle unicode) +trans_valid_chars = str.maketrans('/: ', '---') + +BASE_CONFIG_FILENAMES = [ + 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' +] + +EXTRA_CONFIG_FILENAMES = [ + '.config', '.config-spl', '.config-tpl', + 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', + 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', +] + +class Config: + """Holds information about configuration settings for a board.""" + def __init__(self, config_filename, target): + self.target = target + self.config = {} + for fname in config_filename: + self.config[fname] = {} + + def Add(self, fname, key, value): + self.config[fname][key] = value + + def __hash__(self): + val = 0 + for fname in self.config: + for key, value in self.config[fname].items(): + print(key, value) + val = val ^ hash(key) & hash(value) + return val + +class Environment: + """Holds information about environment variables for a board.""" + def __init__(self, target): + self.target = target + self.environment = {} + + def Add(self, key, value): + self.environment[key] = value + +class Builder: + """Class for building U-Boot for a particular commit. + + Public members: (many should ->private) + already_done: Number of builds already completed + base_dir: Base directory to use for builder + checkout: True to check out source, False to skip that step. + This is used for testing. + col: terminal.Color() object + count: Number of commits to build + do_make: Method to call to invoke Make + fail: Number of builds that failed due to error + force_build: Force building even if a build already exists + force_config_on_failure: If a commit fails for a board, disable + incremental building for the next commit we build for that + board, so that we will see all warnings/errors again. + force_build_failures: If a previously-built build (i.e. built on + a previous run of buildman) is marked as failed, rebuild it. + git_dir: Git directory containing source repository + num_jobs: Number of jobs to run at once (passed to make as -j) + num_threads: Number of builder threads to run + out_queue: Queue of results to process + re_make_err: Compiled regular expression for ignore_lines + queue: Queue of jobs to run + threads: List of active threads + toolchains: Toolchains object to use for building + upto: Current commit number we are building (0.count-1) + warned: Number of builds that produced at least one warning + force_reconfig: Reconfigure U-Boot on each comiit. This disables + incremental building, where buildman reconfigures on the first + commit for a baord, and then just does an incremental build for + the following commits. In fact buildman will reconfigure and + retry for any failing commits, so generally the only effect of + this option is to slow things down. + in_tree: Build U-Boot in-tree instead of specifying an output + directory separate from the source code. This option is really + only useful for testing in-tree builds. + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + thread_exceptions: List of exceptions raised by thread jobs + + Private members: + _base_board_dict: Last-summarised Dict of boards + _base_err_lines: Last-summarised list of errors + _base_warn_lines: Last-summarised list of warnings + _build_period_us: Time taken for a single build (float object). + _complete_delay: Expected delay until completion (timedelta) + _next_delay_update: Next time we plan to display a progress update + (datatime) + _show_unknown: Show unknown boards (those not built) in summary + _start_time: Start time for the build + _timestamps: List of timestamps for the completion of the last + last _timestamp_count builds. Each is a datetime object. + _timestamp_count: Number of timestamps to keep in our list. + _working_dir: Base working directory containing all threads + _single_builder: BuilderThread object for the singer builder, if + threading is not being used + """ + class Outcome: + """Records a build outcome for a single make invocation + + Public Members: + rc: Outcome value (OUTCOME_...) + err_lines: List of error lines or [] if none + sizes: Dictionary of image size information, keyed by filename + - Each value is itself a dictionary containing + values for 'text', 'data' and 'bss', being the integer + size in bytes of each section. + func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each + value is itself a dictionary: + key: function name + value: Size of function in bytes + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. + """ + def __init__(self, rc, err_lines, sizes, func_sizes, config, + environment): + self.rc = rc + self.err_lines = err_lines + self.sizes = sizes + self.func_sizes = func_sizes + self.config = config + self.environment = environment + + def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False, full_path=False, verbose_build=False, + mrproper=False, per_board_out_dir=False, + config_only=False, squash_config_y=False, + warnings_as_errors=False, work_in_output=False, + test_thread_exceptions=False): + """Create a new Builder object + + Args: + toolchains: Toolchains object to use for building + base_dir: Base directory to use for builder + git_dir: Git directory containing source repository + num_threads: Number of builder threads to run + num_jobs: Number of jobs to run at once (passed to make as -j) + gnu_make: the command name of GNU Make. + checkout: True to check out source, False to skip that step. + This is used for testing. + show_unknown: Show unknown boards (those not built) in summary + step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + verbose_build: Run build with V=1 and don't use 'make -s' + mrproper: Always run 'make mrproper' when configuring + per_board_out_dir: Build in a separate persistent directory per + board rather than a thread-specific directory + config_only: Only configure each build, don't build it + squash_config_y: Convert CONFIG options with the value 'y' to '1' + warnings_as_errors: Treat all compiler warnings as errors + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + test_thread_exceptions: Uses for tests only, True to make the + threads raise an exception instead of reporting their result. + This simulates a failure in the code somewhere + """ + self.toolchains = toolchains + self.base_dir = base_dir + if work_in_output: + self._working_dir = base_dir + else: + self._working_dir = os.path.join(base_dir, '.bm-work') + self.threads = [] + self.do_make = self.Make + self.gnu_make = gnu_make + self.checkout = checkout + self.num_threads = num_threads + self.num_jobs = num_jobs + self.already_done = 0 + self.force_build = False + self.git_dir = git_dir + self._show_unknown = show_unknown + self._timestamp_count = 10 + self._build_period_us = None + self._complete_delay = None + self._next_delay_update = datetime.now() + self._start_time = datetime.now() + self.force_config_on_failure = True + self.force_build_failures = False + self.force_reconfig = False + self._step = step + self.in_tree = False + self._error_lines = 0 + self.no_subdirs = no_subdirs + self.full_path = full_path + self.verbose_build = verbose_build + self.config_only = config_only + self.squash_config_y = squash_config_y + self.config_filenames = BASE_CONFIG_FILENAMES + self.work_in_output = work_in_output + if not self.squash_config_y: + self.config_filenames += EXTRA_CONFIG_FILENAMES + + self.warnings_as_errors = warnings_as_errors + self.col = terminal.Color() + + self._re_function = re.compile('(.*): In function.*') + self._re_files = re.compile('In file included from.*') + self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') + self._re_dtb_warning = re.compile('(.*): Warning .*') + self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n', + re.MULTILINE | re.DOTALL) + + self.thread_exceptions = [] + self.test_thread_exceptions = test_thread_exceptions + if self.num_threads: + self._single_builder = None + self.queue = queue.Queue() + self.out_queue = queue.Queue() + for i in range(self.num_threads): + t = builderthread.BuilderThread( + self, i, mrproper, per_board_out_dir, + test_exception=test_thread_exceptions) + t.setDaemon(True) + t.start() + self.threads.append(t) + + t = builderthread.ResultThread(self) + t.setDaemon(True) + t.start() + self.threads.append(t) + else: + self._single_builder = builderthread.BuilderThread( + self, -1, mrproper, per_board_out_dir) + + ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] + self.re_make_err = re.compile('|'.join(ignore_lines)) + + # Handle existing graceful with SIGINT / Ctrl-C + signal.signal(signal.SIGINT, self.signal_handler) + + def __del__(self): + """Get rid of all threads created by the builder""" + for t in self.threads: + del t + + def signal_handler(self, signal, frame): + sys.exit(1) + + def SetDisplayOptions(self, show_errors=False, show_sizes=False, + show_detail=False, show_bloat=False, + list_error_boards=False, show_config=False, + show_environment=False, filter_dtb_warnings=False, + filter_migration_warnings=False): + """Setup display options for the builder. + + Args: + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show size delta detail for each board if show_sizes + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + show_config: Show config deltas + show_environment: Show environment deltas + filter_dtb_warnings: Filter out any warnings from the device-tree + compiler + filter_migration_warnings: Filter out any warnings about migrating + a board to driver model + """ + self._show_errors = show_errors + self._show_sizes = show_sizes + self._show_detail = show_detail + self._show_bloat = show_bloat + self._list_error_boards = list_error_boards + self._show_config = show_config + self._show_environment = show_environment + self._filter_dtb_warnings = filter_dtb_warnings + self._filter_migration_warnings = filter_migration_warnings + + def _AddTimestamp(self): + """Add a new timestamp to the list and record the build period. + + The build period is the length of time taken to perform a single + build (one board, one commit). + """ + now = datetime.now() + self._timestamps.append(now) + count = len(self._timestamps) + delta = self._timestamps[-1] - self._timestamps[0] + seconds = delta.total_seconds() + + # If we have enough data, estimate build period (time taken for a + # single build) and therefore completion time. + if count > 1 and self._next_delay_update < now: + self._next_delay_update = now + timedelta(seconds=2) + if seconds > 0: + self._build_period = float(seconds) / count + todo = self.count - self.upto + self._complete_delay = timedelta(microseconds= + self._build_period * todo * 1000000) + # Round it + self._complete_delay -= timedelta( + microseconds=self._complete_delay.microseconds) + + if seconds > 60: + self._timestamps.popleft() + count -= 1 + + def SelectCommit(self, commit, checkout=True): + """Checkout the selected commit for this build + """ + self.commit = commit + if checkout and self.checkout: + gitutil.Checkout(commit.hash) + + def Make(self, commit, brd, stage, cwd, *args, **kwargs): + """Run make + + Args: + commit: Commit object that is being built + brd: Board object that is being built + stage: Stage that we are at (mrproper, config, build) + cwd: Directory where make should be run + args: Arguments to pass to make + kwargs: Arguments to pass to command.RunPipe() + """ + cmd = [self.gnu_make] + list(args) + result = command.RunPipe([cmd], capture=True, capture_stderr=True, + cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs) + if self.verbose_build: + result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout + result.combined = '%s\n' % (' '.join(cmd)) + result.combined + return result + + def ProcessResult(self, result): + """Process the result of a build, showing progress information + + Args: + result: A CommandResult object, which indicates the result for + a single build + """ + col = terminal.Color() + if result: + target = result.brd.target + + self.upto += 1 + if result.return_code != 0: + self.fail += 1 + elif result.stderr: + self.warned += 1 + if result.already_done: + self.already_done += 1 + if self._verbose: + terminal.PrintClear() + boards_selected = {target : result.brd} + self.ResetResultSummary(boards_selected) + self.ProduceResultSummary(result.commit_upto, self.commits, + boards_selected) + else: + target = '(starting)' + + # Display separate counts for ok, warned and fail + ok = self.upto - self.warned - self.fail + line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) + line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) + line += self.col.Color(self.col.RED, '%5d' % self.fail) + + line += ' /%-5d ' % self.count + remaining = self.count - self.upto + if remaining: + line += self.col.Color(self.col.MAGENTA, ' -%-5d ' % remaining) + else: + line += ' ' * 8 + + # Add our current completion time estimate + self._AddTimestamp() + if self._complete_delay: + line += '%s : ' % self._complete_delay + + line += target + terminal.PrintClear() + Print(line, newline=False, limit_to_line=True) + + def _GetOutputDir(self, commit_upto): + """Get the name of the output directory for a commit number + + The output directory is typically .../<branch>/<commit>. + + Args: + commit_upto: Commit number to use (0..self.count-1) + """ + if self.work_in_output: + return self._working_dir + + commit_dir = None + if self.commits: + commit = self.commits[commit_upto] + subject = commit.subject.translate(trans_valid_chars) + # See _GetOutputSpaceRemovals() which parses this name + commit_dir = ('%02d_g%s_%s' % (commit_upto + 1, + commit.hash, subject[:20])) + elif not self.no_subdirs: + commit_dir = 'current' + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir) + + def GetBuildDir(self, commit_upto, target): + """Get the name of the build directory for a commit number + + The build directory is typically .../<branch>/<commit>/<target>. + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + output_dir = self._GetOutputDir(commit_upto) + if self.work_in_output: + return output_dir + return os.path.join(output_dir, target) + + def GetDoneFile(self, commit_upto, target): + """Get the name of the done file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + return os.path.join(self.GetBuildDir(commit_upto, target), 'done') + + def GetSizesFile(self, commit_upto, target): + """Get the name of the sizes file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') + + def GetFuncSizesFile(self, commit_upto, target, elf_fname): + """Get the name of the funcsizes file for a commit number and ELF file + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + elf_fname: Filename of elf image + """ + return os.path.join(self.GetBuildDir(commit_upto, target), + '%s.sizes' % elf_fname.replace('/', '-')) + + def GetObjdumpFile(self, commit_upto, target, elf_fname): + """Get the name of the objdump file for a commit number and ELF file + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + elf_fname: Filename of elf image + """ + return os.path.join(self.GetBuildDir(commit_upto, target), + '%s.objdump' % elf_fname.replace('/', '-')) + + def GetErrFile(self, commit_upto, target): + """Get the name of the err file for a commit number + + Args: + commit_upto: Commit number to use (0..self.count-1) + target: Target name + """ + output_dir = self.GetBuildDir(commit_upto, target) + return os.path.join(output_dir, 'err') + + def FilterErrors(self, lines): + """Filter out errors in which we have no interest + + We should probably use map(). + + Args: + lines: List of error lines, each a string + Returns: + New list with only interesting lines included + """ + out_lines = [] + if self._filter_migration_warnings: + text = '\n'.join(lines) + text = self._re_migration_warning.sub('', text) + lines = text.splitlines() + for line in lines: + if self.re_make_err.search(line): + continue + if self._filter_dtb_warnings and self._re_dtb_warning.search(line): + continue + out_lines.append(line) + return out_lines + + def ReadFuncSizes(self, fname, fd): + """Read function sizes from the output of 'nm' + + Args: + fd: File containing data to read + fname: Filename we are reading from (just for errors) + + Returns: + Dictionary containing size of each function in bytes, indexed by + function name. + """ + sym = {} + for line in fd.readlines(): + try: + if line.strip(): + size, type, name = line[:-1].split() + except: + Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) + continue + if type in 'tTdDbB': + # function names begin with '.' on 64-bit powerpc + if '.' in name[1:]: + name = 'static.' + name.split('.')[0] + sym[name] = sym.get(name, 0) + int(size, 16) + return sym + + def _ProcessConfig(self, fname): + """Read in a .config, autoconf.mk or autoconf.h file + + This function handles all config file types. It ignores comments and + any #defines which don't start with CONFIG_. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: Config name (e.g. CONFIG_DM) + value: Config value (e.g. 1) + """ + config = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd: + line = line.strip() + if line.startswith('#define'): + values = line[8:].split(' ', 1) + if len(values) > 1: + key, value = values + else: + key = values[0] + value = '1' if self.squash_config_y else '' + if not key.startswith('CONFIG_'): + continue + elif not line or line[0] in ['#', '*', '/']: + continue + else: + key, value = line.split('=', 1) + if self.squash_config_y and value == 'y': + value = '1' + config[key] = value + return config + + def _ProcessEnvironment(self, fname): + """Read in a uboot.env file + + This function reads in environment variables from a file. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: environment variable (e.g. bootlimit) + value: value of environment variable (e.g. 1) + """ + environment = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd.read().split('\0'): + try: + key, value = line.split('=', 1) + environment[key] = value + except ValueError: + # ignore lines we can't parse + pass + return environment + + def GetBuildOutcome(self, commit_upto, target, read_func_sizes, + read_config, read_environment): + """Work out the outcome of a build. + + Args: + commit_upto: Commit number to check (0..n-1) + target: Target board to check + read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files + + Returns: + Outcome object + """ + done_file = self.GetDoneFile(commit_upto, target) + sizes_file = self.GetSizesFile(commit_upto, target) + sizes = {} + func_sizes = {} + config = {} + environment = {} + if os.path.exists(done_file): + with open(done_file, 'r') as fd: + try: + return_code = int(fd.readline()) + except ValueError: + # The file may be empty due to running out of disk space. + # Try a rebuild + return_code = 1 + err_lines = [] + err_file = self.GetErrFile(commit_upto, target) + if os.path.exists(err_file): + with open(err_file, 'r') as fd: + err_lines = self.FilterErrors(fd.readlines()) + + # Decide whether the build was ok, failed or created warnings + if return_code: + rc = OUTCOME_ERROR + elif len(err_lines): + rc = OUTCOME_WARNING + else: + rc = OUTCOME_OK + + # Convert size information to our simple format + if os.path.exists(sizes_file): + with open(sizes_file, 'r') as fd: + for line in fd.readlines(): + values = line.split() + rodata = 0 + if len(values) > 6: + rodata = int(values[6], 16) + size_dict = { + 'all' : int(values[0]) + int(values[1]) + + int(values[2]), + 'text' : int(values[0]) - rodata, + 'data' : int(values[1]), + 'bss' : int(values[2]), + 'rodata' : rodata, + } + sizes[values[5]] = size_dict + + if read_func_sizes: + pattern = self.GetFuncSizesFile(commit_upto, target, '*') + for fname in glob.glob(pattern): + with open(fname, 'r') as fd: + dict_name = os.path.basename(fname).replace('.sizes', + '') + func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) + + if read_config: + output_dir = self.GetBuildDir(commit_upto, target) + for name in self.config_filenames: + fname = os.path.join(output_dir, name) + config[name] = self._ProcessConfig(fname) + + if read_environment: + output_dir = self.GetBuildDir(commit_upto, target) + fname = os.path.join(output_dir, 'uboot.env') + environment = self._ProcessEnvironment(fname) + + return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, + environment) + + return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) + + def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, + read_config, read_environment): + """Calculate a summary of the results of building a commit. + + Args: + board_selected: Dict containing boards to summarise + commit_upto: Commit number to summarize (0..self.count-1) + read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files + + Returns: + Tuple: + Dict containing boards which passed building this commit. + keyed by board.target + List containing a summary of error lines + Dict keyed by error line, containing a list of the Board + objects with that error + List containing a summary of warning lines + Dict keyed by error line, containing a list of the Board + objects with that warning + Dictionary keyed by board.target. Each value is a dictionary: + key: filename - e.g. '.config' + value is itself a dictionary: + key: config name + value: config value + Dictionary keyed by board.target. Each value is a dictionary: + key: environment variable + value: value of environment variable + """ + def AddLine(lines_summary, lines_boards, line, board): + line = line.rstrip() + if line in lines_boards: + lines_boards[line].append(board) + else: + lines_boards[line] = [board] + lines_summary.append(line) + + board_dict = {} + err_lines_summary = [] + err_lines_boards = {} + warn_lines_summary = [] + warn_lines_boards = {} + config = {} + environment = {} + + for board in boards_selected.values(): + outcome = self.GetBuildOutcome(commit_upto, board.target, + read_func_sizes, read_config, + read_environment) + board_dict[board.target] = outcome + last_func = None + last_was_warning = False + for line in outcome.err_lines: + if line: + if (self._re_function.match(line) or + self._re_files.match(line)): + last_func = line + else: + is_warning = (self._re_warning.match(line) or + self._re_dtb_warning.match(line)) + is_note = self._re_note.match(line) + if is_warning or (last_was_warning and is_note): + if last_func: + AddLine(warn_lines_summary, warn_lines_boards, + last_func, board) + AddLine(warn_lines_summary, warn_lines_boards, + line, board) + else: + if last_func: + AddLine(err_lines_summary, err_lines_boards, + last_func, board) + AddLine(err_lines_summary, err_lines_boards, + line, board) + last_was_warning = is_warning + last_func = None + tconfig = Config(self.config_filenames, board.target) + for fname in self.config_filenames: + if outcome.config: + for key, value in outcome.config[fname].items(): + tconfig.Add(fname, key, value) + config[board.target] = tconfig + + tenvironment = Environment(board.target) + if outcome.environment: + for key, value in outcome.environment.items(): + tenvironment.Add(key, value) + environment[board.target] = tenvironment + + return (board_dict, err_lines_summary, err_lines_boards, + warn_lines_summary, warn_lines_boards, config, environment) + + def AddOutcome(self, board_dict, arch_list, changes, char, color): + """Add an output to our list of outcomes for each architecture + + This simple function adds failing boards (changes) to the + relevant architecture string, so we can print the results out + sorted by architecture. + + Args: + board_dict: Dict containing all boards + arch_list: Dict keyed by arch name. Value is a string containing + a list of board names which failed for that arch. + changes: List of boards to add to arch_list + color: terminal.Colour object + """ + done_arch = {} + for target in changes: + if target in board_dict: + arch = board_dict[target].arch + else: + arch = 'unknown' + str = self.col.Color(color, ' ' + target) + if not arch in done_arch: + str = ' %s %s' % (self.col.Color(color, char), str) + done_arch[arch] = True + if not arch in arch_list: + arch_list[arch] = str + else: + arch_list[arch] += str + + + def ColourNum(self, num): + color = self.col.RED if num > 0 else self.col.GREEN + if num == 0: + return '0' + return self.col.Color(color, str(num)) + + def ResetResultSummary(self, board_selected): + """Reset the results summary ready for use. + + Set up the base board list to be all those selected, and set the + error lines to empty. + + Following this, calls to PrintResultSummary() will use this + information to work out what has changed. + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + """ + self._base_board_dict = {} + for board in board_selected: + self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, + {}) + self._base_err_lines = [] + self._base_warn_lines = [] + self._base_err_line_boards = {} + self._base_warn_line_boards = {} + self._base_config = None + self._base_environment = None + + def PrintFuncSizeDetail(self, fname, old, new): + grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 + delta, common = [], {} + + for a in old: + if a in new: + common[a] = 1 + + for name in old: + if name not in common: + remove += 1 + down += old[name] + delta.append([-old[name], name]) + + for name in new: + if name not in common: + add += 1 + up += new[name] + delta.append([new[name], name]) + + for name in common: + diff = new.get(name, 0) - old.get(name, 0) + if diff > 0: + grow, up = grow + 1, up + diff + elif diff < 0: + shrink, down = shrink + 1, down - diff + delta.append([diff, name]) + + delta.sort() + delta.reverse() + + args = [add, -remove, grow, -shrink, up, -down, up - down] + if max(args) == 0 and min(args) == 0: + return + args = [self.ColourNum(x) for x in args] + indent = ' ' * 15 + Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % + tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) + Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', + 'delta')) + for diff, name in delta: + if diff: + color = self.col.RED if diff > 0 else self.col.GREEN + msg = '%s %-38s %7s %7s %+7d' % (indent, name, + old.get(name, '-'), new.get(name,'-'), diff) + Print(msg, colour=color) + + + def PrintSizeDetail(self, target_list, show_bloat): + """Show details size information for each board + + Args: + target_list: List of targets, each a dict containing: + 'target': Target name + 'total_diff': Total difference in bytes across all areas + <part_name>: Difference for that part + show_bloat: Show detail for each function + """ + targets_by_diff = sorted(target_list, reverse=True, + key=lambda x: x['_total_diff']) + for result in targets_by_diff: + printed_target = False + for name in sorted(result): + diff = result[name] + if name.startswith('_'): + continue + if diff != 0: + color = self.col.RED if diff > 0 else self.col.GREEN + msg = ' %s %+d' % (name, diff) + if not printed_target: + Print('%10s %-15s:' % ('', result['_target']), + newline=False) + printed_target = True + Print(msg, colour=color, newline=False) + if printed_target: + Print() + if show_bloat: + target = result['_target'] + outcome = result['_outcome'] + base_outcome = self._base_board_dict[target] + for fname in outcome.func_sizes: + self.PrintFuncSizeDetail(fname, + base_outcome.func_sizes[fname], + outcome.func_sizes[fname]) + + + def PrintSizeSummary(self, board_selected, board_dict, show_detail, + show_bloat): + """Print a summary of image sizes broken down by section. + + The summary takes the form of one line per architecture. The + line contains deltas for each of the sections (+ means the section + got bigger, - means smaller). The numbers are the average number + of bytes that a board in this section increased by. + + For example: + powerpc: (622 boards) text -0.0 + arm: (285 boards) text -0.0 + nds32: (3 boards) text -8.0 + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + board_dict: Dict containing boards for which we built this + commit, keyed by board.target. The value is an Outcome object. + show_detail: Show size delta detail for each board + show_bloat: Show detail for each function + """ + arch_list = {} + arch_count = {} + + # Calculate changes in size for different image parts + # The previous sizes are in Board.sizes, for each board + for target in board_dict: + if target not in board_selected: + continue + base_sizes = self._base_board_dict[target].sizes + outcome = board_dict[target] + sizes = outcome.sizes + + # Loop through the list of images, creating a dict of size + # changes for each image/part. We end up with something like + # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} + # which means that U-Boot data increased by 5 bytes and SPL + # text decreased by 4. + err = {'_target' : target} + for image in sizes: + if image in base_sizes: + base_image = base_sizes[image] + # Loop through the text, data, bss parts + for part in sorted(sizes[image]): + diff = sizes[image][part] - base_image[part] + col = None + if diff: + if image == 'u-boot': + name = part + else: + name = image + ':' + part + err[name] = diff + arch = board_selected[target].arch + if not arch in arch_count: + arch_count[arch] = 1 + else: + arch_count[arch] += 1 + if not sizes: + pass # Only add to our list when we have some stats + elif not arch in arch_list: + arch_list[arch] = [err] + else: + arch_list[arch].append(err) + + # We now have a list of image size changes sorted by arch + # Print out a summary of these + for arch, target_list in arch_list.items(): + # Get total difference for each type + totals = {} + for result in target_list: + total = 0 + for name, diff in result.items(): + if name.startswith('_'): + continue + total += diff + if name in totals: + totals[name] += diff + else: + totals[name] = diff + result['_total_diff'] = total + result['_outcome'] = board_dict[result['_target']] + + count = len(target_list) + printed_arch = False + for name in sorted(totals): + diff = totals[name] + if diff: + # Display the average difference in this name for this + # architecture + avg_diff = float(diff) / count + color = self.col.RED if avg_diff > 0 else self.col.GREEN + msg = ' %s %+1.1f' % (name, avg_diff) + if not printed_arch: + Print('%10s: (for %d/%d boards)' % (arch, count, + arch_count[arch]), newline=False) + printed_arch = True + Print(msg, colour=color, newline=False) + + if printed_arch: + Print() + if show_detail: + self.PrintSizeDetail(target_list, show_bloat) + + + def PrintResultSummary(self, board_selected, board_dict, err_lines, + err_line_boards, warn_lines, warn_line_boards, + config, environment, show_sizes, show_detail, + show_bloat, show_config, show_environment): + """Compare results with the base results and display delta. + + Only boards mentioned in board_selected will be considered. This + function is intended to be called repeatedly with the results of + each commit. It therefore shows a 'diff' between what it saw in + the last call and what it sees now. + + Args: + board_selected: Dict containing boards to summarise, keyed by + board.target + board_dict: Dict containing boards for which we built this + commit, keyed by board.target. The value is an Outcome object. + err_lines: A list of errors for this commit, or [] if there is + none, or we don't want to print errors + err_line_boards: Dict keyed by error line, containing a list of + the Board objects with that error + warn_lines: A list of warnings for this commit, or [] if there is + none, or we don't want to print errors + warn_line_boards: Dict keyed by warning line, containing a list of + the Board objects with that warning + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. + show_sizes: Show image size deltas + show_detail: Show size delta detail for each board if show_sizes + show_bloat: Show detail for each function + show_config: Show config changes + show_environment: Show environment changes + """ + def _BoardList(line, line_boards): + """Helper function to get a line of boards containing a line + + Args: + line: Error line to search for + line_boards: boards to search, each a Board + Return: + List of boards with that error line, or [] if the user has not + requested such a list + """ + boards = [] + board_set = set() + if self._list_error_boards: + for board in line_boards[line]: + if not board in board_set: + boards.append(board) + board_set.add(board) + return boards + + def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, + char): + """Calculate the required output based on changes in errors + + Args: + base_lines: List of errors/warnings for previous commit + base_line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in the previous commit + lines: List of errors/warning for this commit, each a str + line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in this commit + char: Character representing error ('') or warning ('w'). The + broken ('+') or fixed ('-') characters are added in this + function + + Returns: + Tuple + List of ErrLine objects for 'better' lines + List of ErrLine objects for 'worse' lines + """ + better_lines = [] + worse_lines = [] + for line in lines: + if line not in base_lines: + errline = ErrLine(char + '+', _BoardList(line, line_boards), + line) + worse_lines.append(errline) + for line in base_lines: + if line not in lines: + errline = ErrLine(char + '-', + _BoardList(line, base_line_boards), line) + better_lines.append(errline) + return better_lines, worse_lines + + def _CalcConfig(delta, name, config): + """Calculate configuration changes + + Args: + delta: Type of the delta, e.g. '+' + name: name of the file which changed (e.g. .config) + config: configuration change dictionary + key: config name + value: config value + Returns: + String containing the configuration changes which can be + printed + """ + out = '' + for key in sorted(config.keys()): + out += '%s=%s ' % (key, config[key]) + return '%s %s: %s' % (delta, name, out) + + def _AddConfig(lines, name, config_plus, config_minus, config_change): + """Add changes in configuration to a list + + Args: + lines: list to add to + name: config file name + config_plus: configurations added, dictionary + key: config name + value: config value + config_minus: configurations removed, dictionary + key: config name + value: config value + config_change: configurations changed, dictionary + key: config name + value: config value + """ + if config_plus: + lines.append(_CalcConfig('+', name, config_plus)) + if config_minus: + lines.append(_CalcConfig('-', name, config_minus)) + if config_change: + lines.append(_CalcConfig('c', name, config_change)) + + def _OutputConfigInfo(lines): + for line in lines: + if not line: + continue + if line[0] == '+': + col = self.col.GREEN + elif line[0] == '-': + col = self.col.RED + elif line[0] == 'c': + col = self.col.YELLOW + Print(' ' + line, newline=True, colour=col) + + def _OutputErrLines(err_lines, colour): + """Output the line of error/warning lines, if not empty + + Also increments self._error_lines if err_lines not empty + + Args: + err_lines: List of ErrLine objects, each an error or warning + line, possibly including a list of boards with that + error/warning + colour: Colour to use for output + """ + if err_lines: + out_list = [] + for line in err_lines: + boards = '' + names = [board.target for board in line.boards] + board_str = ' '.join(names) if names else '' + if board_str: + out = self.col.Color(colour, line.char + '(') + out += self.col.Color(self.col.MAGENTA, board_str, + bright=False) + out += self.col.Color(colour, ') %s' % line.errline) + else: + out = self.col.Color(colour, line.char + line.errline) + out_list.append(out) + Print('\n'.join(out_list)) + self._error_lines += 1 + + + ok_boards = [] # List of boards fixed since last commit + warn_boards = [] # List of boards with warnings since last commit + err_boards = [] # List of new broken boards since last commit + new_boards = [] # List of boards that didn't exist last time + unknown_boards = [] # List of boards that were not built + + for target in board_dict: + if target not in board_selected: + continue + + # If the board was built last time, add its outcome to a list + if target in self._base_board_dict: + base_outcome = self._base_board_dict[target].rc + outcome = board_dict[target] + if outcome.rc == OUTCOME_UNKNOWN: + unknown_boards.append(target) + elif outcome.rc < base_outcome: + if outcome.rc == OUTCOME_WARNING: + warn_boards.append(target) + else: + ok_boards.append(target) + elif outcome.rc > base_outcome: + if outcome.rc == OUTCOME_WARNING: + warn_boards.append(target) + else: + err_boards.append(target) + else: + new_boards.append(target) + + # Get a list of errors and warnings that have appeared, and disappeared + better_err, worse_err = _CalcErrorDelta(self._base_err_lines, + self._base_err_line_boards, err_lines, err_line_boards, '') + better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, + self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') + + # Display results by arch + if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, + worse_err, better_err, worse_warn, better_warn)): + arch_list = {} + self.AddOutcome(board_selected, arch_list, ok_boards, '', + self.col.GREEN) + self.AddOutcome(board_selected, arch_list, warn_boards, 'w+', + self.col.YELLOW) + self.AddOutcome(board_selected, arch_list, err_boards, '+', + self.col.RED) + self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE) + if self._show_unknown: + self.AddOutcome(board_selected, arch_list, unknown_boards, '?', + self.col.MAGENTA) + for arch, target_list in arch_list.items(): + Print('%10s: %s' % (arch, target_list)) + self._error_lines += 1 + _OutputErrLines(better_err, colour=self.col.GREEN) + _OutputErrLines(worse_err, colour=self.col.RED) + _OutputErrLines(better_warn, colour=self.col.CYAN) + _OutputErrLines(worse_warn, colour=self.col.YELLOW) + + if show_sizes: + self.PrintSizeSummary(board_selected, board_dict, show_detail, + show_bloat) + + if show_environment and self._base_environment: + lines = [] + + for target in board_dict: + if target not in board_selected: + continue + + tbase = self._base_environment[target] + tenvironment = environment[target] + environment_plus = {} + environment_minus = {} + environment_change = {} + base = tbase.environment + for key, value in tenvironment.environment.items(): + if key not in base: + environment_plus[key] = value + for key, value in base.items(): + if key not in tenvironment.environment: + environment_minus[key] = value + for key, value in base.items(): + new_value = tenvironment.environment.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + environment_change[key] = desc + + _AddConfig(lines, target, environment_plus, environment_minus, + environment_change) + + _OutputConfigInfo(lines) + + if show_config and self._base_config: + summary = {} + arch_config_plus = {} + arch_config_minus = {} + arch_config_change = {} + arch_list = [] + + for target in board_dict: + if target not in board_selected: + continue + arch = board_selected[target].arch + if arch not in arch_list: + arch_list.append(arch) + + for arch in arch_list: + arch_config_plus[arch] = {} + arch_config_minus[arch] = {} + arch_config_change[arch] = {} + for name in self.config_filenames: + arch_config_plus[arch][name] = {} + arch_config_minus[arch][name] = {} + arch_config_change[arch][name] = {} + + for target in board_dict: + if target not in board_selected: + continue + + arch = board_selected[target].arch + + all_config_plus = {} + all_config_minus = {} + all_config_change = {} + tbase = self._base_config[target] + tconfig = config[target] + lines = [] + for name in self.config_filenames: + if not tconfig.config[name]: + continue + config_plus = {} + config_minus = {} + config_change = {} + base = tbase.config[name] + for key, value in tconfig.config[name].items(): + if key not in base: + config_plus[key] = value + all_config_plus[key] = value + for key, value in base.items(): + if key not in tconfig.config[name]: + config_minus[key] = value + all_config_minus[key] = value + for key, value in base.items(): + new_value = tconfig.config.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + config_change[key] = desc + all_config_change[key] = desc + + arch_config_plus[arch][name].update(config_plus) + arch_config_minus[arch][name].update(config_minus) + arch_config_change[arch][name].update(config_change) + + _AddConfig(lines, name, config_plus, config_minus, + config_change) + _AddConfig(lines, 'all', all_config_plus, all_config_minus, + all_config_change) + summary[target] = '\n'.join(lines) + + lines_by_target = {} + for target, lines in summary.items(): + if lines in lines_by_target: + lines_by_target[lines].append(target) + else: + lines_by_target[lines] = [target] + + for arch in arch_list: + lines = [] + all_plus = {} + all_minus = {} + all_change = {} + for name in self.config_filenames: + all_plus.update(arch_config_plus[arch][name]) + all_minus.update(arch_config_minus[arch][name]) + all_change.update(arch_config_change[arch][name]) + _AddConfig(lines, name, arch_config_plus[arch][name], + arch_config_minus[arch][name], + arch_config_change[arch][name]) + _AddConfig(lines, 'all', all_plus, all_minus, all_change) + #arch_summary[target] = '\n'.join(lines) + if lines: + Print('%s:' % arch) + _OutputConfigInfo(lines) + + for lines, targets in lines_by_target.items(): + if not lines: + continue + Print('%s :' % ' '.join(sorted(targets))) + _OutputConfigInfo(lines.split('\n')) + + + # Save our updated information for the next call to this function + self._base_board_dict = board_dict + self._base_err_lines = err_lines + self._base_warn_lines = warn_lines + self._base_err_line_boards = err_line_boards + self._base_warn_line_boards = warn_line_boards + self._base_config = config + self._base_environment = environment + + # Get a list of boards that did not get built, if needed + not_built = [] + for board in board_selected: + if not board in board_dict: + not_built.append(board) + if not_built: + Print("Boards not built (%d): %s" % (len(not_built), + ', '.join(not_built))) + + def ProduceResultSummary(self, commit_upto, commits, board_selected): + (board_dict, err_lines, err_line_boards, warn_lines, + warn_line_boards, config, environment) = self.GetResultSummary( + board_selected, commit_upto, + read_func_sizes=self._show_bloat, + read_config=self._show_config, + read_environment=self._show_environment) + if commits: + msg = '%02d: %s' % (commit_upto + 1, + commits[commit_upto].subject) + Print(msg, colour=self.col.BLUE) + self.PrintResultSummary(board_selected, board_dict, + err_lines if self._show_errors else [], err_line_boards, + warn_lines if self._show_errors else [], warn_line_boards, + config, environment, self._show_sizes, self._show_detail, + self._show_bloat, self._show_config, self._show_environment) + + def ShowSummary(self, commits, board_selected): + """Show a build summary for U-Boot for a given board list. + + Reset the result summary, then repeatedly call GetResultSummary on + each commit's results, then display the differences we see. + + Args: + commit: Commit objects to summarise + board_selected: Dict containing boards to summarise + """ + self.commit_count = len(commits) if commits else 1 + self.commits = commits + self.ResetResultSummary(board_selected) + self._error_lines = 0 + + for commit_upto in range(0, self.commit_count, self._step): + self.ProduceResultSummary(commit_upto, commits, board_selected) + if not self._error_lines: + Print('(no errors to report)', colour=self.col.GREEN) + + + def SetupBuild(self, board_selected, commits): + """Set up ready to start a build. + + Args: + board_selected: Selected boards to build + commits: Selected commits to build + """ + # First work out how many commits we will build + count = (self.commit_count + self._step - 1) // self._step + self.count = len(board_selected) * count + self.upto = self.warned = self.fail = 0 + self._timestamps = collections.deque() + + def GetThreadDir(self, thread_num): + """Get the directory path to the working dir for a thread. + + Args: + thread_num: Number of thread to check (-1 for main process, which + is treated as 0) + """ + if self.work_in_output: + return self._working_dir + return os.path.join(self._working_dir, '%02d' % max(thread_num, 0)) + + def _PrepareThread(self, thread_num, setup_git): + """Prepare the working directory for a thread. + + This clones or fetches the repo into the thread's work directory. + Optionally, it can create a linked working tree of the repo in the + thread's work directory instead. + + Args: + thread_num: Thread number (0, 1, ...) + setup_git: + 'clone' to set up a git clone + 'worktree' to set up a git worktree + """ + thread_dir = self.GetThreadDir(thread_num) + builderthread.Mkdir(thread_dir) + git_dir = os.path.join(thread_dir, '.git') + + # Create a worktree or a git repo clone for this thread if it + # doesn't already exist + if setup_git and self.git_dir: + src_dir = os.path.abspath(self.git_dir) + if os.path.isdir(git_dir): + # This is a clone of the src_dir repo, we can keep using + # it but need to fetch from src_dir. + Print('\rFetching repo for thread %d' % thread_num, + newline=False) + gitutil.Fetch(git_dir, thread_dir) + terminal.PrintClear() + elif os.path.isfile(git_dir): + # This is a worktree of the src_dir repo, we don't need to + # create it again or update it in any way. + pass + elif os.path.exists(git_dir): + # Don't know what could trigger this, but we probably + # can't create a git worktree/clone here. + raise ValueError('Git dir %s exists, but is not a file ' + 'or a directory.' % git_dir) + elif setup_git == 'worktree': + Print('\rChecking out worktree for thread %d' % thread_num, + newline=False) + gitutil.AddWorktree(src_dir, thread_dir) + terminal.PrintClear() + elif setup_git == 'clone' or setup_git == True: + Print('\rCloning repo for thread %d' % thread_num, + newline=False) + gitutil.Clone(src_dir, thread_dir) + terminal.PrintClear() + else: + raise ValueError("Can't setup git repo with %s." % setup_git) + + def _PrepareWorkingSpace(self, max_threads, setup_git): + """Prepare the working directory for use. + + Set up the git repo for each thread. Creates a linked working tree + if git-worktree is available, or clones the repo if it isn't. + + Args: + max_threads: Maximum number of threads we expect to need. If 0 then + 1 is set up, since the main process still needs somewhere to + work + setup_git: True to set up a git worktree or a git clone + """ + builderthread.Mkdir(self._working_dir) + if setup_git and self.git_dir: + src_dir = os.path.abspath(self.git_dir) + if gitutil.CheckWorktreeIsAvailable(src_dir): + setup_git = 'worktree' + # If we previously added a worktree but the directory for it + # got deleted, we need to prune its files from the repo so + # that we can check out another in its place. + gitutil.PruneWorktrees(src_dir) + else: + setup_git = 'clone' + + # Always do at least one thread + for thread in range(max(max_threads, 1)): + self._PrepareThread(thread, setup_git) + + def _GetOutputSpaceRemovals(self): + """Get the output directories ready to receive files. + + Figure out what needs to be deleted in the output directory before it + can be used. We only delete old buildman directories which have the + expected name pattern. See _GetOutputDir(). + + Returns: + List of full paths of directories to remove + """ + if not self.commits: + return + dir_list = [] + for commit_upto in range(self.commit_count): + dir_list.append(self._GetOutputDir(commit_upto)) + + to_remove = [] + for dirname in glob.glob(os.path.join(self.base_dir, '*')): + if dirname not in dir_list: + leaf = dirname[len(self.base_dir) + 1:] + m = re.match('[0-9]+_g[0-9a-f]+_.*', leaf) + if m: + to_remove.append(dirname) + return to_remove + + def _PrepareOutputSpace(self): + """Get the output directories ready to receive files. + + We delete any output directories which look like ones we need to + create. Having left over directories is confusing when the user wants + to check the output manually. + """ + to_remove = self._GetOutputSpaceRemovals() + if to_remove: + Print('Removing %d old build directories...' % len(to_remove), + newline=False) + for dirname in to_remove: + shutil.rmtree(dirname) + terminal.PrintClear() + + def BuildBoards(self, commits, board_selected, keep_outputs, verbose): + """Build all commits for a list of boards + + Args: + commits: List of commits to be build, each a Commit object + boards_selected: Dict of selected boards, key is target name, + value is Board object + keep_outputs: True to save build output files + verbose: Display build results as they are completed + Returns: + Tuple containing: + - number of boards that failed to build + - number of boards that issued warnings + - list of thread exceptions raised + """ + self.commit_count = len(commits) if commits else 1 + self.commits = commits + self._verbose = verbose + + self.ResetResultSummary(board_selected) + builderthread.Mkdir(self.base_dir, parents = True) + self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), + commits is not None) + self._PrepareOutputSpace() + Print('\rStarting build...', newline=False) + self.SetupBuild(board_selected, commits) + self.ProcessResult(None) + self.thread_exceptions = [] + # Create jobs to build all commits for each board + for brd in board_selected.values(): + job = builderthread.BuilderJob() + job.board = brd + job.commits = commits + job.keep_outputs = keep_outputs + job.work_in_output = self.work_in_output + job.step = self._step + if self.num_threads: + self.queue.put(job) + else: + results = self._single_builder.RunJob(job) + + if self.num_threads: + term = threading.Thread(target=self.queue.join) + term.setDaemon(True) + term.start() + while term.is_alive(): + term.join(100) + + # Wait until we have processed all output + self.out_queue.join() + Print() + + msg = 'Completed: %d total built' % self.count + if self.already_done: + msg += ' (%d previously' % self.already_done + if self.already_done != self.count: + msg += ', %d newly' % (self.count - self.already_done) + msg += ')' + duration = datetime.now() - self._start_time + if duration > timedelta(microseconds=1000000): + if duration.microseconds >= 500000: + duration = duration + timedelta(seconds=1) + duration = duration - timedelta(microseconds=duration.microseconds) + rate = float(self.count) / duration.total_seconds() + msg += ', duration %s, rate %1.2f' % (duration, rate) + Print(msg) + if self.thread_exceptions: + Print('Failed: %d thread exceptions' % len(self.thread_exceptions), + colour=self.col.RED) + + return (self.fail, self.warned, self.thread_exceptions) diff --git a/roms/u-boot/tools/buildman/builderthread.py b/roms/u-boot/tools/buildman/builderthread.py new file mode 100644 index 000000000..48128cf67 --- /dev/null +++ b/roms/u-boot/tools/buildman/builderthread.py @@ -0,0 +1,562 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2014 Google, Inc +# + +import errno +import glob +import os +import shutil +import sys +import threading + +from patman import command +from patman import gitutil + +RETURN_CODE_RETRY = -1 +BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] + +def Mkdir(dirname, parents = False): + """Make a directory if it doesn't already exist. + + Args: + dirname: Directory to create + """ + try: + if parents: + os.makedirs(dirname) + else: + os.mkdir(dirname) + except OSError as err: + if err.errno == errno.EEXIST: + if os.path.realpath('.') == os.path.realpath(dirname): + print("Cannot create the current working directory '%s'!" % dirname) + sys.exit(1) + pass + else: + raise + +class BuilderJob: + """Holds information about a job to be performed by a thread + + Members: + board: Board object to build + commits: List of Commit objects to build + keep_outputs: True to save build output files + step: 1 to process every commit, n to process every nth commit + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + """ + def __init__(self): + self.board = None + self.commits = [] + self.keep_outputs = False + self.step = 1 + self.work_in_output = False + + +class ResultThread(threading.Thread): + """This thread processes results from builder threads. + + It simply passes the results on to the builder. There is only one + result thread, and this helps to serialise the build output. + """ + def __init__(self, builder): + """Set up a new result thread + + Args: + builder: Builder which will be sent each result + """ + threading.Thread.__init__(self) + self.builder = builder + + def run(self): + """Called to start up the result thread. + + We collect the next result job and pass it on to the build. + """ + while True: + result = self.builder.out_queue.get() + self.builder.ProcessResult(result) + self.builder.out_queue.task_done() + + +class BuilderThread(threading.Thread): + """This thread builds U-Boot for a particular board. + + An input queue provides each new job. We run 'make' to build U-Boot + and then pass the results on to the output queue. + + Members: + builder: The builder which contains information we might need + thread_num: Our thread number (0-n-1), used to decide on a + temporary directory. If this is -1 then there are no threads + and we are the (only) main process + mrproper: Use 'make mrproper' before each reconfigure + per_board_out_dir: True to build in a separate persistent directory per + board rather than a thread-specific directory + test_exception: Used for testing; True to raise an exception instead of + reporting the build result + """ + def __init__(self, builder, thread_num, mrproper, per_board_out_dir, + test_exception=False): + """Set up a new builder thread""" + threading.Thread.__init__(self) + self.builder = builder + self.thread_num = thread_num + self.mrproper = mrproper + self.per_board_out_dir = per_board_out_dir + self.test_exception = test_exception + + def Make(self, commit, brd, stage, cwd, *args, **kwargs): + """Run 'make' on a particular commit and board. + + The source code will already be checked out, so the 'commit' + argument is only for information. + + Args: + commit: Commit object that is being built + brd: Board object that is being built + stage: Stage of the build. Valid stages are: + mrproper - can be called to clean source + config - called to configure for a board + build - the main make invocation - it does the build + args: A list of arguments to pass to 'make' + kwargs: A list of keyword arguments to pass to command.RunPipe() + + Returns: + CommandResult object + """ + return self.builder.do_make(commit, brd, stage, cwd, *args, + **kwargs) + + def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, + force_build, force_build_failures, work_in_output): + """Build a particular commit. + + If the build is already done, and we are not forcing a build, we skip + the build and just return the previously-saved results. + + Args: + commit_upto: Commit number to build (0...n-1) + brd: Board object to build + work_dir: Directory to which the source will be checked out + do_config: True to run a make <board>_defconfig on the source + config_only: Only configure the source, do not build it + force_build: Force a build even if one was previously done + force_build_failures: Force a bulid if the previous result showed + failure + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + + Returns: + tuple containing: + - CommandResult object containing the results of the build + - boolean indicating whether 'make config' is still needed + """ + # Create a default result - it will be overwritte by the call to + # self.Make() below, in the event that we do a build. + result = command.CommandResult() + result.return_code = 0 + if work_in_output or self.builder.in_tree: + out_dir = work_dir + else: + if self.per_board_out_dir: + out_rel_dir = os.path.join('..', brd.target) + else: + out_rel_dir = 'build' + out_dir = os.path.join(work_dir, out_rel_dir) + + # Check if the job was already completed last time + done_file = self.builder.GetDoneFile(commit_upto, brd.target) + result.already_done = os.path.exists(done_file) + will_build = (force_build or force_build_failures or + not result.already_done) + if result.already_done: + # Get the return code from that build and use it + with open(done_file, 'r') as fd: + try: + result.return_code = int(fd.readline()) + except ValueError: + # The file may be empty due to running out of disk space. + # Try a rebuild + result.return_code = RETURN_CODE_RETRY + + # Check the signal that the build needs to be retried + if result.return_code == RETURN_CODE_RETRY: + will_build = True + elif will_build: + err_file = self.builder.GetErrFile(commit_upto, brd.target) + if os.path.exists(err_file) and os.stat(err_file).st_size: + result.stderr = 'bad' + elif not force_build: + # The build passed, so no need to build it again + will_build = False + + if will_build: + # We are going to have to build it. First, get a toolchain + if not self.toolchain: + try: + self.toolchain = self.builder.toolchains.Select(brd.arch) + except ValueError as err: + result.return_code = 10 + result.stdout = '' + result.stderr = str(err) + # TODO(sjg@chromium.org): This gets swallowed, but needs + # to be reported. + + if self.toolchain: + # Checkout the right commit + if self.builder.commits: + commit = self.builder.commits[commit_upto] + if self.builder.checkout: + git_dir = os.path.join(work_dir, '.git') + gitutil.Checkout(commit.hash, git_dir, work_dir, + force=True) + else: + commit = 'current' + + # Set up the environment and command line + env = self.toolchain.MakeEnvironment(self.builder.full_path) + Mkdir(out_dir) + args = [] + cwd = work_dir + src_dir = os.path.realpath(work_dir) + if not self.builder.in_tree: + if commit_upto is None: + # In this case we are building in the original source + # directory (i.e. the current directory where buildman + # is invoked. The output directory is set to this + # thread's selected work directory. + # + # Symlinks can confuse U-Boot's Makefile since + # we may use '..' in our path, so remove them. + out_dir = os.path.realpath(out_dir) + args.append('O=%s' % out_dir) + cwd = None + src_dir = os.getcwd() + else: + args.append('O=%s' % out_rel_dir) + if self.builder.verbose_build: + args.append('V=1') + else: + args.append('-s') + if self.builder.num_jobs is not None: + args.extend(['-j', str(self.builder.num_jobs)]) + if self.builder.warnings_as_errors: + args.append('KCFLAGS=-Werror') + config_args = ['%s_defconfig' % brd.target] + config_out = '' + args.extend(self.builder.toolchains.GetMakeArguments(brd)) + args.extend(self.toolchain.MakeArgs()) + + # Remove any output targets. Since we use a build directory that + # was previously used by another board, it may have produced an + # SPL image. If we don't remove it (i.e. see do_config and + # self.mrproper below) then it will appear to be the output of + # this build, even if it does not produce SPL images. + build_dir = self.builder.GetBuildDir(commit_upto, brd.target) + for elf in BASE_ELF_FILENAMES: + fname = os.path.join(out_dir, elf) + if os.path.exists(fname): + os.remove(fname) + + # If we need to reconfigure, do that now + if do_config: + config_out = '' + if self.mrproper: + result = self.Make(commit, brd, 'mrproper', cwd, + 'mrproper', *args, env=env) + config_out += result.combined + result = self.Make(commit, brd, 'config', cwd, + *(args + config_args), env=env) + config_out += result.combined + do_config = False # No need to configure next time + if result.return_code == 0: + if config_only: + args.append('cfg') + result = self.Make(commit, brd, 'build', cwd, *args, + env=env) + result.stderr = result.stderr.replace(src_dir + '/', '') + if self.builder.verbose_build: + result.stdout = config_out + result.stdout + else: + result.return_code = 1 + result.stderr = 'No tool chain for %s\n' % brd.arch + result.already_done = False + + result.toolchain = self.toolchain + result.brd = brd + result.commit_upto = commit_upto + result.out_dir = out_dir + return result, do_config + + def _WriteResult(self, result, keep_outputs, work_in_output): + """Write a built result to the output directory. + + Args: + result: CommandResult object containing result to write + keep_outputs: True to store the output binaries, False + to delete them + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. + """ + # Fatal error + if result.return_code < 0: + return + + # If we think this might have been aborted with Ctrl-C, record the + # failure but not that we are 'done' with this board. A retry may fix + # it. + maybe_aborted = result.stderr and 'No child processes' in result.stderr + + if result.already_done: + return + + # Write the output and stderr + output_dir = self.builder._GetOutputDir(result.commit_upto) + Mkdir(output_dir) + build_dir = self.builder.GetBuildDir(result.commit_upto, + result.brd.target) + Mkdir(build_dir) + + outfile = os.path.join(build_dir, 'log') + with open(outfile, 'w') as fd: + if result.stdout: + fd.write(result.stdout) + + errfile = self.builder.GetErrFile(result.commit_upto, + result.brd.target) + if result.stderr: + with open(errfile, 'w') as fd: + fd.write(result.stderr) + elif os.path.exists(errfile): + os.remove(errfile) + + if result.toolchain: + # Write the build result and toolchain information. + done_file = self.builder.GetDoneFile(result.commit_upto, + result.brd.target) + with open(done_file, 'w') as fd: + if maybe_aborted: + # Special code to indicate we need to retry + fd.write('%s' % RETURN_CODE_RETRY) + else: + fd.write('%s' % result.return_code) + with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: + print('gcc', result.toolchain.gcc, file=fd) + print('path', result.toolchain.path, file=fd) + print('cross', result.toolchain.cross, file=fd) + print('arch', result.toolchain.arch, file=fd) + fd.write('%s' % result.return_code) + + # Write out the image and function size information and an objdump + env = result.toolchain.MakeEnvironment(self.builder.full_path) + with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: + for var in sorted(env.keys()): + fd.write(b'%s="%s"' % (var, env[var])) + lines = [] + for fname in BASE_ELF_FILENAMES: + cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] + nm_result = command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + if nm_result.stdout: + nm = self.builder.GetFuncSizesFile(result.commit_upto, + result.brd.target, fname) + with open(nm, 'w') as fd: + print(nm_result.stdout, end=' ', file=fd) + + cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] + dump_result = command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + rodata_size = '' + if dump_result.stdout: + objdump = self.builder.GetObjdumpFile(result.commit_upto, + result.brd.target, fname) + with open(objdump, 'w') as fd: + print(dump_result.stdout, end=' ', file=fd) + for line in dump_result.stdout.splitlines(): + fields = line.split() + if len(fields) > 5 and fields[1] == '.rodata': + rodata_size = fields[2] + + cmd = ['%ssize' % self.toolchain.cross, fname] + size_result = command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + if size_result.stdout: + lines.append(size_result.stdout.splitlines()[1] + ' ' + + rodata_size) + + # Extract the environment from U-Boot and dump it out + cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', + '-j', '.rodata.default_environment', + 'env/built-in.o', 'uboot.env'] + command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + ubootenv = os.path.join(result.out_dir, 'uboot.env') + if not work_in_output: + self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) + + # Write out the image sizes file. This is similar to the output + # of binutil's 'size' utility, but it omits the header line and + # adds an additional hex value at the end of each line for the + # rodata size + if len(lines): + sizes = self.builder.GetSizesFile(result.commit_upto, + result.brd.target) + with open(sizes, 'w') as fd: + print('\n'.join(lines), file=fd) + + if not work_in_output: + # Write out the configuration files, with a special case for SPL + for dirname in ['', 'spl', 'tpl']: + self.CopyFiles( + result.out_dir, build_dir, dirname, + ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', + '.config', 'include/autoconf.mk', + 'include/generated/autoconf.h']) + + # Now write the actual build output + if keep_outputs: + self.CopyFiles( + result.out_dir, build_dir, '', + ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', + 'include/autoconf.mk', 'spl/u-boot-spl*']) + + def CopyFiles(self, out_dir, build_dir, dirname, patterns): + """Copy files from the build directory to the output. + + Args: + out_dir: Path to output directory containing the files + build_dir: Place to copy the files + dirname: Source directory, '' for normal U-Boot, 'spl' for SPL + patterns: A list of filenames (strings) to copy, each relative + to the build directory + """ + for pattern in patterns: + file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) + for fname in file_list: + target = os.path.basename(fname) + if dirname: + base, ext = os.path.splitext(target) + if ext: + target = '%s-%s%s' % (base, dirname, ext) + shutil.copy(fname, os.path.join(build_dir, target)) + + def _SendResult(self, result): + """Send a result to the builder for processing + + Args: + result: CommandResult object containing the results of the build + + Raises: + ValueError if self.test_exception is true (for testing) + """ + if self.test_exception: + raise ValueError('test exception') + if self.thread_num != -1: + self.builder.out_queue.put(result) + else: + self.builder.ProcessResult(result) + + def RunJob(self, job): + """Run a single job + + A job consists of a building a list of commits for a particular board. + + Args: + job: Job to build + + Returns: + List of Result objects + """ + brd = job.board + work_dir = self.builder.GetThreadDir(self.thread_num) + self.toolchain = None + if job.commits: + # Run 'make board_defconfig' on the first commit + do_config = True + commit_upto = 0 + force_build = False + for commit_upto in range(0, len(job.commits), job.step): + result, request_config = self.RunCommit(commit_upto, brd, + work_dir, do_config, self.builder.config_only, + force_build or self.builder.force_build, + self.builder.force_build_failures, + work_in_output=job.work_in_output) + failed = result.return_code or result.stderr + did_config = do_config + if failed and not do_config: + # If our incremental build failed, try building again + # with a reconfig. + if self.builder.force_config_on_failure: + result, request_config = self.RunCommit(commit_upto, + brd, work_dir, True, False, True, False, + work_in_output=job.work_in_output) + did_config = True + if not self.builder.force_reconfig: + do_config = request_config + + # If we built that commit, then config is done. But if we got + # an warning, reconfig next time to force it to build the same + # files that created warnings this time. Otherwise an + # incremental build may not build the same file, and we will + # think that the warning has gone away. + # We could avoid this by using -Werror everywhere... + # For errors, the problem doesn't happen, since presumably + # the build stopped and didn't generate output, so will retry + # that file next time. So we could detect warnings and deal + # with them specially here. For now, we just reconfigure if + # anything goes work. + # Of course this is substantially slower if there are build + # errors/warnings (e.g. 2-3x slower even if only 10% of builds + # have problems). + if (failed and not result.already_done and not did_config and + self.builder.force_config_on_failure): + # If this build failed, try the next one with a + # reconfigure. + # Sometimes if the board_config.h file changes it can mess + # with dependencies, and we get: + # make: *** No rule to make target `include/autoconf.mk', + # needed by `depend'. + do_config = True + force_build = True + else: + force_build = False + if self.builder.force_config_on_failure: + if failed: + do_config = True + result.commit_upto = commit_upto + if result.return_code < 0: + raise ValueError('Interrupt') + + # We have the build results, so output the result + self._WriteResult(result, job.keep_outputs, job.work_in_output) + self._SendResult(result) + else: + # Just build the currently checked-out build + result, request_config = self.RunCommit(None, brd, work_dir, True, + self.builder.config_only, True, + self.builder.force_build_failures, + work_in_output=job.work_in_output) + result.commit_upto = 0 + self._WriteResult(result, job.keep_outputs, job.work_in_output) + self._SendResult(result) + + def run(self): + """Our thread's run function + + This thread picks a job from the queue, runs it, and then goes to the + next job. + """ + while True: + job = self.builder.queue.get() + try: + self.RunJob(job) + except Exception as e: + print('Thread exception:', e) + self.builder.thread_exceptions.append(e) + self.builder.queue.task_done() diff --git a/roms/u-boot/tools/buildman/buildman b/roms/u-boot/tools/buildman/buildman new file mode 120000 index 000000000..11a5d8e18 --- /dev/null +++ b/roms/u-boot/tools/buildman/buildman @@ -0,0 +1 @@ +main.py
\ No newline at end of file diff --git a/roms/u-boot/tools/buildman/cmdline.py b/roms/u-boot/tools/buildman/cmdline.py new file mode 100644 index 000000000..274b5ac3f --- /dev/null +++ b/roms/u-boot/tools/buildman/cmdline.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2014 Google, Inc +# + +from optparse import OptionParser + +def ParseArgs(): + """Parse command line arguments from sys.argv[] + + Returns: + tuple containing: + options: command line options + args: command lin arguments + """ + parser = OptionParser() + parser.add_option('-A', '--print-prefix', action='store_true', + help='Print the tool-chain prefix for a board (CROSS_COMPILE=)') + parser.add_option('-b', '--branch', type='string', + help='Branch name to build, or range of commits to build') + parser.add_option('-B', '--bloat', dest='show_bloat', + action='store_true', default=False, + help='Show changes in function code size for each board') + parser.add_option('--boards', type='string', action='append', + help='List of board names to build separated by comma') + parser.add_option('-c', '--count', dest='count', type='int', + default=-1, help='Run build on the top n commits') + parser.add_option('-C', '--force-reconfig', dest='force_reconfig', + action='store_true', default=False, + help='Reconfigure for every commit (disable incremental build)') + parser.add_option('-d', '--detail', dest='show_detail', + action='store_true', default=False, + help='Show detailed size delta for each board in the -S summary') + parser.add_option('-D', '--config-only', action='store_true', default=False, + help="Don't build, just configure each commit") + parser.add_option('-e', '--show_errors', action='store_true', + default=False, help='Show errors and warnings') + parser.add_option('-E', '--warnings-as-errors', action='store_true', + default=False, help='Treat all compiler warnings as errors') + parser.add_option('-f', '--force-build', dest='force_build', + action='store_true', default=False, + help='Force build of boards even if already built') + parser.add_option('-F', '--force-build-failures', dest='force_build_failures', + action='store_true', default=False, + help='Force build of previously-failed build') + parser.add_option('--fetch-arch', type='string', + help="Fetch a toolchain for architecture FETCH_ARCH ('list' to list)." + ' You can also fetch several toolchains separate by comma, or' + " 'all' to download all") + parser.add_option('-g', '--git', type='string', + help='Git repo containing branch to build', default='.') + parser.add_option('-G', '--config-file', type='string', + help='Path to buildman config file', default='') + parser.add_option('-H', '--full-help', action='store_true', dest='full_help', + default=False, help='Display the README file') + parser.add_option('-i', '--in-tree', dest='in_tree', + action='store_true', default=False, + help='Build in the source tree instead of a separate directory') + # -I will be removed after April 2021 + parser.add_option('-I', '--incremental', action='store_true', + default=False, help='Deprecated, does nothing. See -m') + parser.add_option('-j', '--jobs', dest='jobs', type='int', + default=None, help='Number of jobs to run at once (passed to make)') + parser.add_option('-k', '--keep-outputs', action='store_true', + default=False, help='Keep all build output files (e.g. binaries)') + parser.add_option('-K', '--show-config', action='store_true', + default=False, help='Show configuration changes in summary (both board config files and Kconfig)') + parser.add_option('--preserve-config-y', action='store_true', + default=False, help="Don't convert y to 1 in configs") + parser.add_option('-l', '--list-error-boards', action='store_true', + default=False, help='Show a list of boards next to each error/warning') + parser.add_option('--list-tool-chains', action='store_true', default=False, + help='List available tool chains (use -v to see probing detail)') + parser.add_option('-m', '--mrproper', action='store_true', + default=False, help="Run 'make mrproper before reconfiguring") + parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', + default=False, help="Do a dry run (describe actions, but do nothing)") + parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', + default=False, help="Don't create subdirectories when building current source for a single board") + parser.add_option('-o', '--output-dir', type='string', dest='output_dir', + help='Directory where all builds happen and buildman has its workspace (default is ../)') + parser.add_option('-O', '--override-toolchain', type='string', + help="Override host toochain to use for sandbox (e.g. 'clang-7')") + parser.add_option('-Q', '--quick', action='store_true', + default=False, help='Do a rough build, with limited warning resolution') + parser.add_option('-p', '--full-path', action='store_true', + default=False, help="Use full toolchain path in CROSS_COMPILE") + parser.add_option('-P', '--per-board-out-dir', action='store_true', + default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-s', '--summary', action='store_true', + default=False, help='Show a build summary') + parser.add_option('-S', '--show-sizes', action='store_true', + default=False, help='Show image size variation in summary') + parser.add_option('--skip-net-tests', action='store_true', default=False, + help='Skip tests which need the network') + parser.add_option('--step', type='int', + default=1, help='Only build every n commits (0=just first and last)') + parser.add_option('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') + parser.add_option('-T', '--threads', type='int', + default=None, + help='Number of builder threads to use (0=single-thread)') + parser.add_option('-u', '--show_unknown', action='store_true', + default=False, help='Show boards with unknown build result') + parser.add_option('-U', '--show-environment', action='store_true', + default=False, help='Show environment changes in summary') + parser.add_option('-v', '--verbose', action='store_true', + default=False, help='Show build results while the build progresses') + parser.add_option('-V', '--verbose-build', action='store_true', + default=False, help='Run make with V=1, logging all output') + parser.add_option('-w', '--work-in-output', action='store_true', + default=False, help='Use the output directory as the work directory') + parser.add_option('-W', '--ignore-warnings', action='store_true', + default=False, help='Return success even if there are warnings') + parser.add_option('-x', '--exclude', dest='exclude', + type='string', action='append', + help='Specify a list of boards to exclude, separated by comma') + parser.add_option('-y', '--filter-dtb-warnings', action='store_true', + default=False, + help='Filter out device-tree-compiler warnings from output') + parser.add_option('-Y', '--filter-migration-warnings', action='store_true', + default=False, + help='Filter out migration warnings from output') + + parser.usage += """ [list of target/arch/cpu/board/vendor/soc to build] + + Build U-Boot for all commits in a branch. Use -n to do a dry run""" + + return parser.parse_args() diff --git a/roms/u-boot/tools/buildman/control.py b/roms/u-boot/tools/buildman/control.py new file mode 100644 index 000000000..a98d1b4c0 --- /dev/null +++ b/roms/u-boot/tools/buildman/control.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2013 The Chromium OS Authors. +# + +import multiprocessing +import os +import shutil +import subprocess +import sys + +from buildman import board +from buildman import bsettings +from buildman import toolchain +from buildman.builder import Builder +from patman import command +from patman import gitutil +from patman import patchstream +from patman import terminal +from patman.terminal import Print + +def GetPlural(count): + """Returns a plural 's' if count is not 1""" + return 's' if count != 1 else '' + +def GetActionSummary(is_summary, commits, selected, options): + """Return a string summarising the intended action. + + Returns: + Summary string. + """ + if commits: + count = len(commits) + count = (count + options.step - 1) // options.step + commit_str = '%d commit%s' % (count, GetPlural(count)) + else: + commit_str = 'current source' + str = '%s %s for %d boards' % ( + 'Summary of' if is_summary else 'Building', commit_str, + len(selected)) + str += ' (%d thread%s, %d job%s per thread)' % (options.threads, + GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) + return str + +def ShowActions(series, why_selected, boards_selected, builder, options, + board_warnings): + """Display a list of actions that we would take, if not a dry run. + + Args: + series: Series object + why_selected: Dictionary where each key is a buildman argument + provided by the user, and the value is the list of boards + brought in by that argument. For example, 'arm' might bring + in 400 boards, so in this case the key would be 'arm' and + the value would be a list of board names. + boards_selected: Dict of selected boards, key is target name, + value is Board object + builder: The builder that will be used to build the commits + options: Command line options object + board_warnings: List of warnings obtained from board selected + """ + col = terminal.Color() + print('Dry run, so not doing much. But I would do this:') + print() + if series: + commits = series.commits + else: + commits = None + print(GetActionSummary(False, commits, boards_selected, + options)) + print('Build directory: %s' % builder.base_dir) + if commits: + for upto in range(0, len(series.commits), options.step): + commit = series.commits[upto] + print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ') + print(commit.subject) + print() + for arg in why_selected: + if arg != 'all': + print(arg, ': %d boards' % len(why_selected[arg])) + if options.verbose: + print(' %s' % ' '.join(why_selected[arg])) + print(('Total boards to build for each commit: %d\n' % + len(why_selected['all']))) + if board_warnings: + for warning in board_warnings: + print(col.Color(col.YELLOW, warning)) + +def ShowToolchainPrefix(boards, toolchains): + """Show information about a the tool chain used by one or more boards + + The function checks that all boards use the same toolchain, then prints + the correct value for CROSS_COMPILE. + + Args: + boards: Boards object containing selected boards + toolchains: Toolchains object containing available toolchains + + Return: + None on success, string error message otherwise + """ + boards = boards.GetSelectedDict() + tc_set = set() + for brd in boards.values(): + tc_set.add(toolchains.Select(brd.arch)) + if len(tc_set) != 1: + return 'Supplied boards must share one toolchain' + return False + tc = tc_set.pop() + print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + return None + +def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, + clean_dir=False, test_thread_exceptions=False): + """The main control code for buildman + + Args: + options: Command line options object + args: Command line arguments (list of strings) + toolchains: Toolchains to use - this should be a Toolchains() + object. If None, then it will be created and scanned + make_func: Make function to use for the builder. This is called + to execute 'make'. If this is None, the normal function + will be used, which calls the 'make' tool with suitable + arguments. This setting is useful for tests. + board: Boards() object to use, containing a list of available + boards. If this is None it will be created and scanned. + clean_dir: Used for tests only, indicates that the existing output_dir + should be removed before starting the build + test_thread_exceptions: Uses for tests only, True to make the threads + raise an exception instead of reporting their result. This simulates + a failure in the code somewhere + """ + global builder + + if options.full_help: + pager = os.getenv('PAGER') + if not pager: + pager = 'more' + fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), + 'README') + command.Run(pager, fname) + return 0 + + gitutil.Setup() + col = terminal.Color() + + options.git_dir = os.path.join(options.git, '.git') + + no_toolchains = toolchains is None + if no_toolchains: + toolchains = toolchain.Toolchains(options.override_toolchain) + + if options.fetch_arch: + if options.fetch_arch == 'list': + sorted_list = toolchains.ListArchs() + print(col.Color(col.BLUE, 'Available architectures: %s\n' % + ' '.join(sorted_list))) + return 0 + else: + fetch_arch = options.fetch_arch + if fetch_arch == 'all': + fetch_arch = ','.join(toolchains.ListArchs()) + print(col.Color(col.CYAN, '\nDownloading toolchains: %s' % + fetch_arch)) + for arch in fetch_arch.split(','): + print() + ret = toolchains.FetchAndInstall(arch) + if ret: + return ret + return 0 + + if no_toolchains: + toolchains.GetSettings() + toolchains.Scan(options.list_tool_chains and options.verbose) + if options.list_tool_chains: + toolchains.List() + print() + return 0 + + if options.incremental: + print(col.Color(col.RED, + 'Warning: -I has been removed. See documentation')) + if not options.output_dir: + if options.work_in_output: + sys.exit(col.Color(col.RED, '-w requires that you specify -o')) + options.output_dir = '..' + + # Work out what subset of the boards we are building + if not boards: + if not os.path.exists(options.output_dir): + os.makedirs(options.output_dir) + board_file = os.path.join(options.output_dir, 'boards.cfg') + our_path = os.path.dirname(os.path.realpath(__file__)) + genboardscfg = os.path.join(our_path, '../genboardscfg.py') + if not os.path.exists(genboardscfg): + genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') + status = subprocess.call([genboardscfg, '-q', '-o', board_file]) + if status != 0: + # Older versions don't support -q + status = subprocess.call([genboardscfg, '-o', board_file]) + if status != 0: + sys.exit("Failed to generate boards.cfg") + + boards = board.Boards() + boards.ReadBoards(board_file) + + exclude = [] + if options.exclude: + for arg in options.exclude: + exclude += arg.split(',') + + if options.boards: + requested_boards = [] + for b in options.boards: + requested_boards += b.split(',') + else: + requested_boards = None + why_selected, board_warnings = boards.SelectBoards(args, exclude, + requested_boards) + selected = boards.GetSelected() + if not len(selected): + sys.exit(col.Color(col.RED, 'No matching boards found')) + + if options.print_prefix: + err = ShowToolchainPrefix(boards, toolchains) + if err: + sys.exit(col.Color(col.RED, err)) + return 0 + + # Work out how many commits to build. We want to build everything on the + # branch. We also build the upstream commit as a control so we can see + # problems introduced by the first commit on the branch. + count = options.count + has_range = options.branch and '..' in options.branch + if count == -1: + if not options.branch: + count = 1 + else: + if has_range: + count, msg = gitutil.CountCommitsInRange(options.git_dir, + options.branch) + else: + count, msg = gitutil.CountCommitsInBranch(options.git_dir, + options.branch) + if count is None: + sys.exit(col.Color(col.RED, msg)) + elif count == 0: + sys.exit(col.Color(col.RED, "Range '%s' has no commits" % + options.branch)) + if msg: + print(col.Color(col.YELLOW, msg)) + count += 1 # Build upstream commit also + + if not count: + str = ("No commits found to process in branch '%s': " + "set branch's upstream or use -c flag" % options.branch) + sys.exit(col.Color(col.RED, str)) + if options.work_in_output: + if len(selected) != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single board')) + if count != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single commit')) + + # Read the metadata from the commits. First look at the upstream commit, + # then the ones in the branch. We would like to do something like + # upstream/master~..branch but that isn't possible if upstream/master is + # a merge commit (it will list all the commits that form part of the + # merge) + # Conflicting tags are not a problem for buildman, since it does not use + # them. For example, Series-version is not useful for buildman. On the + # other hand conflicting tags will cause an error. So allow later tags + # to overwrite earlier ones by setting allow_overwrite=True + if options.branch: + if count == -1: + if has_range: + range_expr = options.branch + else: + range_expr = gitutil.GetRangeInBranch(options.git_dir, + options.branch) + upstream_commit = gitutil.GetUpstream(options.git_dir, + options.branch) + series = patchstream.get_metadata_for_list(upstream_commit, + options.git_dir, 1, series=None, allow_overwrite=True) + + series = patchstream.get_metadata_for_list(range_expr, + options.git_dir, None, series, allow_overwrite=True) + else: + # Honour the count + series = patchstream.get_metadata_for_list(options.branch, + options.git_dir, count, series=None, allow_overwrite=True) + else: + series = None + if not options.dry_run: + options.verbose = True + if not options.summary: + options.show_errors = True + + # By default we have one thread per CPU. But if there are not enough jobs + # we can have fewer threads and use a high '-j' value for make. + if options.threads is None: + options.threads = min(multiprocessing.cpu_count(), len(selected)) + if not options.jobs: + options.jobs = max(1, (multiprocessing.cpu_count() + + len(selected) - 1) // len(selected)) + + if not options.step: + options.step = len(series.commits) - 1 + + gnu_make = command.Output(os.path.join(options.git, + 'scripts/show-gnu-make'), raise_on_error=False).rstrip() + if not gnu_make: + sys.exit('GNU Make not found') + + # Create a new builder with the selected options. + output_dir = options.output_dir + if options.branch: + dirname = options.branch.replace('/', '_') + # As a special case allow the board directory to be placed in the + # output directory itself rather than any subdirectory. + if not options.no_subdirs: + output_dir = os.path.join(options.output_dir, dirname) + if clean_dir and os.path.exists(output_dir): + shutil.rmtree(output_dir) + builder = Builder(toolchains, output_dir, options.git_dir, + options.threads, options.jobs, gnu_make=gnu_make, checkout=True, + show_unknown=options.show_unknown, step=options.step, + no_subdirs=options.no_subdirs, full_path=options.full_path, + verbose_build=options.verbose_build, + mrproper=options.mrproper, + per_board_out_dir=options.per_board_out_dir, + config_only=options.config_only, + squash_config_y=not options.preserve_config_y, + warnings_as_errors=options.warnings_as_errors, + work_in_output=options.work_in_output, + test_thread_exceptions=test_thread_exceptions) + builder.force_config_on_failure = not options.quick + if make_func: + builder.do_make = make_func + + # For a dry run, just show our actions as a sanity check + if options.dry_run: + ShowActions(series, why_selected, selected, builder, options, + board_warnings) + else: + builder.force_build = options.force_build + builder.force_build_failures = options.force_build_failures + builder.force_reconfig = options.force_reconfig + builder.in_tree = options.in_tree + + # Work out which boards to build + board_selected = boards.GetSelectedDict() + + if series: + commits = series.commits + # Number the commits for test purposes + for commit in range(len(commits)): + commits[commit].sequence = commit + else: + commits = None + + Print(GetActionSummary(options.summary, commits, board_selected, + options)) + + # We can't show function sizes without board details at present + if options.show_bloat: + options.show_detail = True + builder.SetDisplayOptions( + options.show_errors, options.show_sizes, options.show_detail, + options.show_bloat, options.list_error_boards, options.show_config, + options.show_environment, options.filter_dtb_warnings, + options.filter_migration_warnings) + if options.summary: + builder.ShowSummary(commits, board_selected) + else: + fail, warned, excs = builder.BuildBoards( + commits, board_selected, options.keep_outputs, options.verbose) + if excs: + return 102 + elif fail: + return 100 + elif warned and not options.ignore_warnings: + return 101 + return 0 diff --git a/roms/u-boot/tools/buildman/func_test.py b/roms/u-boot/tools/buildman/func_test.py new file mode 100644 index 000000000..7edbee065 --- /dev/null +++ b/roms/u-boot/tools/buildman/func_test.py @@ -0,0 +1,626 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2014 Google, Inc +# + +import os +import shutil +import sys +import tempfile +import unittest + +from buildman import board +from buildman import bsettings +from buildman import cmdline +from buildman import control +from buildman import toolchain +from patman import command +from patman import gitutil +from patman import terminal +from patman import test_util +from patman import tools + +settings_data = ''' +# Buildman settings file + +[toolchain] + +[toolchain-alias] + +[make-flags] +src=/home/sjg/c/src +chroot=/home/sjg/c/chroot +vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference +chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot} +chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot} +chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot} +''' + +boards = [ + ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], + ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], + ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], + ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''], +] + +commit_shortlog = """4aca821 patman: Avoid changing the order of tags +39403bb patman: Use --no-pager' to stop git from forking a pager +db6e6f2 patman: Remove the -a option +f2ccf03 patman: Correct unit tests to run correctly +1d097f9 patman: Fix indentation in terminal.py +d073747 patman: Support the 'reverse' option for 'git log +""" + +commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd +Author: Masahiro Yamada <yamada.m@jp.panasonic.com> +Date: Fri Aug 22 19:12:41 2014 +0900 + + buildman: refactor help message + + "buildman [options]" is displayed by default. + + Append the rest of help messages to parser.usage + instead of replacing it. + + Besides, "-b <branch>" is not mandatory since commit fea5858e. + Drop it from the usage. + + Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com> +""", +"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8 +Author: Simon Glass <sjg@chromium.org> +Date: Thu Aug 14 16:48:25 2014 -0600 + + patman: Support the 'reverse' option for 'git log' + + This option is currently not supported, but needs to be, for buildman to + operate as expected. + + Series-changes: 7 + - Add new patch to fix the 'reverse' bug + + Series-version: 8 + + Change-Id: I79078f792e8b390b8a1272a8023537821d45feda + Reported-by: York Sun <yorksun@freescale.com> + Signed-off-by: Simon Glass <sjg@chromium.org> + +""", +"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc +Author: Simon Glass <sjg@chromium.org> +Date: Sat Aug 9 11:44:32 2014 -0600 + + patman: Fix indentation in terminal.py + + This code came from a different project with 2-character indentation. Fix + it for U-Boot. + + Series-changes: 6 + - Add new patch to fix indentation in teminal.py + + Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34 + Signed-off-by: Simon Glass <sjg@chromium.org> + +""", +"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105 +Author: Simon Glass <sjg@chromium.org> +Date: Sat Aug 9 11:08:24 2014 -0600 + + patman: Correct unit tests to run correctly + + It seems that doctest behaves differently now, and some of the unit tests + do not run. Adjust the tests to work correctly. + + ./tools/patman/patman --test + <unittest.result.TestResult run=10 errors=0 failures=0> + + Series-changes: 6 + - Add new patch to fix patman unit tests + + Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b + +""", +"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c +Author: Simon Glass <sjg@chromium.org> +Date: Sat Aug 9 12:06:02 2014 -0600 + + patman: Remove the -a option + + It seems that this is no longer needed, since checkpatch.pl will catch + whitespace problems in patches. Also the option is not widely used, so + it seems safe to just remove it. + + Series-changes: 6 + - Add new patch to remove patman's -a option + + Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com> + Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc + +""", +"""commit 39403bb4f838153028a6f21ca30bf100f3791133 +Author: Simon Glass <sjg@chromium.org> +Date: Thu Aug 14 21:50:52 2014 -0600 + + patman: Use --no-pager' to stop git from forking a pager + +""", +"""commit 4aca821e27e97925c039e69fd37375b09c6f129c +Author: Simon Glass <sjg@chromium.org> +Date: Fri Aug 22 15:57:39 2014 -0600 + + patman: Avoid changing the order of tags + + patman collects tags that it sees in the commit and places them nicely + sorted at the end of the patch. However, this is not really necessary and + in fact is apparently not desirable. + + Series-changes: 9 + - Add new patch to avoid changing the order of tags + + Series-version: 9 + + Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com> + Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db +"""] + +TEST_BRANCH = '__testbranch' + +class TestFunctional(unittest.TestCase): + """Functional test for buildman. + + This aims to test from just below the invocation of buildman (parsing + of arguments) to 'make' and 'git' invocation. It is not a true + emd-to-end test, as it mocks git, make and the tool chain. But this + makes it easier to detect when the builder is doing the wrong thing, + since in many cases this test code will fail. For example, only a + very limited subset of 'git' arguments is supported - anything + unexpected will fail. + """ + def setUp(self): + self._base_dir = tempfile.mkdtemp() + self._output_dir = tempfile.mkdtemp() + self._git_dir = os.path.join(self._base_dir, 'src') + self._buildman_pathname = sys.argv[0] + self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) + command.test_result = self._HandleCommand + self.setupToolchains() + self._toolchains.Add('arm-gcc', test=False) + self._toolchains.Add('powerpc-gcc', test=False) + bsettings.Setup(None) + bsettings.AddFile(settings_data) + self._boards = board.Boards() + for brd in boards: + self._boards.AddBoard(board.Board(*brd)) + + # Directories where the source been cloned + self._clone_dirs = [] + self._commits = len(commit_shortlog.splitlines()) + 1 + self._total_builds = self._commits * len(boards) + + # Number of calls to make + self._make_calls = 0 + + # Map of [board, commit] to error messages + self._error = {} + + self._test_branch = TEST_BRANCH + + # Avoid sending any output and clear all terminal output + terminal.SetPrintTestMode() + terminal.GetPrintTestLines() + + def tearDown(self): + shutil.rmtree(self._base_dir) + #shutil.rmtree(self._output_dir) + + def setupToolchains(self): + self._toolchains = toolchain.Toolchains() + self._toolchains.Add('gcc', test=False) + + def _RunBuildman(self, *args): + return command.RunPipe([[self._buildman_pathname] + list(args)], + capture=True, capture_stderr=True) + + def _RunControl(self, *args, boards=None, clean_dir=False, + test_thread_exceptions=False): + """Run buildman + + Args: + args: List of arguments to pass + boards: + clean_dir: Used for tests only, indicates that the existing output_dir + should be removed before starting the build + test_thread_exceptions: Uses for tests only, True to make the threads + raise an exception instead of reporting their result. This simulates + a failure in the code somewhere + + Returns: + result code from buildman + """ + sys.argv = [sys.argv[0]] + list(args) + options, args = cmdline.ParseArgs() + result = control.DoBuildman(options, args, toolchains=self._toolchains, + make_func=self._HandleMake, boards=boards or self._boards, + clean_dir=clean_dir, + test_thread_exceptions=test_thread_exceptions) + self._builder = control.builder + return result + + def testFullHelp(self): + command.test_result = None + result = self._RunBuildman('-H') + help_file = os.path.join(self._buildman_dir, 'README') + # Remove possible extraneous strings + extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n' + gothelp = result.stdout.replace(extra, '') + self.assertEqual(len(gothelp), os.path.getsize(help_file)) + self.assertEqual(0, len(result.stderr)) + self.assertEqual(0, result.return_code) + + def testHelp(self): + command.test_result = None + result = self._RunBuildman('-h') + help_file = os.path.join(self._buildman_dir, 'README') + self.assertTrue(len(result.stdout) > 1000) + self.assertEqual(0, len(result.stderr)) + self.assertEqual(0, result.return_code) + + def testGitSetup(self): + """Test gitutils.Setup(), from outside the module itself""" + command.test_result = command.CommandResult(return_code=1) + gitutil.Setup() + self.assertEqual(gitutil.use_no_decorate, False) + + command.test_result = command.CommandResult(return_code=0) + gitutil.Setup() + self.assertEqual(gitutil.use_no_decorate, True) + + def _HandleCommandGitLog(self, args): + if args[-1] == '--': + args = args[:-1] + if '-n0' in args: + return command.CommandResult(return_code=0) + elif args[-1] == 'upstream/master..%s' % self._test_branch: + return command.CommandResult(return_code=0, stdout=commit_shortlog) + elif args[:3] == ['--no-color', '--no-decorate', '--reverse']: + if args[-1] == self._test_branch: + count = int(args[3][2:]) + return command.CommandResult(return_code=0, + stdout=''.join(commit_log[:count])) + + # Not handled, so abort + print('git log', args) + sys.exit(1) + + def _HandleCommandGitConfig(self, args): + config = args[0] + if config == 'sendemail.aliasesfile': + return command.CommandResult(return_code=0) + elif config.startswith('branch.badbranch'): + return command.CommandResult(return_code=1) + elif config == 'branch.%s.remote' % self._test_branch: + return command.CommandResult(return_code=0, stdout='upstream\n') + elif config == 'branch.%s.merge' % self._test_branch: + return command.CommandResult(return_code=0, + stdout='refs/heads/master\n') + + # Not handled, so abort + print('git config', args) + sys.exit(1) + + def _HandleCommandGit(self, in_args): + """Handle execution of a git command + + This uses a hacked-up parser. + + Args: + in_args: Arguments after 'git' from the command line + """ + git_args = [] # Top-level arguments to git itself + sub_cmd = None # Git sub-command selected + args = [] # Arguments to the git sub-command + for arg in in_args: + if sub_cmd: + args.append(arg) + elif arg[0] == '-': + git_args.append(arg) + else: + if git_args and git_args[-1] in ['--git-dir', '--work-tree']: + git_args.append(arg) + else: + sub_cmd = arg + if sub_cmd == 'config': + return self._HandleCommandGitConfig(args) + elif sub_cmd == 'log': + return self._HandleCommandGitLog(args) + elif sub_cmd == 'clone': + return command.CommandResult(return_code=0) + elif sub_cmd == 'checkout': + return command.CommandResult(return_code=0) + elif sub_cmd == 'worktree': + return command.CommandResult(return_code=0) + + # Not handled, so abort + print('git', git_args, sub_cmd, args) + sys.exit(1) + + def _HandleCommandNm(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommandObjdump(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommandObjcopy(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommandSize(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommand(self, **kwargs): + """Handle a command execution. + + The command is in kwargs['pipe-list'], as a list of pipes, each a + list of commands. The command should be emulated as required for + testing purposes. + + Returns: + A CommandResult object + """ + pipe_list = kwargs['pipe_list'] + wc = False + if len(pipe_list) != 1: + if pipe_list[1] == ['wc', '-l']: + wc = True + else: + print('invalid pipe', kwargs) + sys.exit(1) + cmd = pipe_list[0][0] + args = pipe_list[0][1:] + result = None + if cmd == 'git': + result = self._HandleCommandGit(args) + elif cmd == './scripts/show-gnu-make': + return command.CommandResult(return_code=0, stdout='make') + elif cmd.endswith('nm'): + return self._HandleCommandNm(args) + elif cmd.endswith('objdump'): + return self._HandleCommandObjdump(args) + elif cmd.endswith('objcopy'): + return self._HandleCommandObjcopy(args) + elif cmd.endswith( 'size'): + return self._HandleCommandSize(args) + + if not result: + # Not handled, so abort + print('unknown command', kwargs) + sys.exit(1) + + if wc: + result.stdout = len(result.stdout.splitlines()) + return result + + def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs): + """Handle execution of 'make' + + Args: + commit: Commit object that is being built + brd: Board object that is being built + stage: Stage that we are at (mrproper, config, build) + cwd: Directory where make should be run + args: Arguments to pass to make + kwargs: Arguments to pass to command.RunPipe() + """ + self._make_calls += 1 + if stage == 'mrproper': + return command.CommandResult(return_code=0) + elif stage == 'config': + return command.CommandResult(return_code=0, + combined='Test configuration complete') + elif stage == 'build': + stderr = '' + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] + fname = os.path.join(cwd or '', out_dir, 'u-boot') + tools.WriteFile(fname, b'U-Boot') + if type(commit) is not str: + stderr = self._error.get((brd.target, commit.sequence)) + if stderr: + return command.CommandResult(return_code=1, stderr=stderr) + return command.CommandResult(return_code=0) + + # Not handled, so abort + print('make', stage) + sys.exit(1) + + # Example function to print output lines + def print_lines(self, lines): + print(len(lines)) + for line in lines: + print(line) + #self.print_lines(terminal.GetPrintTestLines()) + + def testNoBoards(self): + """Test that buildman aborts when there are no boards""" + self._boards = board.Boards() + with self.assertRaises(SystemExit): + self._RunControl() + + def testCurrentSource(self): + """Very simple test to invoke buildman on the current source""" + self.setupToolchains(); + self._RunControl('-o', self._output_dir) + lines = terminal.GetPrintTestLines() + self.assertIn('Building current source for %d boards' % len(boards), + lines[0].text) + + def testBadBranch(self): + """Test that we can detect an invalid branch""" + with self.assertRaises(ValueError): + self._RunControl('-b', 'badbranch') + + def testBadToolchain(self): + """Test that missing toolchains are detected""" + self.setupToolchains(); + ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) + lines = terminal.GetPrintTestLines() + + # Buildman always builds the upstream commit as well + self.assertIn('Building %d commits for %d boards' % + (self._commits, len(boards)), lines[0].text) + self.assertEqual(self._builder.count, self._total_builds) + + # Only sandbox should succeed, the others don't have toolchains + self.assertEqual(self._builder.fail, + self._total_builds - self._commits) + self.assertEqual(ret_code, 100) + + for commit in range(self._commits): + for board in self._boards.GetList(): + if board.arch != 'sandbox': + errfile = self._builder.GetErrFile(commit, board.target) + fd = open(errfile) + self.assertEqual(fd.readlines(), + ['No tool chain for %s\n' % board.arch]) + fd.close() + + def testBranch(self): + """Test building a branch with all toolchains present""" + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._builder.fail, 0) + + def testCount(self): + """Test building a specific number of commitst""" + self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir) + self.assertEqual(self._builder.count, 2 * len(boards)) + self.assertEqual(self._builder.fail, 0) + # Each board has a config, and then one make per commit + self.assertEqual(self._make_calls, len(boards) * (1 + 2)) + + def testIncremental(self): + """Test building a branch twice - the second time should do nothing""" + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) + + # Each board has a mrproper, config, and then one make per commit + self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) + self._make_calls = 0 + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) + self.assertEqual(self._make_calls, 0) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._builder.fail, 0) + + def testForceBuild(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) + self._make_calls = 0 + self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False) + # Each board has a config and one make per commit + self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) + + def testForceReconfigure(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) + # Each commit has a config and make + self.assertEqual(self._make_calls, len(boards) * self._commits * 2) + + def testForceReconfigure(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) + # Each commit has a config and make + self.assertEqual(self._make_calls, len(boards) * self._commits * 2) + + def testMrproper(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir) + # Each board has a mkproper, config and then one make per commit + self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) + + def testErrors(self): + """Test handling of build errors""" + self._error['board2', 1] = 'fred\n' + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._builder.fail, 1) + + # Remove the error. This should have no effect since the commit will + # not be rebuilt + del self._error['board2', 1] + self._make_calls = 0 + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._make_calls, 0) + self.assertEqual(self._builder.fail, 1) + + # Now use the -F flag to force rebuild of the bad commit + self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._builder.fail, 0) + self.assertEqual(self._make_calls, 2) + + def testBranchWithSlash(self): + """Test building a branch with a '/' in the name""" + self._test_branch = '/__dev/__testbranch' + self._RunControl('-b', self._test_branch, clean_dir=False) + self.assertEqual(self._builder.count, self._total_builds) + self.assertEqual(self._builder.fail, 0) + + def testEnvironment(self): + """Test that the done and environment files are written to out-env""" + self._RunControl('-o', self._output_dir) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env'))) + + def testEnvironmentUnicode(self): + """Test there are no unicode errors when the env has non-ASCII chars""" + try: + varname = b'buildman_test_var' + os.environb[varname] = b'strange\x80chars' + self.assertEqual(0, self._RunControl('-o', self._output_dir)) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env'))) + finally: + del os.environb[varname] + + def testWorkInOutput(self): + """Test the -w option which should write directly to the output dir""" + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + self._RunControl('-o', self._output_dir, '-w', clean_dir=False, + boards=board_list) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'done'))) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'out-env'))) + + def testWorkInOutputFail(self): + """Test the -w option failures""" + with self.assertRaises(SystemExit) as e: + self._RunControl('-o', self._output_dir, '-w', clean_dir=False) + self.assertIn("single board", str(e.exception)) + self.assertFalse( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + with self.assertRaises(SystemExit) as e: + self._RunControl('-b', self._test_branch, '-o', self._output_dir, + '-w', clean_dir=False, boards=board_list) + self.assertIn("single commit", str(e.exception)) + + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + with self.assertRaises(SystemExit) as e: + self._RunControl('-w', clean_dir=False) + self.assertIn("specify -o", str(e.exception)) + + def testThreadExceptions(self): + """Test that exceptions in threads are reported""" + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(102, self._RunControl('-o', self._output_dir, + test_thread_exceptions=True)) + self.assertIn('Thread exception: test exception', stdout.getvalue()) diff --git a/roms/u-boot/tools/buildman/kconfiglib.py b/roms/u-boot/tools/buildman/kconfiglib.py new file mode 100644 index 000000000..c67895ced --- /dev/null +++ b/roms/u-boot/tools/buildman/kconfiglib.py @@ -0,0 +1,7160 @@ +# Copyright (c) 2011-2019, Ulf Magnusson +# SPDX-License-Identifier: ISC + +""" +Overview +======== + +Kconfiglib is a Python 2/3 library for scripting and extracting information +from Kconfig (https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt) +configuration systems. + +See the homepage at https://github.com/ulfalizer/Kconfiglib for a longer +overview. + +Since Kconfiglib 12.0.0, the library version is available in +kconfiglib.VERSION, which is a (<major>, <minor>, <patch>) tuple, e.g. +(12, 0, 0). + + +Using Kconfiglib on the Linux kernel with the Makefile targets +============================================================== + +For the Linux kernel, a handy interface is provided by the +scripts/kconfig/Makefile patch, which can be applied with either 'git am' or +the 'patch' utility: + + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | git am + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | patch -p1 + +Warning: Not passing -p1 to patch will cause the wrong file to be patched. + +Please tell me if the patch does not apply. It should be trivial to apply +manually, as it's just a block of text that needs to be inserted near the other +*conf: targets in scripts/kconfig/Makefile. + +Look further down for a motivation for the Makefile patch and for instructions +on how you can use Kconfiglib without it. + +If you do not wish to install Kconfiglib via pip, the Makefile patch is set up +so that you can also just clone Kconfiglib into the kernel root: + + $ git clone git://github.com/ulfalizer/Kconfiglib.git + $ git am Kconfiglib/makefile.patch (or 'patch -p1 < Kconfiglib/makefile.patch') + +Warning: The directory name Kconfiglib/ is significant in this case, because +it's added to PYTHONPATH by the new targets in makefile.patch. + +The targets added by the Makefile patch are described in the following +sections. + + +make kmenuconfig +---------------- + +This target runs the curses menuconfig interface with Python 3. As of +Kconfiglib 12.2.0, both Python 2 and Python 3 are supported (previously, only +Python 3 was supported, so this was a backport). + + +make guiconfig +-------------- + +This target runs the Tkinter menuconfig interface. Both Python 2 and Python 3 +are supported. To change the Python interpreter used, pass +PYTHONCMD=<executable> to 'make'. The default is 'python'. + + +make [ARCH=<arch>] iscriptconfig +-------------------------------- + +This target gives an interactive Python prompt where a Kconfig instance has +been preloaded and is available in 'kconf'. To change the Python interpreter +used, pass PYTHONCMD=<executable> to 'make'. The default is 'python'. + +To get a feel for the API, try evaluating and printing the symbols in +kconf.defined_syms, and explore the MenuNode menu tree starting at +kconf.top_node by following 'next' and 'list' pointers. + +The item contained in a menu node is found in MenuNode.item (note that this can +be one of the constants kconfiglib.MENU and kconfiglib.COMMENT), and all +symbols and choices have a 'nodes' attribute containing their menu nodes +(usually only one). Printing a menu node will print its item, in Kconfig +format. + +If you want to look up a symbol by name, use the kconf.syms dictionary. + + +make scriptconfig SCRIPT=<script> [SCRIPT_ARG=<arg>] +---------------------------------------------------- + +This target runs the Python script given by the SCRIPT parameter on the +configuration. sys.argv[1] holds the name of the top-level Kconfig file +(currently always "Kconfig" in practice), and sys.argv[2] holds the SCRIPT_ARG +argument, if given. + +See the examples/ subdirectory for example scripts. + + +make dumpvarsconfig +------------------- + +This target prints a list of all environment variables referenced from the +Kconfig files, together with their values. See the +Kconfiglib/examples/dumpvars.py script. + +Only environment variables that are referenced via the Kconfig preprocessor +$(FOO) syntax are included. The preprocessor was added in Linux 4.18. + + +Using Kconfiglib without the Makefile targets +============================================= + +The make targets are only needed to pick up environment variables exported from +the Kbuild makefiles and referenced inside Kconfig files, via e.g. +'source "arch/$(SRCARCH)/Kconfig" and commands run via '$(shell,...)'. + +These variables are referenced as of writing (Linux 4.18), together with sample +values: + + srctree (.) + ARCH (x86) + SRCARCH (x86) + KERNELVERSION (4.18.0) + CC (gcc) + HOSTCC (gcc) + HOSTCXX (g++) + CC_VERSION_TEXT (gcc (Ubuntu 7.3.0-16ubuntu3) 7.3.0) + +Older kernels only reference ARCH, SRCARCH, and KERNELVERSION. + +If your kernel is recent enough (4.18+), you can get a list of referenced +environment variables via 'make dumpvarsconfig' (see above). Note that this +command is added by the Makefile patch. + +To run Kconfiglib without the Makefile patch, set the environment variables +manually: + + $ srctree=. ARCH=x86 SRCARCH=x86 KERNELVERSION=`make kernelversion` ... python(3) + >>> import kconfiglib + >>> kconf = kconfiglib.Kconfig() # filename defaults to "Kconfig" + +Search the top-level Makefile for "Additional ARCH settings" to see other +possibilities for ARCH and SRCARCH. + + +Intro to symbol values +====================== + +Kconfiglib has the same assignment semantics as the C implementation. + +Any symbol can be assigned a value by the user (via Kconfig.load_config() or +Symbol.set_value()), but this user value is only respected if the symbol is +visible, which corresponds to it (currently) being visible in the menuconfig +interface. + +For symbols with prompts, the visibility of the symbol is determined by the +condition on the prompt. Symbols without prompts are never visible, so setting +a user value on them is pointless. A warning will be printed by default if +Symbol.set_value() is called on a promptless symbol. Assignments to promptless +symbols are normal within a .config file, so no similar warning will be printed +by load_config(). + +Dependencies from parents and 'if'/'depends on' are propagated to properties, +including prompts, so these two configurations are logically equivalent: + +(1) + + menu "menu" + depends on A + + if B + + config FOO + tristate "foo" if D + default y + depends on C + + endif + + endmenu + +(2) + + menu "menu" + depends on A + + config FOO + tristate "foo" if A && B && C && D + default y if A && B && C + + endmenu + +In this example, A && B && C && D (the prompt condition) needs to be non-n for +FOO to be visible (assignable). If its value is m, the symbol can only be +assigned the value m: The visibility sets an upper bound on the value that can +be assigned by the user, and any higher user value will be truncated down. + +'default' properties are independent of the visibility, though a 'default' will +often get the same condition as the prompt due to dependency propagation. +'default' properties are used if the symbol is not visible or has no user +value. + +Symbols with no user value (or that have a user value but are not visible) and +no (active) 'default' default to n for bool/tristate symbols, and to the empty +string for other symbol types. + +'select' works similarly to symbol visibility, but sets a lower bound on the +value of the symbol. The lower bound is determined by the value of the +select*ing* symbol. 'select' does not respect visibility, so non-visible +symbols can be forced to a particular (minimum) value by a select as well. + +For non-bool/tristate symbols, it only matters whether the visibility is n or +non-n: m visibility acts the same as y visibility. + +Conditions on 'default' and 'select' work in mostly intuitive ways. If the +condition is n, the 'default' or 'select' is disabled. If it is m, the +'default' or 'select' value (the value of the selecting symbol) is truncated +down to m. + +When writing a configuration with Kconfig.write_config(), only symbols that are +visible, have an (active) default, or are selected will get written out (note +that this includes all symbols that would accept user values). Kconfiglib +matches the .config format produced by the C implementations down to the +character. This eases testing. + +For a visible bool/tristate symbol FOO with value n, this line is written to +.config: + + # CONFIG_FOO is not set + +The point is to remember the user n selection (which might differ from the +default value the symbol would get), while at the same sticking to the rule +that undefined corresponds to n (.config uses Makefile format, making the line +above a comment). When the .config file is read back in, this line will be +treated the same as the following assignment: + + CONFIG_FOO=n + +In Kconfiglib, the set of (currently) assignable values for a bool/tristate +symbol appear in Symbol.assignable. For other symbol types, just check if +sym.visibility is non-0 (non-n) to see whether the user value will have an +effect. + + +Intro to the menu tree +====================== + +The menu structure, as seen in e.g. menuconfig, is represented by a tree of +MenuNode objects. The top node of the configuration corresponds to an implicit +top-level menu, the title of which is shown at the top in the standard +menuconfig interface. (The title is also available in Kconfig.mainmenu_text in +Kconfiglib.) + +The top node is found in Kconfig.top_node. From there, you can visit child menu +nodes by following the 'list' pointer, and any following menu nodes by +following the 'next' pointer. Usually, a non-None 'list' pointer indicates a +menu or Choice, but menu nodes for symbols can sometimes have a non-None 'list' +pointer too due to submenus created implicitly from dependencies. + +MenuNode.item is either a Symbol or a Choice object, or one of the constants +MENU and COMMENT. The prompt of the menu node can be found in MenuNode.prompt, +which also holds the title for menus and comments. For Symbol and Choice, +MenuNode.help holds the help text (if any, otherwise None). + +Most symbols will only have a single menu node. A symbol defined in multiple +locations will have one menu node for each location. The list of menu nodes for +a Symbol or Choice can be found in the Symbol/Choice.nodes attribute. + +Note that prompts and help texts for symbols and choices are stored in their +menu node(s) rather than in the Symbol or Choice objects themselves. This makes +it possible to define a symbol in multiple locations with a different prompt or +help text in each location. To get the help text or prompt for a symbol with a +single menu node, do sym.nodes[0].help and sym.nodes[0].prompt, respectively. +The prompt is a (text, condition) tuple, where condition determines the +visibility (see 'Intro to expressions' below). + +This organization mirrors the C implementation. MenuNode is called +'struct menu' there, but I thought "menu" was a confusing name. + +It is possible to give a Choice a name and define it in multiple locations, +hence why Choice.nodes is also a list. + +As a convenience, the properties added at a particular definition location are +available on the MenuNode itself, in e.g. MenuNode.defaults. This is helpful +when generating documentation, so that symbols/choices defined in multiple +locations can be shown with the correct properties at each location. + + +Intro to expressions +==================== + +Expressions can be evaluated with the expr_value() function and printed with +the expr_str() function (these are used internally as well). Evaluating an +expression always yields a tristate value, where n, m, and y are represented as +0, 1, and 2, respectively. + +The following table should help you figure out how expressions are represented. +A, B, C, ... are symbols (Symbol instances), NOT is the kconfiglib.NOT +constant, etc. + +Expression Representation +---------- -------------- +A A +"A" A (constant symbol) +!A (NOT, A) +A && B (AND, A, B) +A && B && C (AND, A, (AND, B, C)) +A || B (OR, A, B) +A || (B && C && D) (OR, A, (AND, B, (AND, C, D))) +A = B (EQUAL, A, B) +A != "foo" (UNEQUAL, A, foo (constant symbol)) +A && B = C && D (AND, A, (AND, (EQUAL, B, C), D)) +n Kconfig.n (constant symbol) +m Kconfig.m (constant symbol) +y Kconfig.y (constant symbol) +"y" Kconfig.y (constant symbol) + +Strings like "foo" in 'default "foo"' or 'depends on SYM = "foo"' are +represented as constant symbols, so the only values that appear in expressions +are symbols***. This mirrors the C implementation. + +***For choice symbols, the parent Choice will appear in expressions as well, +but it's usually invisible as the value interfaces of Symbol and Choice are +identical. This mirrors the C implementation and makes different choice modes +"just work". + +Manual evaluation examples: + + - The value of A && B is min(A.tri_value, B.tri_value) + + - The value of A || B is max(A.tri_value, B.tri_value) + + - The value of !A is 2 - A.tri_value + + - The value of A = B is 2 (y) if A.str_value == B.str_value, and 0 (n) + otherwise. Note that str_value is used here instead of tri_value. + + For constant (as well as undefined) symbols, str_value matches the name of + the symbol. This mirrors the C implementation and explains why + 'depends on SYM = "foo"' above works as expected. + +n/m/y are automatically converted to the corresponding constant symbols +"n"/"m"/"y" (Kconfig.n/m/y) during parsing. + +Kconfig.const_syms is a dictionary like Kconfig.syms but for constant symbols. + +If a condition is missing (e.g., <cond> when the 'if <cond>' is removed from +'default A if <cond>'), it is actually Kconfig.y. The standard __str__() +functions just avoid printing 'if y' conditions to give cleaner output. + + +Kconfig extensions +================== + +Kconfiglib includes a couple of Kconfig extensions: + +'source' with relative path +--------------------------- + +The 'rsource' statement sources Kconfig files with a path relative to directory +of the Kconfig file containing the 'rsource' statement, instead of relative to +the project root. + +Consider following directory tree: + + Project + +--Kconfig + | + +--src + +--Kconfig + | + +--SubSystem1 + +--Kconfig + | + +--ModuleA + +--Kconfig + +In this example, assume that src/SubSystem1/Kconfig wants to source +src/SubSystem1/ModuleA/Kconfig. + +With 'source', this statement would be used: + + source "src/SubSystem1/ModuleA/Kconfig" + +With 'rsource', this turns into + + rsource "ModuleA/Kconfig" + +If an absolute path is given to 'rsource', it acts the same as 'source'. + +'rsource' can be used to create "position-independent" Kconfig trees that can +be moved around freely. + + +Globbing 'source' +----------------- + +'source' and 'rsource' accept glob patterns, sourcing all matching Kconfig +files. They require at least one matching file, raising a KconfigError +otherwise. + +For example, the following statement might source sub1/foofoofoo and +sub2/foobarfoo: + + source "sub[12]/foo*foo" + +The glob patterns accepted are the same as for the standard glob.glob() +function. + +Two additional statements are provided for cases where it's acceptable for a +pattern to match no files: 'osource' and 'orsource' (the o is for "optional"). + +For example, the following statements will be no-ops if neither "foo" nor any +files matching "bar*" exist: + + osource "foo" + osource "bar*" + +'orsource' does a relative optional source. + +'source' and 'osource' are analogous to 'include' and '-include' in Make. + + +Generalized def_* keywords +-------------------------- + +def_int, def_hex, and def_string are available in addition to def_bool and +def_tristate, allowing int, hex, and string symbols to be given a type and a +default at the same time. + + +Extra optional warnings +----------------------- + +Some optional warnings can be controlled via environment variables: + + - KCONFIG_WARN_UNDEF: If set to 'y', warnings will be generated for all + references to undefined symbols within Kconfig files. The only gotcha is + that all hex literals must be prefixed with "0x" or "0X", to make it + possible to distinguish them from symbol references. + + Some projects (e.g. the Linux kernel) use multiple Kconfig trees with many + shared Kconfig files, leading to some safe undefined symbol references. + KCONFIG_WARN_UNDEF is useful in projects that only have a single Kconfig + tree though. + + KCONFIG_STRICT is an older alias for this environment variable, supported + for backwards compatibility. + + - KCONFIG_WARN_UNDEF_ASSIGN: If set to 'y', warnings will be generated for + all assignments to undefined symbols within .config files. By default, no + such warnings are generated. + + This warning can also be enabled/disabled via the Kconfig.warn_assign_undef + variable. + + +Preprocessor user functions defined in Python +--------------------------------------------- + +Preprocessor functions can be defined in Python, which makes it simple to +integrate information from existing Python tools into Kconfig (e.g. to have +Kconfig symbols depend on hardware information stored in some other format). + +Putting a Python module named kconfigfunctions(.py) anywhere in sys.path will +cause it to be imported by Kconfiglib (in Kconfig.__init__()). Note that +sys.path can be customized via PYTHONPATH, and includes the directory of the +module being run by default, as well as installation directories. + +If the KCONFIG_FUNCTIONS environment variable is set, it gives a different +module name to use instead of 'kconfigfunctions'. + +The imported module is expected to define a global dictionary named 'functions' +that maps function names to Python functions, as follows: + + def my_fn(kconf, name, arg_1, arg_2, ...): + # kconf: + # Kconfig instance + # + # name: + # Name of the user-defined function ("my-fn"). Think argv[0]. + # + # arg_1, arg_2, ...: + # Arguments passed to the function from Kconfig (strings) + # + # Returns a string to be substituted as the result of calling the + # function + ... + + def my_other_fn(kconf, name, arg_1, arg_2, ...): + ... + + functions = { + "my-fn": (my_fn, <min.args>, <max.args>/None), + "my-other-fn": (my_other_fn, <min.args>, <max.args>/None), + ... + } + + ... + +<min.args> and <max.args> are the minimum and maximum number of arguments +expected by the function (excluding the implicit 'name' argument). If +<max.args> is None, there is no upper limit to the number of arguments. Passing +an invalid number of arguments will generate a KconfigError exception. + +Functions can access the current parsing location as kconf.filename/linenr. +Accessing other fields of the Kconfig object is not safe. See the warning +below. + +Keep in mind that for a variable defined like 'foo = $(fn)', 'fn' will be +called only when 'foo' is expanded. If 'fn' uses the parsing location and the +intent is to use the location of the assignment, you want 'foo := $(fn)' +instead, which calls the function immediately. + +Once defined, user functions can be called from Kconfig in the same way as +other preprocessor functions: + + config FOO + ... + depends on $(my-fn,arg1,arg2) + +If my_fn() returns "n", this will result in + + config FOO + ... + depends on n + +Warning +******* + +User-defined preprocessor functions are called as they're encountered at parse +time, before all Kconfig files have been processed, and before the menu tree +has been finalized. There are no guarantees that accessing Kconfig symbols or +the menu tree via the 'kconf' parameter will work, and it could potentially +lead to a crash. + +Preferably, user-defined functions should be stateless. + + +Feedback +======== + +Send bug reports, suggestions, and questions to ulfalizer a.t Google's email +service, or open a ticket on the GitHub page. +""" +import errno +import importlib +import os +import re +import sys + +# Get rid of some attribute lookups. These are obvious in context. +from glob import iglob +from os.path import dirname, exists, expandvars, islink, join, realpath + + +VERSION = (14, 1, 0) + + +# File layout: +# +# Public classes +# Public functions +# Internal functions +# Global constants + +# Line length: 79 columns + + +# +# Public classes +# + + +class Kconfig(object): + """ + Represents a Kconfig configuration, e.g. for x86 or ARM. This is the set of + symbols, choices, and menu nodes appearing in the configuration. Creating + any number of Kconfig objects (including for different architectures) is + safe. Kconfiglib doesn't keep any global state. + + The following attributes are available. They should be treated as + read-only, and some are implemented through @property magic. + + syms: + A dictionary with all symbols in the configuration, indexed by name. Also + includes all symbols that are referenced in expressions but never + defined, except for constant (quoted) symbols. + + Undefined symbols can be recognized by Symbol.nodes being empty -- see + the 'Intro to the menu tree' section in the module docstring. + + const_syms: + A dictionary like 'syms' for constant (quoted) symbols + + named_choices: + A dictionary like 'syms' for named choices (choice FOO) + + defined_syms: + A list with all defined symbols, in the same order as they appear in the + Kconfig files. Symbols defined in multiple locations appear multiple + times. + + Note: You probably want to use 'unique_defined_syms' instead. This + attribute is mostly maintained for backwards compatibility. + + unique_defined_syms: + A list like 'defined_syms', but with duplicates removed. Just the first + instance is kept for symbols defined in multiple locations. Kconfig order + is preserved otherwise. + + Using this attribute instead of 'defined_syms' can save work, and + automatically gives reasonable behavior when writing configuration output + (symbols defined in multiple locations only generate output once, while + still preserving Kconfig order for readability). + + choices: + A list with all choices, in the same order as they appear in the Kconfig + files. + + Note: You probably want to use 'unique_choices' instead. This attribute + is mostly maintained for backwards compatibility. + + unique_choices: + Analogous to 'unique_defined_syms', for choices. Named choices can have + multiple definition locations. + + menus: + A list with all menus, in the same order as they appear in the Kconfig + files + + comments: + A list with all comments, in the same order as they appear in the Kconfig + files + + kconfig_filenames: + A list with the filenames of all Kconfig files included in the + configuration, relative to $srctree (or relative to the current directory + if $srctree isn't set), except absolute paths (e.g. + 'source "/foo/Kconfig"') are kept as-is. + + The files are listed in the order they are source'd, starting with the + top-level Kconfig file. If a file is source'd multiple times, it will + appear multiple times. Use set() to get unique filenames. + + Note that Kconfig.sync_deps() already indirectly catches any file + modifications that change configuration output. + + env_vars: + A set() with the names of all environment variables referenced in the + Kconfig files. + + Only environment variables referenced with the preprocessor $(FOO) syntax + will be registered. The older $FOO syntax is only supported for backwards + compatibility. + + Also note that $(FOO) won't be registered unless the environment variable + $FOO is actually set. If it isn't, $(FOO) is an expansion of an unset + preprocessor variable (which gives the empty string). + + Another gotcha is that environment variables referenced in the values of + recursively expanded preprocessor variables (those defined with =) will + only be registered if the variable is actually used (expanded) somewhere. + + The note from the 'kconfig_filenames' documentation applies here too. + + n/m/y: + The predefined constant symbols n/m/y. Also available in const_syms. + + modules: + The Symbol instance for the modules symbol. Currently hardcoded to + MODULES, which is backwards compatible. Kconfiglib will warn if + 'option modules' is set on some other symbol. Tell me if you need proper + 'option modules' support. + + 'modules' is never None. If the MODULES symbol is not explicitly defined, + its tri_value will be 0 (n), as expected. + + A simple way to enable modules is to do 'kconf.modules.set_value(2)' + (provided the MODULES symbol is defined and visible). Modules are + disabled by default in the kernel Kconfig files as of writing, though + nearly all defconfig files enable them (with 'CONFIG_MODULES=y'). + + defconfig_list: + The Symbol instance for the 'option defconfig_list' symbol, or None if no + defconfig_list symbol exists. The defconfig filename derived from this + symbol can be found in Kconfig.defconfig_filename. + + defconfig_filename: + The filename given by the defconfig_list symbol. This is taken from the + first 'default' with a satisfied condition where the specified file + exists (can be opened for reading). If a defconfig file foo/defconfig is + not found and $srctree was set when the Kconfig was created, + $srctree/foo/defconfig is looked up as well. + + 'defconfig_filename' is None if either no defconfig_list symbol exists, + or if the defconfig_list symbol has no 'default' with a satisfied + condition that specifies a file that exists. + + Gotcha: scripts/kconfig/Makefile might pass --defconfig=<defconfig> to + scripts/kconfig/conf when running e.g. 'make defconfig'. This option + overrides the defconfig_list symbol, meaning defconfig_filename might not + always match what 'make defconfig' would use. + + top_node: + The menu node (see the MenuNode class) of the implicit top-level menu. + Acts as the root of the menu tree. + + mainmenu_text: + The prompt (title) of the top menu (top_node). Defaults to "Main menu". + Can be changed with the 'mainmenu' statement (see kconfig-language.txt). + + variables: + A dictionary with all preprocessor variables, indexed by name. See the + Variable class. + + warn: + Set this variable to True/False to enable/disable warnings. See + Kconfig.__init__(). + + When 'warn' is False, the values of the other warning-related variables + are ignored. + + This variable as well as the other warn* variables can be read to check + the current warning settings. + + warn_to_stderr: + Set this variable to True/False to enable/disable warnings on stderr. See + Kconfig.__init__(). + + warn_assign_undef: + Set this variable to True to generate warnings for assignments to + undefined symbols in configuration files. + + This variable is False by default unless the KCONFIG_WARN_UNDEF_ASSIGN + environment variable was set to 'y' when the Kconfig instance was + created. + + warn_assign_override: + Set this variable to True to generate warnings for multiple assignments + to the same symbol in configuration files, where the assignments set + different values (e.g. CONFIG_FOO=m followed by CONFIG_FOO=y, where the + last value would get used). + + This variable is True by default. Disabling it might be useful when + merging configurations. + + warn_assign_redun: + Like warn_assign_override, but for multiple assignments setting a symbol + to the same value. + + This variable is True by default. Disabling it might be useful when + merging configurations. + + warnings: + A list of strings containing all warnings that have been generated, for + cases where more flexibility is needed. + + See the 'warn_to_stderr' parameter to Kconfig.__init__() and the + Kconfig.warn_to_stderr variable as well. Note that warnings still get + added to Kconfig.warnings when 'warn_to_stderr' is True. + + Just as for warnings printed to stderr, only warnings that are enabled + will get added to Kconfig.warnings. See the various Kconfig.warn* + variables. + + missing_syms: + A list with (name, value) tuples for all assignments to undefined symbols + within the most recently loaded .config file(s). 'name' is the symbol + name without the 'CONFIG_' prefix. 'value' is a string that gives the + right-hand side of the assignment verbatim. + + See Kconfig.load_config() as well. + + srctree: + The value the $srctree environment variable had when the Kconfig instance + was created, or the empty string if $srctree wasn't set. This gives nice + behavior with os.path.join(), which treats "" as the current directory, + without adding "./". + + Kconfig files are looked up relative to $srctree (unless absolute paths + are used), and .config files are looked up relative to $srctree if they + are not found in the current directory. This is used to support + out-of-tree builds. The C tools use this environment variable in the same + way. + + Changing $srctree after creating the Kconfig instance has no effect. Only + the value when the configuration is loaded matters. This avoids surprises + if multiple configurations are loaded with different values for $srctree. + + config_prefix: + The value the CONFIG_ environment variable had when the Kconfig instance + was created, or "CONFIG_" if CONFIG_ wasn't set. This is the prefix used + (and expected) on symbol names in .config files and C headers. Used in + the same way in the C tools. + + config_header: + The value the KCONFIG_CONFIG_HEADER environment variable had when the + Kconfig instance was created, or the empty string if + KCONFIG_CONFIG_HEADER wasn't set. This string is inserted verbatim at the + beginning of configuration files. See write_config(). + + header_header: + The value the KCONFIG_AUTOHEADER_HEADER environment variable had when the + Kconfig instance was created, or the empty string if + KCONFIG_AUTOHEADER_HEADER wasn't set. This string is inserted verbatim at + the beginning of header files. See write_autoconf(). + + filename/linenr: + The current parsing location, for use in Python preprocessor functions. + See the module docstring. + """ + __slots__ = ( + "_encoding", + "_functions", + "_set_match", + "_srctree_prefix", + "_unset_match", + "_warn_assign_no_prompt", + "choices", + "comments", + "config_header", + "config_prefix", + "const_syms", + "defconfig_list", + "defined_syms", + "env_vars", + "header_header", + "kconfig_filenames", + "m", + "menus", + "missing_syms", + "modules", + "n", + "named_choices", + "srctree", + "syms", + "top_node", + "unique_choices", + "unique_defined_syms", + "variables", + "warn", + "warn_assign_override", + "warn_assign_redun", + "warn_assign_undef", + "warn_to_stderr", + "warnings", + "y", + + # Parsing-related + "_parsing_kconfigs", + "_readline", + "filename", + "linenr", + "_include_path", + "_filestack", + "_line", + "_tokens", + "_tokens_i", + "_reuse_tokens", + ) + + # + # Public interface + # + + def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True, + encoding="utf-8", suppress_traceback=False): + """ + Creates a new Kconfig object by parsing Kconfig files. + Note that Kconfig files are not the same as .config files (which store + configuration symbol values). + + See the module docstring for some environment variables that influence + default warning settings (KCONFIG_WARN_UNDEF and + KCONFIG_WARN_UNDEF_ASSIGN). + + Raises KconfigError on syntax/semantic errors, and OSError or (possibly + a subclass of) IOError on IO errors ('errno', 'strerror', and + 'filename' are available). Note that IOError is an alias for OSError on + Python 3, so it's enough to catch OSError there. If you need Python 2/3 + compatibility, it's easiest to catch EnvironmentError, which is a + common base class of OSError/IOError on Python 2 and an alias for + OSError on Python 3. + + filename (default: "Kconfig"): + The Kconfig file to load. For the Linux kernel, you'll want "Kconfig" + from the top-level directory, as environment variables will make sure + the right Kconfig is included from there (arch/$SRCARCH/Kconfig as of + writing). + + If $srctree is set, 'filename' will be looked up relative to it. + $srctree is also used to look up source'd files within Kconfig files. + See the class documentation. + + If you are using Kconfiglib via 'make scriptconfig', the filename of + the base base Kconfig file will be in sys.argv[1]. It's currently + always "Kconfig" in practice. + + warn (default: True): + True if warnings related to this configuration should be generated. + This can be changed later by setting Kconfig.warn to True/False. It + is provided as a constructor argument since warnings might be + generated during parsing. + + See the other Kconfig.warn_* variables as well, which enable or + suppress certain warnings when warnings are enabled. + + All generated warnings are added to the Kconfig.warnings list. See + the class documentation. + + warn_to_stderr (default: True): + True if warnings should be printed to stderr in addition to being + added to Kconfig.warnings. + + This can be changed later by setting Kconfig.warn_to_stderr to + True/False. + + encoding (default: "utf-8"): + The encoding to use when reading and writing files, and when decoding + output from commands run via $(shell). If None, the encoding + specified in the current locale will be used. + + The "utf-8" default avoids exceptions on systems that are configured + to use the C locale, which implies an ASCII encoding. + + This parameter has no effect on Python 2, due to implementation + issues (regular strings turning into Unicode strings, which are + distinct in Python 2). Python 2 doesn't decode regular strings + anyway. + + Related PEP: https://www.python.org/dev/peps/pep-0538/ + + suppress_traceback (default: False): + Helper for tools. When True, any EnvironmentError or KconfigError + generated during parsing is caught, the exception message is printed + to stderr together with the command name, and sys.exit(1) is called + (which generates SystemExit). + + This hides the Python traceback for "expected" errors like syntax + errors in Kconfig files. + + Other exceptions besides EnvironmentError and KconfigError are still + propagated when suppress_traceback is True. + """ + try: + self._init(filename, warn, warn_to_stderr, encoding) + except (EnvironmentError, KconfigError) as e: + if suppress_traceback: + cmd = sys.argv[0] # Empty string if missing + if cmd: + cmd += ": " + # Some long exception messages have extra newlines for better + # formatting when reported as an unhandled exception. Strip + # them here. + sys.exit(cmd + str(e).strip()) + raise + + def _init(self, filename, warn, warn_to_stderr, encoding): + # See __init__() + + self._encoding = encoding + + self.srctree = os.getenv("srctree", "") + # A prefix we can reliably strip from glob() results to get a filename + # relative to $srctree. relpath() can cause issues for symlinks, + # because it assumes symlink/../foo is the same as foo/. + self._srctree_prefix = realpath(self.srctree) + os.sep + + self.warn = warn + self.warn_to_stderr = warn_to_stderr + self.warn_assign_undef = os.getenv("KCONFIG_WARN_UNDEF_ASSIGN") == "y" + self.warn_assign_override = True + self.warn_assign_redun = True + self._warn_assign_no_prompt = True + + self.warnings = [] + + self.config_prefix = os.getenv("CONFIG_", "CONFIG_") + # Regular expressions for parsing .config files + self._set_match = _re_match(self.config_prefix + r"([^=]+)=(.*)") + self._unset_match = _re_match(r"# {}([^ ]+) is not set".format( + self.config_prefix)) + + self.config_header = os.getenv("KCONFIG_CONFIG_HEADER", "") + self.header_header = os.getenv("KCONFIG_AUTOHEADER_HEADER", "") + + self.syms = {} + self.const_syms = {} + self.defined_syms = [] + self.missing_syms = [] + self.named_choices = {} + self.choices = [] + self.menus = [] + self.comments = [] + + for nmy in "n", "m", "y": + sym = Symbol() + sym.kconfig = self + sym.name = nmy + sym.is_constant = True + sym.orig_type = TRISTATE + sym._cached_tri_val = STR_TO_TRI[nmy] + + self.const_syms[nmy] = sym + + self.n = self.const_syms["n"] + self.m = self.const_syms["m"] + self.y = self.const_syms["y"] + + # Make n/m/y well-formed symbols + for nmy in "n", "m", "y": + sym = self.const_syms[nmy] + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + # Maps preprocessor variables names to Variable instances + self.variables = {} + + # Predefined preprocessor functions, with min/max number of arguments + self._functions = { + "info": (_info_fn, 1, 1), + "error-if": (_error_if_fn, 2, 2), + "filename": (_filename_fn, 0, 0), + "lineno": (_lineno_fn, 0, 0), + "shell": (_shell_fn, 1, 1), + "warning-if": (_warning_if_fn, 2, 2), + } + + # Add any user-defined preprocessor functions + try: + self._functions.update( + importlib.import_module( + os.getenv("KCONFIG_FUNCTIONS", "kconfigfunctions") + ).functions) + except ImportError: + pass + + # This determines whether previously unseen symbols are registered. + # They shouldn't be if we parse expressions after parsing, as part of + # Kconfig.eval_string(). + self._parsing_kconfigs = True + + self.modules = self._lookup_sym("MODULES") + self.defconfig_list = None + + self.top_node = MenuNode() + self.top_node.kconfig = self + self.top_node.item = MENU + self.top_node.is_menuconfig = True + self.top_node.visibility = self.y + self.top_node.prompt = ("Main menu", self.y) + self.top_node.parent = None + self.top_node.dep = self.y + self.top_node.filename = filename + self.top_node.linenr = 1 + self.top_node.include_path = () + + # Parse the Kconfig files + + # Not used internally. Provided as a convenience. + self.kconfig_filenames = [filename] + self.env_vars = set() + + # Keeps track of the location in the parent Kconfig files. Kconfig + # files usually source other Kconfig files. See _enter_file(). + self._filestack = [] + self._include_path = () + + # The current parsing location + self.filename = filename + self.linenr = 0 + + # Used to avoid retokenizing lines when we discover that they're not + # part of the construct currently being parsed. This is kinda like an + # unget operation. + self._reuse_tokens = False + + # Open the top-level Kconfig file. Store the readline() method directly + # as a small optimization. + self._readline = self._open(join(self.srctree, filename), "r").readline + + try: + # Parse the Kconfig files. Returns the last node, which we + # terminate with '.next = None'. + self._parse_block(None, self.top_node, self.top_node).next = None + self.top_node.list = self.top_node.next + self.top_node.next = None + except UnicodeDecodeError as e: + _decoding_error(e, self.filename) + + # Close the top-level Kconfig file. __self__ fetches the 'file' object + # for the method. + self._readline.__self__.close() + + self._parsing_kconfigs = False + + # Do various menu tree post-processing + self._finalize_node(self.top_node, self.y) + + self.unique_defined_syms = _ordered_unique(self.defined_syms) + self.unique_choices = _ordered_unique(self.choices) + + # Do sanity checks. Some of these depend on everything being finalized. + self._check_sym_sanity() + self._check_choice_sanity() + + # KCONFIG_STRICT is an older alias for KCONFIG_WARN_UNDEF, supported + # for backwards compatibility + if os.getenv("KCONFIG_WARN_UNDEF") == "y" or \ + os.getenv("KCONFIG_STRICT") == "y": + + self._check_undef_syms() + + # Build Symbol._dependents for all symbols and choices + self._build_dep() + + # Check for dependency loops + check_dep_loop_sym = _check_dep_loop_sym # Micro-optimization + for sym in self.unique_defined_syms: + check_dep_loop_sym(sym, False) + + # Add extra dependencies from choices to choice symbols that get + # awkward during dependency loop detection + self._add_choice_deps() + + @property + def mainmenu_text(self): + """ + See the class documentation. + """ + return self.top_node.prompt[0] + + @property + def defconfig_filename(self): + """ + See the class documentation. + """ + if self.defconfig_list: + for filename, cond in self.defconfig_list.defaults: + if expr_value(cond): + try: + with self._open_config(filename.str_value) as f: + return f.name + except EnvironmentError: + continue + + return None + + def load_config(self, filename=None, replace=True, verbose=None): + """ + Loads symbol values from a file in the .config format. Equivalent to + calling Symbol.set_value() to set each of the values. + + "# CONFIG_FOO is not set" within a .config file sets the user value of + FOO to n. The C tools work the same way. + + For each symbol, the Symbol.user_value attribute holds the value the + symbol was assigned in the .config file (if any). The user value might + differ from Symbol.str/tri_value if there are unsatisfied dependencies. + + Calling this function also updates the Kconfig.missing_syms attribute + with a list of all assignments to undefined symbols within the + configuration file. Kconfig.missing_syms is cleared if 'replace' is + True, and appended to otherwise. See the documentation for + Kconfig.missing_syms as well. + + See the Kconfig.__init__() docstring for raised exceptions + (OSError/IOError). KconfigError is never raised here. + + filename (default: None): + Path to load configuration from (a string). Respects $srctree if set + (see the class documentation). + + If 'filename' is None (the default), the configuration file to load + (if any) is calculated automatically, giving the behavior you'd + usually want: + + 1. If the KCONFIG_CONFIG environment variable is set, it gives the + path to the configuration file to load. Otherwise, ".config" is + used. See standard_config_filename(). + + 2. If the path from (1.) doesn't exist, the configuration file + given by kconf.defconfig_filename is loaded instead, which is + derived from the 'option defconfig_list' symbol. + + 3. If (1.) and (2.) fail to find a configuration file to load, no + configuration file is loaded, and symbols retain their current + values (e.g., their default values). This is not an error. + + See the return value as well. + + replace (default: True): + If True, all existing user values will be cleared before loading the + .config. Pass False to merge configurations. + + verbose (default: None): + Limited backwards compatibility to prevent crashes. A warning is + printed if anything but None is passed. + + Prior to Kconfiglib 12.0.0, this option enabled printing of messages + to stdout when 'filename' was None. A message is (always) returned + now instead, which is more flexible. + + Will probably be removed in some future version. + + Returns a string with a message saying which file got loaded (or + possibly that no file got loaded, when 'filename' is None). This is + meant to reduce boilerplate in tools, which can do e.g. + print(kconf.load_config()). The returned message distinguishes between + loading (replace == True) and merging (replace == False). + """ + if verbose is not None: + _warn_verbose_deprecated("load_config") + + msg = None + if filename is None: + filename = standard_config_filename() + if not exists(filename) and \ + not exists(join(self.srctree, filename)): + defconfig = self.defconfig_filename + if defconfig is None: + return "Using default symbol values (no '{}')" \ + .format(filename) + + msg = " default configuration '{}' (no '{}')" \ + .format(defconfig, filename) + filename = defconfig + + if not msg: + msg = " configuration '{}'".format(filename) + + # Disable the warning about assigning to symbols without prompts. This + # is normal and expected within a .config file. + self._warn_assign_no_prompt = False + + # This stub only exists to make sure _warn_assign_no_prompt gets + # reenabled + try: + self._load_config(filename, replace) + except UnicodeDecodeError as e: + _decoding_error(e, filename) + finally: + self._warn_assign_no_prompt = True + + return ("Loaded" if replace else "Merged") + msg + + def _load_config(self, filename, replace): + with self._open_config(filename) as f: + if replace: + self.missing_syms = [] + + # If we're replacing the configuration, keep track of which + # symbols and choices got set so that we can unset the rest + # later. This avoids invalidating everything and is faster. + # Another benefit is that invalidation must be rock solid for + # it to work, making it a good test. + + for sym in self.unique_defined_syms: + sym._was_set = False + + for choice in self.unique_choices: + choice._was_set = False + + # Small optimizations + set_match = self._set_match + unset_match = self._unset_match + get_sym = self.syms.get + + for linenr, line in enumerate(f, 1): + # The C tools ignore trailing whitespace + line = line.rstrip() + + match = set_match(line) + if match: + name, val = match.groups() + sym = get_sym(name) + if not sym or not sym.nodes: + self._undef_assign(name, val, filename, linenr) + continue + + if sym.orig_type in _BOOL_TRISTATE: + # The C implementation only checks the first character + # to the right of '=', for whatever reason + if not (sym.orig_type is BOOL + and val.startswith(("y", "n")) or + sym.orig_type is TRISTATE + and val.startswith(("y", "m", "n"))): + self._warn("'{}' is not a valid value for the {} " + "symbol {}. Assignment ignored." + .format(val, TYPE_TO_STR[sym.orig_type], + sym.name_and_loc), + filename, linenr) + continue + + val = val[0] + + if sym.choice and val != "n": + # During .config loading, we infer the mode of the + # choice from the kind of values that are assigned + # to the choice symbols + + prev_mode = sym.choice.user_value + if prev_mode is not None and \ + TRI_TO_STR[prev_mode] != val: + + self._warn("both m and y assigned to symbols " + "within the same choice", + filename, linenr) + + # Set the choice's mode + sym.choice.set_value(val) + + elif sym.orig_type is STRING: + match = _conf_string_match(val) + if not match: + self._warn("malformed string literal in " + "assignment to {}. Assignment ignored." + .format(sym.name_and_loc), + filename, linenr) + continue + + val = unescape(match.group(1)) + + else: + match = unset_match(line) + if not match: + # Print a warning for lines that match neither + # set_match() nor unset_match() and that are not blank + # lines or comments. 'line' has already been + # rstrip()'d, so blank lines show up as "" here. + if line and not line.lstrip().startswith("#"): + self._warn("ignoring malformed line '{}'" + .format(line), + filename, linenr) + + continue + + name = match.group(1) + sym = get_sym(name) + if not sym or not sym.nodes: + self._undef_assign(name, "n", filename, linenr) + continue + + if sym.orig_type not in _BOOL_TRISTATE: + continue + + val = "n" + + # Done parsing the assignment. Set the value. + + if sym._was_set: + self._assigned_twice(sym, val, filename, linenr) + + sym.set_value(val) + + if replace: + # If we're replacing the configuration, unset the symbols that + # didn't get set + + for sym in self.unique_defined_syms: + if not sym._was_set: + sym.unset_value() + + for choice in self.unique_choices: + if not choice._was_set: + choice.unset_value() + + def _undef_assign(self, name, val, filename, linenr): + # Called for assignments to undefined symbols during .config loading + + self.missing_syms.append((name, val)) + if self.warn_assign_undef: + self._warn( + "attempt to assign the value '{}' to the undefined symbol {}" + .format(val, name), filename, linenr) + + def _assigned_twice(self, sym, new_val, filename, linenr): + # Called when a symbol is assigned more than once in a .config file + + # Use strings for bool/tristate user values in the warning + if sym.orig_type in _BOOL_TRISTATE: + user_val = TRI_TO_STR[sym.user_value] + else: + user_val = sym.user_value + + msg = '{} set more than once. Old value "{}", new value "{}".'.format( + sym.name_and_loc, user_val, new_val) + + if user_val == new_val: + if self.warn_assign_redun: + self._warn(msg, filename, linenr) + elif self.warn_assign_override: + self._warn(msg, filename, linenr) + + def load_allconfig(self, filename): + """ + Helper for all*config. Loads (merges) the configuration file specified + by KCONFIG_ALLCONFIG, if any. See Documentation/kbuild/kconfig.txt in + the Linux kernel. + + Disables warnings for duplicated assignments within configuration files + for the duration of the call + (kconf.warn_assign_override/warn_assign_redun = False), and restores + the previous warning settings at the end. The KCONFIG_ALLCONFIG + configuration file is expected to override symbols. + + Exits with sys.exit() (which raises a SystemExit exception) and prints + an error to stderr if KCONFIG_ALLCONFIG is set but the configuration + file can't be opened. + + filename: + Command-specific configuration filename - "allyes.config", + "allno.config", etc. + """ + load_allconfig(self, filename) + + def write_autoconf(self, filename=None, header=None): + r""" + Writes out symbol values as a C header file, matching the format used + by include/generated/autoconf.h in the kernel. + + The ordering of the #defines matches the one generated by + write_config(). The order in the C implementation depends on the hash + table implementation as of writing, and so won't match. + + If 'filename' exists and its contents is identical to what would get + written out, it is left untouched. This avoids updating file metadata + like the modification time and possibly triggering redundant work in + build tools. + + filename (default: None): + Path to write header to. + + If None (the default), the path in the environment variable + KCONFIG_AUTOHEADER is used if set, and "include/generated/autoconf.h" + otherwise. This is compatible with the C tools. + + header (default: None): + Text inserted verbatim at the beginning of the file. You would + usually want it enclosed in '/* */' to make it a C comment, and + include a trailing newline. + + If None (the default), the value of the environment variable + KCONFIG_AUTOHEADER_HEADER had when the Kconfig instance was created + will be used if it was set, and no header otherwise. See the + Kconfig.header_header attribute. + + Returns a string with a message saying that the header got saved, or + that there were no changes to it. This is meant to reduce boilerplate + in tools, which can do e.g. print(kconf.write_autoconf()). + """ + if filename is None: + filename = os.getenv("KCONFIG_AUTOHEADER", + "include/generated/autoconf.h") + + if self._write_if_changed(filename, self._autoconf_contents(header)): + return "Kconfig header saved to '{}'".format(filename) + return "No change to Kconfig header in '{}'".format(filename) + + def _autoconf_contents(self, header): + # write_autoconf() helper. Returns the contents to write as a string, + # with 'header' or KCONFIG_AUTOHEADER_HEADER at the beginning. + + if header is None: + header = self.header_header + + chunks = [header] # "".join()ed later + add = chunks.append + + for sym in self.unique_defined_syms: + # _write_to_conf is determined when the value is calculated. This + # is a hidden function call due to property magic. + # + # Note: In client code, you can check if sym.config_string is empty + # instead, to avoid accessing the internal _write_to_conf variable + # (though it's likely to keep working). + val = sym.str_value + if not sym._write_to_conf: + continue + + if sym.orig_type in _BOOL_TRISTATE: + if val == "y": + add("#define {}{} 1\n" + .format(self.config_prefix, sym.name)) + elif val == "m": + add("#define {}{}_MODULE 1\n" + .format(self.config_prefix, sym.name)) + + elif sym.orig_type is STRING: + add('#define {}{} "{}"\n' + .format(self.config_prefix, sym.name, escape(val))) + + else: # sym.orig_type in _INT_HEX: + if sym.orig_type is HEX and \ + not val.startswith(("0x", "0X")): + val = "0x" + val + + add("#define {}{} {}\n" + .format(self.config_prefix, sym.name, val)) + + return "".join(chunks) + + def write_config(self, filename=None, header=None, save_old=True, + verbose=None): + r""" + Writes out symbol values in the .config format. The format matches the + C implementation, including ordering. + + Symbols appear in the same order in generated .config files as they do + in the Kconfig files. For symbols defined in multiple locations, a + single assignment is written out corresponding to the first location + where the symbol is defined. + + See the 'Intro to symbol values' section in the module docstring to + understand which symbols get written out. + + If 'filename' exists and its contents is identical to what would get + written out, it is left untouched. This avoids updating file metadata + like the modification time and possibly triggering redundant work in + build tools. + + See the Kconfig.__init__() docstring for raised exceptions + (OSError/IOError). KconfigError is never raised here. + + filename (default: None): + Path to write configuration to (a string). + + If None (the default), the path in the environment variable + KCONFIG_CONFIG is used if set, and ".config" otherwise. See + standard_config_filename(). + + header (default: None): + Text inserted verbatim at the beginning of the file. You would + usually want each line to start with '#' to make it a comment, and + include a trailing newline. + + if None (the default), the value of the environment variable + KCONFIG_CONFIG_HEADER had when the Kconfig instance was created will + be used if it was set, and no header otherwise. See the + Kconfig.config_header attribute. + + save_old (default: True): + If True and <filename> already exists, a copy of it will be saved to + <filename>.old in the same directory before the new configuration is + written. + + Errors are silently ignored if <filename>.old cannot be written (e.g. + due to being a directory, or <filename> being something like + /dev/null). + + verbose (default: None): + Limited backwards compatibility to prevent crashes. A warning is + printed if anything but None is passed. + + Prior to Kconfiglib 12.0.0, this option enabled printing of messages + to stdout when 'filename' was None. A message is (always) returned + now instead, which is more flexible. + + Will probably be removed in some future version. + + Returns a string with a message saying which file got saved. This is + meant to reduce boilerplate in tools, which can do e.g. + print(kconf.write_config()). + """ + if verbose is not None: + _warn_verbose_deprecated("write_config") + + if filename is None: + filename = standard_config_filename() + + contents = self._config_contents(header) + if self._contents_eq(filename, contents): + return "No change to configuration in '{}'".format(filename) + + if save_old: + _save_old(filename) + + with self._open(filename, "w") as f: + f.write(contents) + + return "Configuration saved to '{}'".format(filename) + + def _config_contents(self, header): + # write_config() helper. Returns the contents to write as a string, + # with 'header' or KCONFIG_CONFIG_HEADER at the beginning. + # + # More memory friendly would be to 'yield' the strings and + # "".join(_config_contents()), but it was a bit slower on my system. + + # node_iter() was used here before commit 3aea9f7 ("Add '# end of + # <menu>' after menus in .config"). Those comments get tricky to + # implement with it. + + for sym in self.unique_defined_syms: + sym._visited = False + + if header is None: + header = self.config_header + + chunks = [header] # "".join()ed later + add = chunks.append + + # Did we just print an '# end of ...' comment? + after_end_comment = False + + node = self.top_node + while 1: + # Jump to the next node with an iterative tree walk + if node.list: + node = node.list + elif node.next: + node = node.next + else: + while node.parent: + node = node.parent + + # Add a comment when leaving visible menus + if node.item is MENU and expr_value(node.dep) and \ + expr_value(node.visibility) and \ + node is not self.top_node: + add("# end of {}\n".format(node.prompt[0])) + after_end_comment = True + + if node.next: + node = node.next + break + else: + # No more nodes + return "".join(chunks) + + # Generate configuration output for the node + + item = node.item + + if item.__class__ is Symbol: + if item._visited: + continue + item._visited = True + + conf_string = item.config_string + if not conf_string: + continue + + if after_end_comment: + # Add a blank line before the first symbol printed after an + # '# end of ...' comment + after_end_comment = False + add("\n") + add(conf_string) + + elif expr_value(node.dep) and \ + ((item is MENU and expr_value(node.visibility)) or + item is COMMENT): + + add("\n#\n# {}\n#\n".format(node.prompt[0])) + after_end_comment = False + + def write_min_config(self, filename, header=None): + """ + Writes out a "minimal" configuration file, omitting symbols whose value + matches their default value. The format matches the one produced by + 'make savedefconfig'. + + The resulting configuration file is incomplete, but a complete + configuration can be derived from it by loading it. Minimal + configuration files can serve as a more manageable configuration format + compared to a "full" .config file, especially when configurations files + are merged or edited by hand. + + See the Kconfig.__init__() docstring for raised exceptions + (OSError/IOError). KconfigError is never raised here. + + filename: + Path to write minimal configuration to. + + header (default: None): + Text inserted verbatim at the beginning of the file. You would + usually want each line to start with '#' to make it a comment, and + include a final terminating newline. + + if None (the default), the value of the environment variable + KCONFIG_CONFIG_HEADER had when the Kconfig instance was created will + be used if it was set, and no header otherwise. See the + Kconfig.config_header attribute. + + Returns a string with a message saying the minimal configuration got + saved, or that there were no changes to it. This is meant to reduce + boilerplate in tools, which can do e.g. + print(kconf.write_min_config()). + """ + if self._write_if_changed(filename, self._min_config_contents(header)): + return "Minimal configuration saved to '{}'".format(filename) + return "No change to minimal configuration in '{}'".format(filename) + + def _min_config_contents(self, header): + # write_min_config() helper. Returns the contents to write as a string, + # with 'header' or KCONFIG_CONFIG_HEADER at the beginning. + + if header is None: + header = self.config_header + + chunks = [header] # "".join()ed later + add = chunks.append + + for sym in self.unique_defined_syms: + # Skip symbols that cannot be changed. Only check + # non-choice symbols, as selects don't affect choice + # symbols. + if not sym.choice and \ + sym.visibility <= expr_value(sym.rev_dep): + continue + + # Skip symbols whose value matches their default + if sym.str_value == sym._str_default(): + continue + + # Skip symbols that would be selected by default in a + # choice, unless the choice is optional or the symbol type + # isn't bool (it might be possible to set the choice mode + # to n or the symbol to m in those cases). + if sym.choice and \ + not sym.choice.is_optional and \ + sym.choice._selection_from_defaults() is sym and \ + sym.orig_type is BOOL and \ + sym.tri_value == 2: + continue + + add(sym.config_string) + + return "".join(chunks) + + def sync_deps(self, path): + """ + Creates or updates a directory structure that can be used to avoid + doing a full rebuild whenever the configuration is changed, mirroring + include/config/ in the kernel. + + This function is intended to be called during each build, before + compiling source files that depend on configuration symbols. + + See the Kconfig.__init__() docstring for raised exceptions + (OSError/IOError). KconfigError is never raised here. + + path: + Path to directory + + sync_deps(path) does the following: + + 1. If the directory <path> does not exist, it is created. + + 2. If <path>/auto.conf exists, old symbol values are loaded from it, + which are then compared against the current symbol values. If a + symbol has changed value (would generate different output in + autoconf.h compared to before), the change is signaled by + touch'ing a file corresponding to the symbol. + + The first time sync_deps() is run on a directory, <path>/auto.conf + won't exist, and no old symbol values will be available. This + logically has the same effect as updating the entire + configuration. + + The path to a symbol's file is calculated from the symbol's name + by replacing all '_' with '/' and appending '.h'. For example, the + symbol FOO_BAR_BAZ gets the file <path>/foo/bar/baz.h, and FOO + gets the file <path>/foo.h. + + This scheme matches the C tools. The point is to avoid having a + single directory with a huge number of files, which the underlying + filesystem might not handle well. + + 3. A new auto.conf with the current symbol values is written, to keep + track of them for the next build. + + If auto.conf exists and its contents is identical to what would + get written out, it is left untouched. This avoids updating file + metadata like the modification time and possibly triggering + redundant work in build tools. + + + The last piece of the puzzle is knowing what symbols each source file + depends on. Knowing that, dependencies can be added from source files + to the files corresponding to the symbols they depends on. The source + file will then get recompiled (only) when the symbol value changes + (provided sync_deps() is run first during each build). + + The tool in the kernel that extracts symbol dependencies from source + files is scripts/basic/fixdep.c. Missing symbol files also correspond + to "not changed", which fixdep deals with by using the $(wildcard) Make + function when adding symbol prerequisites to source files. + + In case you need a different scheme for your project, the sync_deps() + implementation can be used as a template. + """ + if not exists(path): + os.mkdir(path, 0o755) + + # Load old values from auto.conf, if any + self._load_old_vals(path) + + for sym in self.unique_defined_syms: + # _write_to_conf is determined when the value is calculated. This + # is a hidden function call due to property magic. + # + # Note: In client code, you can check if sym.config_string is empty + # instead, to avoid accessing the internal _write_to_conf variable + # (though it's likely to keep working). + val = sym.str_value + + # n tristate values do not get written to auto.conf and autoconf.h, + # making a missing symbol logically equivalent to n + + if sym._write_to_conf: + if sym._old_val is None and \ + sym.orig_type in _BOOL_TRISTATE and \ + val == "n": + # No old value (the symbol was missing or n), new value n. + # No change. + continue + + if val == sym._old_val: + # New value matches old. No change. + continue + + elif sym._old_val is None: + # The symbol wouldn't appear in autoconf.h (because + # _write_to_conf is false), and it wouldn't have appeared in + # autoconf.h previously either (because it didn't appear in + # auto.conf). No change. + continue + + # 'sym' has a new value. Flag it. + _touch_dep_file(path, sym.name) + + # Remember the current values as the "new old" values. + # + # This call could go anywhere after the call to _load_old_vals(), but + # putting it last means _sync_deps() can be safely rerun if it fails + # before this point. + self._write_old_vals(path) + + def _load_old_vals(self, path): + # Loads old symbol values from auto.conf into a dedicated + # Symbol._old_val field. Mirrors load_config(). + # + # The extra field could be avoided with some trickery involving dumping + # symbol values and restoring them later, but this is simpler and + # faster. The C tools also use a dedicated field for this purpose. + + for sym in self.unique_defined_syms: + sym._old_val = None + + try: + auto_conf = self._open(join(path, "auto.conf"), "r") + except EnvironmentError as e: + if e.errno == errno.ENOENT: + # No old values + return + raise + + with auto_conf as f: + for line in f: + match = self._set_match(line) + if not match: + # We only expect CONFIG_FOO=... (and possibly a header + # comment) in auto.conf + continue + + name, val = match.groups() + if name in self.syms: + sym = self.syms[name] + + if sym.orig_type is STRING: + match = _conf_string_match(val) + if not match: + continue + val = unescape(match.group(1)) + + self.syms[name]._old_val = val + else: + # Flag that the symbol no longer exists, in + # case something still depends on it + _touch_dep_file(path, name) + + def _write_old_vals(self, path): + # Helper for writing auto.conf. Basically just a simplified + # write_config() that doesn't write any comments (including + # '# CONFIG_FOO is not set' comments). The format matches the C + # implementation, though the ordering is arbitrary there (depends on + # the hash table implementation). + # + # A separate helper function is neater than complicating write_config() + # by passing a flag to it, plus we only need to look at symbols here. + + self._write_if_changed( + os.path.join(path, "auto.conf"), + self._old_vals_contents()) + + def _old_vals_contents(self): + # _write_old_vals() helper. Returns the contents to write as a string. + + # Temporary list instead of generator makes this a bit faster + return "".join([ + sym.config_string for sym in self.unique_defined_syms + if not (sym.orig_type in _BOOL_TRISTATE and not sym.tri_value) + ]) + + def node_iter(self, unique_syms=False): + """ + Returns a generator for iterating through all MenuNode's in the Kconfig + tree. The iteration is done in Kconfig definition order (each node is + visited before its children, and the children of a node are visited + before the next node). + + The Kconfig.top_node menu node is skipped. It contains an implicit menu + that holds the top-level items. + + As an example, the following code will produce a list equal to + Kconfig.defined_syms: + + defined_syms = [node.item for node in kconf.node_iter() + if isinstance(node.item, Symbol)] + + unique_syms (default: False): + If True, only the first MenuNode will be included for symbols defined + in multiple locations. + + Using kconf.node_iter(True) in the example above would give a list + equal to unique_defined_syms. + """ + if unique_syms: + for sym in self.unique_defined_syms: + sym._visited = False + + node = self.top_node + while 1: + # Jump to the next node with an iterative tree walk + if node.list: + node = node.list + elif node.next: + node = node.next + else: + while node.parent: + node = node.parent + if node.next: + node = node.next + break + else: + # No more nodes + return + + if unique_syms and node.item.__class__ is Symbol: + if node.item._visited: + continue + node.item._visited = True + + yield node + + def eval_string(self, s): + """ + Returns the tristate value of the expression 's', represented as 0, 1, + and 2 for n, m, and y, respectively. Raises KconfigError on syntax + errors. Warns if undefined symbols are referenced. + + As an example, if FOO and BAR are tristate symbols at least one of + which has the value y, then eval_string("y && (FOO || BAR)") returns + 2 (y). + + To get the string value of non-bool/tristate symbols, use + Symbol.str_value. eval_string() always returns a tristate value, and + all non-bool/tristate symbols have the tristate value 0 (n). + + The expression parsing is consistent with how parsing works for + conditional ('if ...') expressions in the configuration, and matches + the C implementation. m is rewritten to 'm && MODULES', so + eval_string("m") will return 0 (n) unless modules are enabled. + """ + # The parser is optimized to be fast when parsing Kconfig files (where + # an expression can never appear at the beginning of a line). We have + # to monkey-patch things a bit here to reuse it. + + self.filename = None + + self._tokens = self._tokenize("if " + s) + # Strip "if " to avoid giving confusing error messages + self._line = s + self._tokens_i = 1 # Skip the 'if' token + + return expr_value(self._expect_expr_and_eol()) + + def unset_values(self): + """ + Removes any user values from all symbols, as if Kconfig.load_config() + or Symbol.set_value() had never been called. + """ + self._warn_assign_no_prompt = False + try: + # set_value() already rejects undefined symbols, and they don't + # need to be invalidated (because their value never changes), so we + # can just iterate over defined symbols + for sym in self.unique_defined_syms: + sym.unset_value() + + for choice in self.unique_choices: + choice.unset_value() + finally: + self._warn_assign_no_prompt = True + + def enable_warnings(self): + """ + Do 'Kconfig.warn = True' instead. Maintained for backwards + compatibility. + """ + self.warn = True + + def disable_warnings(self): + """ + Do 'Kconfig.warn = False' instead. Maintained for backwards + compatibility. + """ + self.warn = False + + def enable_stderr_warnings(self): + """ + Do 'Kconfig.warn_to_stderr = True' instead. Maintained for backwards + compatibility. + """ + self.warn_to_stderr = True + + def disable_stderr_warnings(self): + """ + Do 'Kconfig.warn_to_stderr = False' instead. Maintained for backwards + compatibility. + """ + self.warn_to_stderr = False + + def enable_undef_warnings(self): + """ + Do 'Kconfig.warn_assign_undef = True' instead. Maintained for backwards + compatibility. + """ + self.warn_assign_undef = True + + def disable_undef_warnings(self): + """ + Do 'Kconfig.warn_assign_undef = False' instead. Maintained for + backwards compatibility. + """ + self.warn_assign_undef = False + + def enable_override_warnings(self): + """ + Do 'Kconfig.warn_assign_override = True' instead. Maintained for + backwards compatibility. + """ + self.warn_assign_override = True + + def disable_override_warnings(self): + """ + Do 'Kconfig.warn_assign_override = False' instead. Maintained for + backwards compatibility. + """ + self.warn_assign_override = False + + def enable_redun_warnings(self): + """ + Do 'Kconfig.warn_assign_redun = True' instead. Maintained for backwards + compatibility. + """ + self.warn_assign_redun = True + + def disable_redun_warnings(self): + """ + Do 'Kconfig.warn_assign_redun = False' instead. Maintained for + backwards compatibility. + """ + self.warn_assign_redun = False + + def __repr__(self): + """ + Returns a string with information about the Kconfig object when it is + evaluated on e.g. the interactive Python prompt. + """ + def status(flag): + return "enabled" if flag else "disabled" + + return "<{}>".format(", ".join(( + "configuration with {} symbols".format(len(self.syms)), + 'main menu prompt "{}"'.format(self.mainmenu_text), + "srctree is current directory" if not self.srctree else + 'srctree "{}"'.format(self.srctree), + 'config symbol prefix "{}"'.format(self.config_prefix), + "warnings " + status(self.warn), + "printing of warnings to stderr " + status(self.warn_to_stderr), + "undef. symbol assignment warnings " + + status(self.warn_assign_undef), + "overriding symbol assignment warnings " + + status(self.warn_assign_override), + "redundant symbol assignment warnings " + + status(self.warn_assign_redun) + ))) + + # + # Private methods + # + + + # + # File reading + # + + def _open_config(self, filename): + # Opens a .config file. First tries to open 'filename', then + # '$srctree/filename' if $srctree was set when the configuration was + # loaded. + + try: + return self._open(filename, "r") + except EnvironmentError as e: + # This will try opening the same file twice if $srctree is unset, + # but it's not a big deal + try: + return self._open(join(self.srctree, filename), "r") + except EnvironmentError as e2: + # This is needed for Python 3, because e2 is deleted after + # the try block: + # + # https://docs.python.org/3/reference/compound_stmts.html#the-try-statement + e = e2 + + raise _KconfigIOError( + e, "Could not open '{}' ({}: {}). Check that the $srctree " + "environment variable ({}) is set correctly." + .format(filename, errno.errorcode[e.errno], e.strerror, + "set to '{}'".format(self.srctree) if self.srctree + else "unset or blank")) + + def _enter_file(self, filename): + # Jumps to the beginning of a sourced Kconfig file, saving the previous + # position and file object. + # + # filename: + # Absolute path to file + + # Path relative to $srctree, stored in e.g. self.filename (which makes + # it indirectly show up in MenuNode.filename). Equals 'filename' for + # absolute paths passed to 'source'. + if filename.startswith(self._srctree_prefix): + # Relative path (or a redundant absolute path to within $srctree, + # but it's probably fine to reduce those too) + rel_filename = filename[len(self._srctree_prefix):] + else: + # Absolute path + rel_filename = filename + + self.kconfig_filenames.append(rel_filename) + + # The parent Kconfig files are represented as a list of + # (<include path>, <Python 'file' object for Kconfig file>) tuples. + # + # <include path> is immutable and holds a *tuple* of + # (<filename>, <linenr>) tuples, giving the locations of the 'source' + # statements in the parent Kconfig files. The current include path is + # also available in Kconfig._include_path. + # + # The point of this redundant setup is to allow Kconfig._include_path + # to be assigned directly to MenuNode.include_path without having to + # copy it, sharing it wherever possible. + + # Save include path and 'file' object (via its 'readline' function) + # before entering the file + self._filestack.append((self._include_path, self._readline)) + + # _include_path is a tuple, so this rebinds the variable instead of + # doing in-place modification + self._include_path += ((self.filename, self.linenr),) + + # Check for recursive 'source' + for name, _ in self._include_path: + if name == rel_filename: + raise KconfigError( + "\n{}:{}: recursive 'source' of '{}' detected. Check that " + "environment variables are set correctly.\n" + "Include path:\n{}" + .format(self.filename, self.linenr, rel_filename, + "\n".join("{}:{}".format(name, linenr) + for name, linenr in self._include_path))) + + try: + self._readline = self._open(filename, "r").readline + except EnvironmentError as e: + # We already know that the file exists + raise _KconfigIOError( + e, "{}:{}: Could not open '{}' (in '{}') ({}: {})" + .format(self.filename, self.linenr, filename, + self._line.strip(), + errno.errorcode[e.errno], e.strerror)) + + self.filename = rel_filename + self.linenr = 0 + + def _leave_file(self): + # Returns from a Kconfig file to the file that sourced it. See + # _enter_file(). + + # Restore location from parent Kconfig file + self.filename, self.linenr = self._include_path[-1] + # Restore include path and 'file' object + self._readline.__self__.close() # __self__ fetches the 'file' object + self._include_path, self._readline = self._filestack.pop() + + def _next_line(self): + # Fetches and tokenizes the next line from the current Kconfig file. + # Returns False at EOF and True otherwise. + + # We might already have tokens from parsing a line and discovering that + # it's part of a different construct + if self._reuse_tokens: + self._reuse_tokens = False + # self._tokens_i is known to be 1 here, because _parse_props() + # leaves it like that when it can't recognize a line (or parses a + # help text) + return True + + # readline() returns '' over and over at EOF, which we rely on for help + # texts at the end of files (see _line_after_help()) + line = self._readline() + if not line: + return False + self.linenr += 1 + + # Handle line joining + while line.endswith("\\\n"): + line = line[:-2] + self._readline() + self.linenr += 1 + + self._tokens = self._tokenize(line) + # Initialize to 1 instead of 0 to factor out code from _parse_block() + # and _parse_props(). They immediately fetch self._tokens[0]. + self._tokens_i = 1 + + return True + + def _line_after_help(self, line): + # Tokenizes a line after a help text. This case is special in that the + # line has already been fetched (to discover that it isn't part of the + # help text). + # + # An earlier version used a _saved_line variable instead that was + # checked in _next_line(). This special-casing gets rid of it and makes + # _reuse_tokens alone sufficient to handle unget. + + # Handle line joining + while line.endswith("\\\n"): + line = line[:-2] + self._readline() + self.linenr += 1 + + self._tokens = self._tokenize(line) + self._reuse_tokens = True + + def _write_if_changed(self, filename, contents): + # Writes 'contents' into 'filename', but only if it differs from the + # current contents of the file. + # + # Another variant would be write a temporary file on the same + # filesystem, compare the files, and rename() the temporary file if it + # differs, but it breaks stuff like write_config("/dev/null"), which is + # used out there to force evaluation-related warnings to be generated. + # This simple version is pretty failsafe and portable. + # + # Returns True if the file has changed and is updated, and False + # otherwise. + + if self._contents_eq(filename, contents): + return False + with self._open(filename, "w") as f: + f.write(contents) + return True + + def _contents_eq(self, filename, contents): + # Returns True if the contents of 'filename' is 'contents' (a string), + # and False otherwise (including if 'filename' can't be opened/read) + + try: + with self._open(filename, "r") as f: + # Robust re. things like encoding and line endings (mmap() + # trickery isn't) + return f.read(len(contents) + 1) == contents + except EnvironmentError: + # If the error here would prevent writing the file as well, we'll + # notice it later + return False + + # + # Tokenization + # + + def _lookup_sym(self, name): + # Fetches the symbol 'name' from the symbol table, creating and + # registering it if it does not exist. If '_parsing_kconfigs' is False, + # it means we're in eval_string(), and new symbols won't be registered. + + if name in self.syms: + return self.syms[name] + + sym = Symbol() + sym.kconfig = self + sym.name = name + sym.is_constant = False + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + if self._parsing_kconfigs: + self.syms[name] = sym + else: + self._warn("no symbol {} in configuration".format(name)) + + return sym + + def _lookup_const_sym(self, name): + # Like _lookup_sym(), for constant (quoted) symbols + + if name in self.const_syms: + return self.const_syms[name] + + sym = Symbol() + sym.kconfig = self + sym.name = name + sym.is_constant = True + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + if self._parsing_kconfigs: + self.const_syms[name] = sym + + return sym + + def _tokenize(self, s): + # Parses 's', returning a None-terminated list of tokens. Registers any + # new symbols encountered with _lookup(_const)_sym(). + # + # Tries to be reasonably speedy by processing chunks of text via + # regexes and string operations where possible. This is the biggest + # hotspot during parsing. + # + # It might be possible to rewrite this to 'yield' tokens instead, + # working across multiple lines. Lookback and compatibility with old + # janky versions of the C tools complicate things though. + + self._line = s # Used for error reporting + + # Initial token on the line + match = _command_match(s) + if not match: + if s.isspace() or s.lstrip().startswith("#"): + return (None,) + self._parse_error("unknown token at start of line") + + # Tricky implementation detail: While parsing a token, 'token' refers + # to the previous token. See _STRING_LEX for why this is needed. + token = _get_keyword(match.group(1)) + if not token: + # Backwards compatibility with old versions of the C tools, which + # (accidentally) accepted stuff like "--help--" and "-help---". + # This was fixed in the C tools by commit c2264564 ("kconfig: warn + # of unhandled characters in Kconfig commands"), committed in July + # 2015, but it seems people still run Kconfiglib on older kernels. + if s.strip(" \t\n-") == "help": + return (_T_HELP, None) + + # If the first token is not a keyword (and not a weird help token), + # we have a preprocessor variable assignment (or a bare macro on a + # line) + self._parse_assignment(s) + return (None,) + + tokens = [token] + # The current index in the string being tokenized + i = match.end() + + # Main tokenization loop (for tokens past the first one) + while i < len(s): + # Test for an identifier/keyword first. This is the most common + # case. + match = _id_keyword_match(s, i) + if match: + # We have an identifier or keyword + + # Check what it is. lookup_sym() will take care of allocating + # new symbols for us the first time we see them. Note that + # 'token' still refers to the previous token. + + name = match.group(1) + keyword = _get_keyword(name) + if keyword: + # It's a keyword + token = keyword + # Jump past it + i = match.end() + + elif token not in _STRING_LEX: + # It's a non-const symbol, except we translate n, m, and y + # into the corresponding constant symbols, like the C + # implementation + + if "$" in name: + # Macro expansion within symbol name + name, s, i = self._expand_name(s, i) + else: + i = match.end() + + token = self.const_syms[name] if name in STR_TO_TRI else \ + self._lookup_sym(name) + + else: + # It's a case of missing quotes. For example, the + # following is accepted: + # + # menu unquoted_title + # + # config A + # tristate unquoted_prompt + # + # endmenu + # + # Named choices ('choice FOO') also end up here. + + if token is not _T_CHOICE: + self._warn("style: quotes recommended around '{}' in '{}'" + .format(name, self._line.strip()), + self.filename, self.linenr) + + token = name + i = match.end() + + else: + # Neither a keyword nor a non-const symbol + + # We always strip whitespace after tokens, so it is safe to + # assume that s[i] is the start of a token here. + c = s[i] + + if c in "\"'": + if "$" not in s and "\\" not in s: + # Fast path for lines without $ and \. Find the + # matching quote. + end_i = s.find(c, i + 1) + 1 + if not end_i: + self._parse_error("unterminated string") + val = s[i + 1:end_i - 1] + i = end_i + else: + # Slow path + s, end_i = self._expand_str(s, i) + + # os.path.expandvars() and the $UNAME_RELEASE replace() + # is a backwards compatibility hack, which should be + # reasonably safe as expandvars() leaves references to + # undefined env. vars. as is. + # + # The preprocessor functionality changed how + # environment variables are referenced, to $(FOO). + val = expandvars(s[i + 1:end_i - 1] + .replace("$UNAME_RELEASE", + _UNAME_RELEASE)) + + i = end_i + + # This is the only place where we don't survive with a + # single token of lookback: 'option env="FOO"' does not + # refer to a constant symbol named "FOO". + token = \ + val if token in _STRING_LEX or tokens[0] is _T_OPTION \ + else self._lookup_const_sym(val) + + elif s.startswith("&&", i): + token = _T_AND + i += 2 + + elif s.startswith("||", i): + token = _T_OR + i += 2 + + elif c == "=": + token = _T_EQUAL + i += 1 + + elif s.startswith("!=", i): + token = _T_UNEQUAL + i += 2 + + elif c == "!": + token = _T_NOT + i += 1 + + elif c == "(": + token = _T_OPEN_PAREN + i += 1 + + elif c == ")": + token = _T_CLOSE_PAREN + i += 1 + + elif c == "#": + break + + + # Very rare + + elif s.startswith("<=", i): + token = _T_LESS_EQUAL + i += 2 + + elif c == "<": + token = _T_LESS + i += 1 + + elif s.startswith(">=", i): + token = _T_GREATER_EQUAL + i += 2 + + elif c == ">": + token = _T_GREATER + i += 1 + + + else: + self._parse_error("unknown tokens in line") + + + # Skip trailing whitespace + while i < len(s) and s[i].isspace(): + i += 1 + + + # Add the token + tokens.append(token) + + # None-terminating the token list makes token fetching simpler/faster + tokens.append(None) + + return tokens + + # Helpers for syntax checking and token fetching. See the + # 'Intro to expressions' section for what a constant symbol is. + # + # More of these could be added, but the single-use cases are inlined as an + # optimization. + + def _expect_sym(self): + token = self._tokens[self._tokens_i] + self._tokens_i += 1 + + if token.__class__ is not Symbol: + self._parse_error("expected symbol") + + return token + + def _expect_nonconst_sym(self): + # Used for 'select' and 'imply' only. We know the token indices. + + token = self._tokens[1] + self._tokens_i = 2 + + if token.__class__ is not Symbol or token.is_constant: + self._parse_error("expected nonconstant symbol") + + return token + + def _expect_str_and_eol(self): + token = self._tokens[self._tokens_i] + self._tokens_i += 1 + + if token.__class__ is not str: + self._parse_error("expected string") + + if self._tokens[self._tokens_i] is not None: + self._trailing_tokens_error() + + return token + + def _expect_expr_and_eol(self): + expr = self._parse_expr(True) + + if self._tokens[self._tokens_i] is not None: + self._trailing_tokens_error() + + return expr + + def _check_token(self, token): + # If the next token is 'token', removes it and returns True + + if self._tokens[self._tokens_i] is token: + self._tokens_i += 1 + return True + return False + + # + # Preprocessor logic + # + + def _parse_assignment(self, s): + # Parses a preprocessor variable assignment, registering the variable + # if it doesn't already exist. Also takes care of bare macros on lines + # (which are allowed, and can be useful for their side effects). + + # Expand any macros in the left-hand side of the assignment (the + # variable name) + s = s.lstrip() + i = 0 + while 1: + i = _assignment_lhs_fragment_match(s, i).end() + if s.startswith("$(", i): + s, i = self._expand_macro(s, i, ()) + else: + break + + if s.isspace(): + # We also accept a bare macro on a line (e.g. + # $(warning-if,$(foo),ops)), provided it expands to a blank string + return + + # Assigned variable + name = s[:i] + + + # Extract assignment operator (=, :=, or +=) and value + rhs_match = _assignment_rhs_match(s, i) + if not rhs_match: + self._parse_error("syntax error") + + op, val = rhs_match.groups() + + + if name in self.variables: + # Already seen variable + var = self.variables[name] + else: + # New variable + var = Variable() + var.kconfig = self + var.name = name + var._n_expansions = 0 + self.variables[name] = var + + # += acts like = on undefined variables (defines a recursive + # variable) + if op == "+=": + op = "=" + + if op == "=": + var.is_recursive = True + var.value = val + elif op == ":=": + var.is_recursive = False + var.value = self._expand_whole(val, ()) + else: # op == "+=" + # += does immediate expansion if the variable was last set + # with := + var.value += " " + (val if var.is_recursive else + self._expand_whole(val, ())) + + def _expand_whole(self, s, args): + # Expands preprocessor macros in all of 's'. Used whenever we don't + # have to worry about delimiters. See _expand_macro() re. the 'args' + # parameter. + # + # Returns the expanded string. + + i = 0 + while 1: + i = s.find("$(", i) + if i == -1: + break + s, i = self._expand_macro(s, i, args) + return s + + def _expand_name(self, s, i): + # Expands a symbol name starting at index 'i' in 's'. + # + # Returns the expanded name, the expanded 's' (including the part + # before the name), and the index of the first character in the next + # token after the name. + + s, end_i = self._expand_name_iter(s, i) + name = s[i:end_i] + # isspace() is False for empty strings + if not name.strip(): + # Avoid creating a Kconfig symbol with a blank name. It's almost + # guaranteed to be an error. + self._parse_error("macro expanded to blank string") + + # Skip trailing whitespace + while end_i < len(s) and s[end_i].isspace(): + end_i += 1 + + return name, s, end_i + + def _expand_name_iter(self, s, i): + # Expands a symbol name starting at index 'i' in 's'. + # + # Returns the expanded 's' (including the part before the name) and the + # index of the first character after the expanded name in 's'. + + while 1: + match = _name_special_search(s, i) + + if match.group() != "$(": + return (s, match.start()) + s, i = self._expand_macro(s, match.start(), ()) + + def _expand_str(self, s, i): + # Expands a quoted string starting at index 'i' in 's'. Handles both + # backslash escapes and macro expansion. + # + # Returns the expanded 's' (including the part before the string) and + # the index of the first character after the expanded string in 's'. + + quote = s[i] + i += 1 # Skip over initial "/' + while 1: + match = _string_special_search(s, i) + if not match: + self._parse_error("unterminated string") + + + if match.group() == quote: + # Found the end of the string + return (s, match.end()) + + elif match.group() == "\\": + # Replace '\x' with 'x'. 'i' ends up pointing to the character + # after 'x', which allows macros to be canceled with '\$(foo)'. + i = match.end() + s = s[:match.start()] + s[i:] + + elif match.group() == "$(": + # A macro call within the string + s, i = self._expand_macro(s, match.start(), ()) + + else: + # A ' quote within " quotes or vice versa + i += 1 + + def _expand_macro(self, s, i, args): + # Expands a macro starting at index 'i' in 's'. If this macro resulted + # from the expansion of another macro, 'args' holds the arguments + # passed to that macro. + # + # Returns the expanded 's' (including the part before the macro) and + # the index of the first character after the expanded macro in 's'. + + res = s[:i] + i += 2 # Skip over "$(" + + arg_start = i # Start of current macro argument + new_args = [] # Arguments of this macro call + nesting = 0 # Current parentheses nesting level + + while 1: + match = _macro_special_search(s, i) + if not match: + self._parse_error("missing end parenthesis in macro expansion") + + + if match.group() == "(": + nesting += 1 + i = match.end() + + elif match.group() == ")": + if nesting: + nesting -= 1 + i = match.end() + continue + + # Found the end of the macro + + new_args.append(s[arg_start:match.start()]) + + # $(1) is replaced by the first argument to the function, etc., + # provided at least that many arguments were passed + + try: + # Does the macro look like an integer, with a corresponding + # argument? If so, expand it to the value of the argument. + res += args[int(new_args[0])] + except (ValueError, IndexError): + # Regular variables are just functions without arguments, + # and also go through the function value path + res += self._fn_val(new_args) + + return (res + s[match.end():], len(res)) + + elif match.group() == ",": + i = match.end() + if nesting: + continue + + # Found the end of a macro argument + new_args.append(s[arg_start:match.start()]) + arg_start = i + + else: # match.group() == "$(" + # A nested macro call within the macro + s, i = self._expand_macro(s, match.start(), args) + + def _fn_val(self, args): + # Returns the result of calling the function args[0] with the arguments + # args[1..len(args)-1]. Plain variables are treated as functions + # without arguments. + + fn = args[0] + + if fn in self.variables: + var = self.variables[fn] + + if len(args) == 1: + # Plain variable + if var._n_expansions: + self._parse_error("Preprocessor variable {} recursively " + "references itself".format(var.name)) + elif var._n_expansions > 100: + # Allow functions to call themselves, but guess that functions + # that are overly recursive are stuck + self._parse_error("Preprocessor function {} seems stuck " + "in infinite recursion".format(var.name)) + + var._n_expansions += 1 + res = self._expand_whole(self.variables[fn].value, args) + var._n_expansions -= 1 + return res + + if fn in self._functions: + # Built-in or user-defined function + + py_fn, min_arg, max_arg = self._functions[fn] + + if len(args) - 1 < min_arg or \ + (max_arg is not None and len(args) - 1 > max_arg): + + if min_arg == max_arg: + expected_args = min_arg + elif max_arg is None: + expected_args = "{} or more".format(min_arg) + else: + expected_args = "{}-{}".format(min_arg, max_arg) + + raise KconfigError("{}:{}: bad number of arguments in call " + "to {}, expected {}, got {}" + .format(self.filename, self.linenr, fn, + expected_args, len(args) - 1)) + + return py_fn(self, *args) + + # Environment variables are tried last + if fn in os.environ: + self.env_vars.add(fn) + return os.environ[fn] + + return "" + + # + # Parsing + # + + def _make_and(self, e1, e2): + # Constructs an AND (&&) expression. Performs trivial simplification. + + if e1 is self.y: + return e2 + + if e2 is self.y: + return e1 + + if e1 is self.n or e2 is self.n: + return self.n + + return (AND, e1, e2) + + def _make_or(self, e1, e2): + # Constructs an OR (||) expression. Performs trivial simplification. + + if e1 is self.n: + return e2 + + if e2 is self.n: + return e1 + + if e1 is self.y or e2 is self.y: + return self.y + + return (OR, e1, e2) + + def _parse_block(self, end_token, parent, prev): + # Parses a block, which is the contents of either a file or an if, + # menu, or choice statement. + # + # end_token: + # The token that ends the block, e.g. _T_ENDIF ("endif") for ifs. + # None for files. + # + # parent: + # The parent menu node, corresponding to a menu, Choice, or 'if'. + # 'if's are flattened after parsing. + # + # prev: + # The previous menu node. New nodes will be added after this one (by + # modifying 'next' pointers). + # + # 'prev' is reused to parse a list of child menu nodes (for a menu or + # Choice): After parsing the children, the 'next' pointer is assigned + # to the 'list' pointer to "tilt up" the children above the node. + # + # Returns the final menu node in the block (or 'prev' if the block is + # empty). This allows chaining. + + while self._next_line(): + t0 = self._tokens[0] + + if t0 is _T_CONFIG or t0 is _T_MENUCONFIG: + # The tokenizer allocates Symbol objects for us + sym = self._tokens[1] + + if sym.__class__ is not Symbol or sym.is_constant: + self._parse_error("missing or bad symbol name") + + if self._tokens[2] is not None: + self._trailing_tokens_error() + + self.defined_syms.append(sym) + + node = MenuNode() + node.kconfig = self + node.item = sym + node.is_menuconfig = (t0 is _T_MENUCONFIG) + node.prompt = node.help = node.list = None + node.parent = parent + node.filename = self.filename + node.linenr = self.linenr + node.include_path = self._include_path + + sym.nodes.append(node) + + self._parse_props(node) + + if node.is_menuconfig and not node.prompt: + self._warn("the menuconfig symbol {} has no prompt" + .format(sym.name_and_loc)) + + # Equivalent to + # + # prev.next = node + # prev = node + # + # due to tricky Python semantics. The order matters. + prev.next = prev = node + + elif t0 is None: + # Blank line + continue + + elif t0 in _SOURCE_TOKENS: + pattern = self._expect_str_and_eol() + + if t0 in _REL_SOURCE_TOKENS: + # Relative source + pattern = join(dirname(self.filename), pattern) + + # - glob() doesn't support globbing relative to a directory, so + # we need to prepend $srctree to 'pattern'. Use join() + # instead of '+' so that an absolute path in 'pattern' is + # preserved. + # + # - Sort the glob results to ensure a consistent ordering of + # Kconfig symbols, which indirectly ensures a consistent + # ordering in e.g. .config files + filenames = sorted(iglob(join(self._srctree_prefix, pattern))) + + if not filenames and t0 in _OBL_SOURCE_TOKENS: + raise KconfigError( + "{}:{}: '{}' not found (in '{}'). Check that " + "environment variables are set correctly (e.g. " + "$srctree, which is {}). Also note that unset " + "environment variables expand to the empty string." + .format(self.filename, self.linenr, pattern, + self._line.strip(), + "set to '{}'".format(self.srctree) + if self.srctree else "unset or blank")) + + for filename in filenames: + self._enter_file(filename) + prev = self._parse_block(None, parent, prev) + self._leave_file() + + elif t0 is end_token: + # Reached the end of the block. Terminate the final node and + # return it. + + if self._tokens[1] is not None: + self._trailing_tokens_error() + + prev.next = None + return prev + + elif t0 is _T_IF: + node = MenuNode() + node.item = node.prompt = None + node.parent = parent + node.dep = self._expect_expr_and_eol() + + self._parse_block(_T_ENDIF, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 is _T_MENU: + node = MenuNode() + node.kconfig = self + node.item = t0 # _T_MENU == MENU + node.is_menuconfig = True + node.prompt = (self._expect_str_and_eol(), self.y) + node.visibility = self.y + node.parent = parent + node.filename = self.filename + node.linenr = self.linenr + node.include_path = self._include_path + + self.menus.append(node) + + self._parse_props(node) + self._parse_block(_T_ENDMENU, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 is _T_COMMENT: + node = MenuNode() + node.kconfig = self + node.item = t0 # _T_COMMENT == COMMENT + node.is_menuconfig = False + node.prompt = (self._expect_str_and_eol(), self.y) + node.list = None + node.parent = parent + node.filename = self.filename + node.linenr = self.linenr + node.include_path = self._include_path + + self.comments.append(node) + + self._parse_props(node) + + prev.next = prev = node + + elif t0 is _T_CHOICE: + if self._tokens[1] is None: + choice = Choice() + choice.direct_dep = self.n + else: + # Named choice + name = self._expect_str_and_eol() + choice = self.named_choices.get(name) + if not choice: + choice = Choice() + choice.name = name + choice.direct_dep = self.n + self.named_choices[name] = choice + + self.choices.append(choice) + + node = MenuNode() + node.kconfig = choice.kconfig = self + node.item = choice + node.is_menuconfig = True + node.prompt = node.help = None + node.parent = parent + node.filename = self.filename + node.linenr = self.linenr + node.include_path = self._include_path + + choice.nodes.append(node) + + self._parse_props(node) + self._parse_block(_T_ENDCHOICE, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 is _T_MAINMENU: + self.top_node.prompt = (self._expect_str_and_eol(), self.y) + + else: + # A valid endchoice/endif/endmenu is caught by the 'end_token' + # check above + self._parse_error( + "no corresponding 'choice'" if t0 is _T_ENDCHOICE else + "no corresponding 'if'" if t0 is _T_ENDIF else + "no corresponding 'menu'" if t0 is _T_ENDMENU else + "unrecognized construct") + + # End of file reached. Return the last node. + + if end_token: + raise KconfigError( + "error: expected '{}' at end of '{}'" + .format("endchoice" if end_token is _T_ENDCHOICE else + "endif" if end_token is _T_ENDIF else + "endmenu", + self.filename)) + + return prev + + def _parse_cond(self): + # Parses an optional 'if <expr>' construct and returns the parsed + # <expr>, or self.y if the next token is not _T_IF + + expr = self._parse_expr(True) if self._check_token(_T_IF) else self.y + + if self._tokens[self._tokens_i] is not None: + self._trailing_tokens_error() + + return expr + + def _parse_props(self, node): + # Parses and adds properties to the MenuNode 'node' (type, 'prompt', + # 'default's, etc.) Properties are later copied up to symbols and + # choices in a separate pass after parsing, in e.g. + # _add_props_to_sym(). + # + # An older version of this code added properties directly to symbols + # and choices instead of to their menu nodes (and handled dependency + # propagation simultaneously), but that loses information on where a + # property is added when a symbol or choice is defined in multiple + # locations. Some Kconfig configuration systems rely heavily on such + # symbols, and better docs can be generated by keeping track of where + # properties are added. + # + # node: + # The menu node we're parsing properties on + + # Dependencies from 'depends on'. Will get propagated to the properties + # below. + node.dep = self.y + + while self._next_line(): + t0 = self._tokens[0] + + if t0 in _TYPE_TOKENS: + # Relies on '_T_BOOL is BOOL', etc., to save a conversion + self._set_type(node.item, t0) + if self._tokens[1] is not None: + self._parse_prompt(node) + + elif t0 is _T_DEPENDS: + if not self._check_token(_T_ON): + self._parse_error("expected 'on' after 'depends'") + + node.dep = self._make_and(node.dep, + self._expect_expr_and_eol()) + + elif t0 is _T_HELP: + self._parse_help(node) + + elif t0 is _T_SELECT: + if node.item.__class__ is not Symbol: + self._parse_error("only symbols can select") + + node.selects.append((self._expect_nonconst_sym(), + self._parse_cond())) + + elif t0 is None: + # Blank line + continue + + elif t0 is _T_DEFAULT: + node.defaults.append((self._parse_expr(False), + self._parse_cond())) + + elif t0 in _DEF_TOKEN_TO_TYPE: + self._set_type(node.item, _DEF_TOKEN_TO_TYPE[t0]) + node.defaults.append((self._parse_expr(False), + self._parse_cond())) + + elif t0 is _T_PROMPT: + self._parse_prompt(node) + + elif t0 is _T_RANGE: + node.ranges.append((self._expect_sym(), self._expect_sym(), + self._parse_cond())) + + elif t0 is _T_IMPLY: + if node.item.__class__ is not Symbol: + self._parse_error("only symbols can imply") + + node.implies.append((self._expect_nonconst_sym(), + self._parse_cond())) + + elif t0 is _T_VISIBLE: + if not self._check_token(_T_IF): + self._parse_error("expected 'if' after 'visible'") + + node.visibility = self._make_and(node.visibility, + self._expect_expr_and_eol()) + + elif t0 is _T_OPTION: + if self._check_token(_T_ENV): + if not self._check_token(_T_EQUAL): + self._parse_error("expected '=' after 'env'") + + env_var = self._expect_str_and_eol() + node.item.env_var = env_var + + if env_var in os.environ: + node.defaults.append( + (self._lookup_const_sym(os.environ[env_var]), + self.y)) + else: + self._warn("{1} has 'option env=\"{0}\"', " + "but the environment variable {0} is not " + "set".format(node.item.name, env_var), + self.filename, self.linenr) + + if env_var != node.item.name: + self._warn("Kconfiglib expands environment variables " + "in strings directly, meaning you do not " + "need 'option env=...' \"bounce\" symbols. " + "For compatibility with the C tools, " + "rename {} to {} (so that the symbol name " + "matches the environment variable name)." + .format(node.item.name, env_var), + self.filename, self.linenr) + + elif self._check_token(_T_DEFCONFIG_LIST): + if not self.defconfig_list: + self.defconfig_list = node.item + else: + self._warn("'option defconfig_list' set on multiple " + "symbols ({0} and {1}). Only {0} will be " + "used.".format(self.defconfig_list.name, + node.item.name), + self.filename, self.linenr) + + elif self._check_token(_T_MODULES): + # To reduce warning spam, only warn if 'option modules' is + # set on some symbol that isn't MODULES, which should be + # safe. I haven't run into any projects that make use + # modules besides the kernel yet, and there it's likely to + # keep being called "MODULES". + if node.item is not self.modules: + self._warn("the 'modules' option is not supported. " + "Let me know if this is a problem for you, " + "as it wouldn't be that hard to implement. " + "Note that modules are supported -- " + "Kconfiglib just assumes the symbol name " + "MODULES, like older versions of the C " + "implementation did when 'option modules' " + "wasn't used.", + self.filename, self.linenr) + + elif self._check_token(_T_ALLNOCONFIG_Y): + if node.item.__class__ is not Symbol: + self._parse_error("the 'allnoconfig_y' option is only " + "valid for symbols") + + node.item.is_allnoconfig_y = True + + else: + self._parse_error("unrecognized option") + + elif t0 is _T_OPTIONAL: + if node.item.__class__ is not Choice: + self._parse_error('"optional" is only valid for choices') + + node.item.is_optional = True + + else: + # Reuse the tokens for the non-property line later + self._reuse_tokens = True + return + + def _set_type(self, sc, new_type): + # Sets the type of 'sc' (symbol or choice) to 'new_type' + + # UNKNOWN is falsy + if sc.orig_type and sc.orig_type is not new_type: + self._warn("{} defined with multiple types, {} will be used" + .format(sc.name_and_loc, TYPE_TO_STR[new_type])) + + sc.orig_type = new_type + + def _parse_prompt(self, node): + # 'prompt' properties override each other within a single definition of + # a symbol, but additional prompts can be added by defining the symbol + # multiple times + + if node.prompt: + self._warn(node.item.name_and_loc + + " defined with multiple prompts in single location") + + prompt = self._tokens[1] + self._tokens_i = 2 + + if prompt.__class__ is not str: + self._parse_error("expected prompt string") + + if prompt != prompt.strip(): + self._warn(node.item.name_and_loc + + " has leading or trailing whitespace in its prompt") + + # This avoid issues for e.g. reStructuredText documentation, where + # '*prompt *' is invalid + prompt = prompt.strip() + + node.prompt = (prompt, self._parse_cond()) + + def _parse_help(self, node): + if node.help is not None: + self._warn(node.item.name_and_loc + " defined with more than " + "one help text -- only the last one will be used") + + # Micro-optimization. This code is pretty hot. + readline = self._readline + + # Find first non-blank (not all-space) line and get its + # indentation + + while 1: + line = readline() + self.linenr += 1 + if not line: + self._empty_help(node, line) + return + if not line.isspace(): + break + + len_ = len # Micro-optimization + + # Use a separate 'expline' variable here and below to avoid stomping on + # any tabs people might've put deliberately into the first line after + # the help text + expline = line.expandtabs() + indent = len_(expline) - len_(expline.lstrip()) + if not indent: + self._empty_help(node, line) + return + + # The help text goes on till the first non-blank line with less indent + # than the first line + + # Add the first line + lines = [expline[indent:]] + add_line = lines.append # Micro-optimization + + while 1: + line = readline() + if line.isspace(): + # No need to preserve the exact whitespace in these + add_line("\n") + elif not line: + # End of file + break + else: + expline = line.expandtabs() + if len_(expline) - len_(expline.lstrip()) < indent: + break + add_line(expline[indent:]) + + self.linenr += len_(lines) + node.help = "".join(lines).rstrip() + if line: + self._line_after_help(line) + + def _empty_help(self, node, line): + self._warn(node.item.name_and_loc + + " has 'help' but empty help text") + node.help = "" + if line: + self._line_after_help(line) + + def _parse_expr(self, transform_m): + # Parses an expression from the tokens in Kconfig._tokens using a + # simple top-down approach. See the module docstring for the expression + # format. + # + # transform_m: + # True if m should be rewritten to m && MODULES. See the + # Kconfig.eval_string() documentation. + + # Grammar: + # + # expr: and_expr ['||' expr] + # and_expr: factor ['&&' and_expr] + # factor: <symbol> ['='/'!='/'<'/... <symbol>] + # '!' factor + # '(' expr ')' + # + # It helps to think of the 'expr: and_expr' case as a single-operand OR + # (no ||), and of the 'and_expr: factor' case as a single-operand AND + # (no &&). Parsing code is always a bit tricky. + + # Mind dump: parse_factor() and two nested loops for OR and AND would + # work as well. The straightforward implementation there gives a + # (op, (op, (op, A, B), C), D) parse for A op B op C op D. Representing + # expressions as (op, [list of operands]) instead goes nicely with that + # version, but is wasteful for short expressions and complicates + # expression evaluation and other code that works on expressions (more + # complicated code likely offsets any performance gain from less + # recursion too). If we also try to optimize the list representation by + # merging lists when possible (e.g. when ANDing two AND expressions), + # we end up allocating a ton of lists instead of reusing expressions, + # which is bad. + + and_expr = self._parse_and_expr(transform_m) + + # Return 'and_expr' directly if we have a "single-operand" OR. + # Otherwise, parse the expression on the right and make an OR node. + # This turns A || B || C || D into (OR, A, (OR, B, (OR, C, D))). + return and_expr if not self._check_token(_T_OR) else \ + (OR, and_expr, self._parse_expr(transform_m)) + + def _parse_and_expr(self, transform_m): + factor = self._parse_factor(transform_m) + + # Return 'factor' directly if we have a "single-operand" AND. + # Otherwise, parse the right operand and make an AND node. This turns + # A && B && C && D into (AND, A, (AND, B, (AND, C, D))). + return factor if not self._check_token(_T_AND) else \ + (AND, factor, self._parse_and_expr(transform_m)) + + def _parse_factor(self, transform_m): + token = self._tokens[self._tokens_i] + self._tokens_i += 1 + + if token.__class__ is Symbol: + # Plain symbol or relation + + if self._tokens[self._tokens_i] not in _RELATIONS: + # Plain symbol + + # For conditional expressions ('depends on <expr>', + # '... if <expr>', etc.), m is rewritten to m && MODULES. + if transform_m and token is self.m: + return (AND, self.m, self.modules) + + return token + + # Relation + # + # _T_EQUAL, _T_UNEQUAL, etc., deliberately have the same values as + # EQUAL, UNEQUAL, etc., so we can just use the token directly + self._tokens_i += 1 + return (self._tokens[self._tokens_i - 1], token, + self._expect_sym()) + + if token is _T_NOT: + # token == _T_NOT == NOT + return (token, self._parse_factor(transform_m)) + + if token is _T_OPEN_PAREN: + expr_parse = self._parse_expr(transform_m) + if self._check_token(_T_CLOSE_PAREN): + return expr_parse + + self._parse_error("malformed expression") + + # + # Caching and invalidation + # + + def _build_dep(self): + # Populates the Symbol/Choice._dependents sets, which contain all other + # items (symbols and choices) that immediately depend on the item in + # the sense that changing the value of the item might affect the value + # of the dependent items. This is used for caching/invalidation. + # + # The calculated sets might be larger than necessary as we don't do any + # complex analysis of the expressions. + + depend_on = _depend_on # Micro-optimization + + # Only calculate _dependents for defined symbols. Constant and + # undefined symbols could theoretically be selected/implied, but it + # wouldn't change their value, so it's not a true dependency. + for sym in self.unique_defined_syms: + # Symbols depend on the following: + + # The prompt conditions + for node in sym.nodes: + if node.prompt: + depend_on(sym, node.prompt[1]) + + # The default values and their conditions + for value, cond in sym.defaults: + depend_on(sym, value) + depend_on(sym, cond) + + # The reverse and weak reverse dependencies + depend_on(sym, sym.rev_dep) + depend_on(sym, sym.weak_rev_dep) + + # The ranges along with their conditions + for low, high, cond in sym.ranges: + depend_on(sym, low) + depend_on(sym, high) + depend_on(sym, cond) + + # The direct dependencies. This is usually redundant, as the direct + # dependencies get propagated to properties, but it's needed to get + # invalidation solid for 'imply', which only checks the direct + # dependencies (even if there are no properties to propagate it + # to). + depend_on(sym, sym.direct_dep) + + # In addition to the above, choice symbols depend on the choice + # they're in, but that's handled automatically since the Choice is + # propagated to the conditions of the properties before + # _build_dep() runs. + + for choice in self.unique_choices: + # Choices depend on the following: + + # The prompt conditions + for node in choice.nodes: + if node.prompt: + depend_on(choice, node.prompt[1]) + + # The default symbol conditions + for _, cond in choice.defaults: + depend_on(choice, cond) + + def _add_choice_deps(self): + # Choices also depend on the choice symbols themselves, because the + # y-mode selection of the choice might change if a choice symbol's + # visibility changes. + # + # We add these dependencies separately after dependency loop detection. + # The invalidation algorithm can handle the resulting + # <choice symbol> <-> <choice> dependency loops, but they make loop + # detection awkward. + + for choice in self.unique_choices: + for sym in choice.syms: + sym._dependents.add(choice) + + def _invalidate_all(self): + # Undefined symbols never change value and don't need to be + # invalidated, so we can just iterate over defined symbols. + # Invalidating constant symbols would break things horribly. + for sym in self.unique_defined_syms: + sym._invalidate() + + for choice in self.unique_choices: + choice._invalidate() + + # + # Post-parsing menu tree processing, including dependency propagation and + # implicit submenu creation + # + + def _finalize_node(self, node, visible_if): + # Finalizes a menu node and its children: + # + # - Copies properties from menu nodes up to their contained + # symbols/choices + # + # - Propagates dependencies from parent to child nodes + # + # - Creates implicit menus (see kconfig-language.txt) + # + # - Removes 'if' nodes + # + # - Sets 'choice' types and registers choice symbols + # + # menu_finalize() in the C implementation is similar. + # + # node: + # The menu node to finalize. This node and its children will have + # been finalized when the function returns, and any implicit menus + # will have been created. + # + # visible_if: + # Dependencies from 'visible if' on parent menus. These are added to + # the prompts of symbols and choices. + + if node.item.__class__ is Symbol: + # Copy defaults, ranges, selects, and implies to the Symbol + self._add_props_to_sym(node) + + # Find any items that should go in an implicit menu rooted at the + # symbol + cur = node + while cur.next and _auto_menu_dep(node, cur.next): + # This makes implicit submenu creation work recursively, with + # implicit menus inside implicit menus + self._finalize_node(cur.next, visible_if) + cur = cur.next + cur.parent = node + + if cur is not node: + # Found symbols that should go in an implicit submenu. Tilt + # them up above us. + node.list = node.next + node.next = cur.next + cur.next = None + + elif node.list: + # The menu node is a choice, menu, or if. Finalize each child node. + + if node.item is MENU: + visible_if = self._make_and(visible_if, node.visibility) + + # Propagate the menu node's dependencies to each child menu node. + # + # This needs to go before the recursive _finalize_node() call so + # that implicit submenu creation can look ahead at dependencies. + self._propagate_deps(node, visible_if) + + # Finalize the children + cur = node.list + while cur: + self._finalize_node(cur, visible_if) + cur = cur.next + + if node.list: + # node's children have been individually finalized. Do final steps + # to finalize this "level" in the menu tree. + _flatten(node.list) + _remove_ifs(node) + + # Empty choices (node.list None) are possible, so this needs to go + # outside + if node.item.__class__ is Choice: + # Add the node's non-node-specific properties to the choice, like + # _add_props_to_sym() does + choice = node.item + choice.direct_dep = self._make_or(choice.direct_dep, node.dep) + choice.defaults += node.defaults + + _finalize_choice(node) + + def _propagate_deps(self, node, visible_if): + # Propagates 'node's dependencies to its child menu nodes + + # If the parent node holds a Choice, we use the Choice itself as the + # parent dependency. This makes sense as the value (mode) of the choice + # limits the visibility of the contained choice symbols. The C + # implementation works the same way. + # + # Due to the similar interface, Choice works as a drop-in replacement + # for Symbol here. + basedep = node.item if node.item.__class__ is Choice else node.dep + + cur = node.list + while cur: + dep = cur.dep = self._make_and(cur.dep, basedep) + + if cur.item.__class__ in _SYMBOL_CHOICE: + # Propagate 'visible if' and dependencies to the prompt + if cur.prompt: + cur.prompt = (cur.prompt[0], + self._make_and( + cur.prompt[1], + self._make_and(visible_if, dep))) + + # Propagate dependencies to defaults + if cur.defaults: + cur.defaults = [(default, self._make_and(cond, dep)) + for default, cond in cur.defaults] + + # Propagate dependencies to ranges + if cur.ranges: + cur.ranges = [(low, high, self._make_and(cond, dep)) + for low, high, cond in cur.ranges] + + # Propagate dependencies to selects + if cur.selects: + cur.selects = [(target, self._make_and(cond, dep)) + for target, cond in cur.selects] + + # Propagate dependencies to implies + if cur.implies: + cur.implies = [(target, self._make_and(cond, dep)) + for target, cond in cur.implies] + + elif cur.prompt: # Not a symbol/choice + # Propagate dependencies to the prompt. 'visible if' is only + # propagated to symbols/choices. + cur.prompt = (cur.prompt[0], + self._make_and(cur.prompt[1], dep)) + + cur = cur.next + + def _add_props_to_sym(self, node): + # Copies properties from the menu node 'node' up to its contained + # symbol, and adds (weak) reverse dependencies to selected/implied + # symbols. + # + # This can't be rolled into _propagate_deps(), because that function + # traverses the menu tree roughly breadth-first, meaning properties on + # symbols defined in multiple locations could end up in the wrong + # order. + + sym = node.item + + # See the Symbol class docstring + sym.direct_dep = self._make_or(sym.direct_dep, node.dep) + + sym.defaults += node.defaults + sym.ranges += node.ranges + sym.selects += node.selects + sym.implies += node.implies + + # Modify the reverse dependencies of the selected symbol + for target, cond in node.selects: + target.rev_dep = self._make_or( + target.rev_dep, + self._make_and(sym, cond)) + + # Modify the weak reverse dependencies of the implied + # symbol + for target, cond in node.implies: + target.weak_rev_dep = self._make_or( + target.weak_rev_dep, + self._make_and(sym, cond)) + + # + # Misc. + # + + def _check_sym_sanity(self): + # Checks various symbol properties that are handiest to check after + # parsing. Only generates errors and warnings. + + def num_ok(sym, type_): + # Returns True if the (possibly constant) symbol 'sym' is valid as a value + # for a symbol of type type_ (INT or HEX) + + # 'not sym.nodes' implies a constant or undefined symbol, e.g. a plain + # "123" + if not sym.nodes: + return _is_base_n(sym.name, _TYPE_TO_BASE[type_]) + + return sym.orig_type is type_ + + for sym in self.unique_defined_syms: + if sym.orig_type in _BOOL_TRISTATE: + # A helper function could be factored out here, but keep it + # speedy/straightforward + + for target_sym, _ in sym.selects: + if target_sym.orig_type not in _BOOL_TRISTATE_UNKNOWN: + self._warn("{} selects the {} symbol {}, which is not " + "bool or tristate" + .format(sym.name_and_loc, + TYPE_TO_STR[target_sym.orig_type], + target_sym.name_and_loc)) + + for target_sym, _ in sym.implies: + if target_sym.orig_type not in _BOOL_TRISTATE_UNKNOWN: + self._warn("{} implies the {} symbol {}, which is not " + "bool or tristate" + .format(sym.name_and_loc, + TYPE_TO_STR[target_sym.orig_type], + target_sym.name_and_loc)) + + elif sym.orig_type: # STRING/INT/HEX + for default, _ in sym.defaults: + if default.__class__ is not Symbol: + raise KconfigError( + "the {} symbol {} has a malformed default {} -- " + "expected a single symbol" + .format(TYPE_TO_STR[sym.orig_type], + sym.name_and_loc, expr_str(default))) + + if sym.orig_type is STRING: + if not default.is_constant and not default.nodes and \ + not default.name.isupper(): + # 'default foo' on a string symbol could be either a symbol + # reference or someone leaving out the quotes. Guess that + # the quotes were left out if 'foo' isn't all-uppercase + # (and no symbol named 'foo' exists). + self._warn("style: quotes recommended around " + "default value for string symbol " + + sym.name_and_loc) + + elif not num_ok(default, sym.orig_type): # INT/HEX + self._warn("the {0} symbol {1} has a non-{0} default {2}" + .format(TYPE_TO_STR[sym.orig_type], + sym.name_and_loc, + default.name_and_loc)) + + if sym.selects or sym.implies: + self._warn("the {} symbol {} has selects or implies" + .format(TYPE_TO_STR[sym.orig_type], + sym.name_and_loc)) + + else: # UNKNOWN + self._warn("{} defined without a type" + .format(sym.name_and_loc)) + + + if sym.ranges: + if sym.orig_type not in _INT_HEX: + self._warn( + "the {} symbol {} has ranges, but is not int or hex" + .format(TYPE_TO_STR[sym.orig_type], + sym.name_and_loc)) + else: + for low, high, _ in sym.ranges: + if not num_ok(low, sym.orig_type) or \ + not num_ok(high, sym.orig_type): + + self._warn("the {0} symbol {1} has a non-{0} " + "range [{2}, {3}]" + .format(TYPE_TO_STR[sym.orig_type], + sym.name_and_loc, + low.name_and_loc, + high.name_and_loc)) + + def _check_choice_sanity(self): + # Checks various choice properties that are handiest to check after + # parsing. Only generates errors and warnings. + + def warn_select_imply(sym, expr, expr_type): + msg = "the choice symbol {} is {} by the following symbols, but " \ + "select/imply has no effect on choice symbols" \ + .format(sym.name_and_loc, expr_type) + + # si = select/imply + for si in split_expr(expr, OR): + msg += "\n - " + split_expr(si, AND)[0].name_and_loc + + self._warn(msg) + + for choice in self.unique_choices: + if choice.orig_type not in _BOOL_TRISTATE: + self._warn("{} defined with type {}" + .format(choice.name_and_loc, + TYPE_TO_STR[choice.orig_type])) + + for node in choice.nodes: + if node.prompt: + break + else: + self._warn(choice.name_and_loc + " defined without a prompt") + + for default, _ in choice.defaults: + if default.__class__ is not Symbol: + raise KconfigError( + "{} has a malformed default {}" + .format(choice.name_and_loc, expr_str(default))) + + if default.choice is not choice: + self._warn("the default selection {} of {} is not " + "contained in the choice" + .format(default.name_and_loc, + choice.name_and_loc)) + + for sym in choice.syms: + if sym.defaults: + self._warn("default on the choice symbol {} will have " + "no effect, as defaults do not affect choice " + "symbols".format(sym.name_and_loc)) + + if sym.rev_dep is not sym.kconfig.n: + warn_select_imply(sym, sym.rev_dep, "selected") + + if sym.weak_rev_dep is not sym.kconfig.n: + warn_select_imply(sym, sym.weak_rev_dep, "implied") + + for node in sym.nodes: + if node.parent.item is choice: + if not node.prompt: + self._warn("the choice symbol {} has no prompt" + .format(sym.name_and_loc)) + + elif node.prompt: + self._warn("the choice symbol {} is defined with a " + "prompt outside the choice" + .format(sym.name_and_loc)) + + def _parse_error(self, msg): + raise KconfigError("{}error: couldn't parse '{}': {}".format( + "" if self.filename is None else + "{}:{}: ".format(self.filename, self.linenr), + self._line.strip(), msg)) + + def _trailing_tokens_error(self): + self._parse_error("extra tokens at end of line") + + def _open(self, filename, mode): + # open() wrapper: + # + # - Enable universal newlines mode on Python 2 to ease + # interoperability between Linux and Windows. It's already the + # default on Python 3. + # + # The "U" flag would currently work for both Python 2 and 3, but it's + # deprecated on Python 3, so play it future-safe. + # + # io.open() defaults to universal newlines on Python 2 (and is an + # alias for open() on Python 3), but it returns 'unicode' strings and + # slows things down: + # + # Parsing x86 Kconfigs on Python 2 + # + # with open(..., "rU"): + # + # real 0m0.930s + # user 0m0.905s + # sys 0m0.025s + # + # with io.open(): + # + # real 0m1.069s + # user 0m1.040s + # sys 0m0.029s + # + # There's no appreciable performance difference between "r" and + # "rU" for parsing performance on Python 2. + # + # - For Python 3, force the encoding. Forcing the encoding on Python 2 + # turns strings into Unicode strings, which gets messy. Python 2 + # doesn't decode regular strings anyway. + return open(filename, "rU" if mode == "r" else mode) if _IS_PY2 else \ + open(filename, mode, encoding=self._encoding) + + def _check_undef_syms(self): + # Prints warnings for all references to undefined symbols within the + # Kconfig files + + def is_num(s): + # Returns True if the string 's' looks like a number. + # + # Internally, all operands in Kconfig are symbols, only undefined symbols + # (which numbers usually are) get their name as their value. + # + # Only hex numbers that start with 0x/0X are classified as numbers. + # Otherwise, symbols whose names happen to contain only the letters A-F + # would trigger false positives. + + try: + int(s) + except ValueError: + if not s.startswith(("0x", "0X")): + return False + + try: + int(s, 16) + except ValueError: + return False + + return True + + for sym in (self.syms.viewvalues if _IS_PY2 else self.syms.values)(): + # - sym.nodes empty means the symbol is undefined (has no + # definition locations) + # + # - Due to Kconfig internals, numbers show up as undefined Kconfig + # symbols, but shouldn't be flagged + # + # - The MODULES symbol always exists + if not sym.nodes and not is_num(sym.name) and \ + sym.name != "MODULES": + + msg = "undefined symbol {}:".format(sym.name) + for node in self.node_iter(): + if sym in node.referenced: + msg += "\n\n- Referenced at {}:{}:\n\n{}" \ + .format(node.filename, node.linenr, node) + self._warn(msg) + + def _warn(self, msg, filename=None, linenr=None): + # For printing general warnings + + if not self.warn: + return + + msg = "warning: " + msg + if filename is not None: + msg = "{}:{}: {}".format(filename, linenr, msg) + + self.warnings.append(msg) + if self.warn_to_stderr: + sys.stderr.write(msg + "\n") + + +class Symbol(object): + """ + Represents a configuration symbol: + + (menu)config FOO + ... + + The following attributes are available. They should be viewed as read-only, + and some are implemented through @property magic (but are still efficient + to access due to internal caching). + + Note: Prompts, help texts, and locations are stored in the Symbol's + MenuNode(s) rather than in the Symbol itself. Check the MenuNode class and + the Symbol.nodes attribute. This organization matches the C tools. + + name: + The name of the symbol, e.g. "FOO" for 'config FOO'. + + type: + The type of the symbol. One of BOOL, TRISTATE, STRING, INT, HEX, UNKNOWN. + UNKNOWN is for undefined symbols, (non-special) constant symbols, and + symbols defined without a type. + + When running without modules (MODULES having the value n), TRISTATE + symbols magically change type to BOOL. This also happens for symbols + within choices in "y" mode. This matches the C tools, and makes sense for + menuconfig-like functionality. + + orig_type: + The type as given in the Kconfig file, without any magic applied. Used + when printing the symbol. + + tri_value: + The tristate value of the symbol as an integer. One of 0, 1, 2, + representing n, m, y. Always 0 (n) for non-bool/tristate symbols. + + This is the symbol value that's used outside of relation expressions + (A, !A, A && B, A || B). + + str_value: + The value of the symbol as a string. Gives the value for string/int/hex + symbols. For bool/tristate symbols, gives "n", "m", or "y". + + This is the symbol value that's used in relational expressions + (A = B, A != B, etc.) + + Gotcha: For int/hex symbols, the exact format of the value is often + preserved (e.g. when writing a .config file), hence why you can't get it + directly as an int. Do int(int_sym.str_value) or + int(hex_sym.str_value, 16) to get the integer value. + + user_value: + The user value of the symbol. None if no user value has been assigned + (via Kconfig.load_config() or Symbol.set_value()). + + Holds 0, 1, or 2 for bool/tristate symbols, and a string for the other + symbol types. + + WARNING: Do not assign directly to this. It will break things. Use + Symbol.set_value(). + + assignable: + A tuple containing the tristate user values that can currently be + assigned to the symbol (that would be respected), ordered from lowest (0, + representing n) to highest (2, representing y). This corresponds to the + selections available in the menuconfig interface. The set of assignable + values is calculated from the symbol's visibility and selects/implies. + + Returns the empty set for non-bool/tristate symbols and for symbols with + visibility n. The other possible values are (0, 2), (0, 1, 2), (1, 2), + (1,), and (2,). A (1,) or (2,) result means the symbol is visible but + "locked" to m or y through a select, perhaps in combination with the + visibility. menuconfig represents this as -M- and -*-, respectively. + + For string/hex/int symbols, check if Symbol.visibility is non-0 (non-n) + instead to determine if the value can be changed. + + Some handy 'assignable' idioms: + + # Is 'sym' an assignable (visible) bool/tristate symbol? + if sym.assignable: + # What's the highest value it can be assigned? [-1] in Python + # gives the last element. + sym_high = sym.assignable[-1] + + # The lowest? + sym_low = sym.assignable[0] + + # Can the symbol be set to at least m? + if sym.assignable[-1] >= 1: + ... + + # Can the symbol be set to m? + if 1 in sym.assignable: + ... + + visibility: + The visibility of the symbol. One of 0, 1, 2, representing n, m, y. See + the module documentation for an overview of symbol values and visibility. + + config_string: + The .config assignment string that would get written out for the symbol + by Kconfig.write_config(). Returns the empty string if no .config + assignment would get written out. + + In general, visible symbols, symbols with (active) defaults, and selected + symbols get written out. This includes all non-n-valued bool/tristate + symbols, and all visible string/int/hex symbols. + + Symbols with the (no longer needed) 'option env=...' option generate no + configuration output, and neither does the special + 'option defconfig_list' symbol. + + Tip: This field is useful when generating custom configuration output, + even for non-.config-like formats. To write just the symbols that would + get written out to .config files, do this: + + if sym.config_string: + *Write symbol, e.g. by looking sym.str_value* + + This is a superset of the symbols written out by write_autoconf(). + That function skips all n-valued symbols. + + There usually won't be any great harm in just writing all symbols either, + though you might get some special symbols and possibly some "redundant" + n-valued symbol entries in there. + + name_and_loc: + Holds a string like + + "MY_SYMBOL (defined at foo/Kconfig:12, bar/Kconfig:14)" + + , giving the name of the symbol and its definition location(s). + + If the symbol is undefined, the location is given as "(undefined)". + + nodes: + A list of MenuNodes for this symbol. Will contain a single MenuNode for + most symbols. Undefined and constant symbols have an empty nodes list. + Symbols defined in multiple locations get one node for each location. + + choice: + Holds the parent Choice for choice symbols, and None for non-choice + symbols. Doubles as a flag for whether a symbol is a choice symbol. + + defaults: + List of (default, cond) tuples for the symbol's 'default' properties. For + example, 'default A && B if C || D' is represented as + ((AND, A, B), (OR, C, D)). If no condition was given, 'cond' is + self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to + 'default' conditions. + + selects: + List of (symbol, cond) tuples for the symbol's 'select' properties. For + example, 'select A if B && C' is represented as (A, (AND, B, C)). If no + condition was given, 'cond' is self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to 'select' + conditions. + + implies: + Like 'selects', for imply. + + ranges: + List of (low, high, cond) tuples for the symbol's 'range' properties. For + example, 'range 1 2 if A' is represented as (1, 2, A). If there is no + condition, 'cond' is self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to 'range' + conditions. + + Gotcha: 1 and 2 above will be represented as (undefined) Symbols rather + than plain integers. Undefined symbols get their name as their string + value, so this works out. The C tools work the same way. + + orig_defaults: + orig_selects: + orig_implies: + orig_ranges: + See the corresponding attributes on the MenuNode class. + + rev_dep: + Reverse dependency expression from other symbols selecting this symbol. + Multiple selections get ORed together. A condition on a select is ANDed + with the selecting symbol. + + For example, if A has 'select FOO' and B has 'select FOO if C', then + FOO's rev_dep will be (OR, A, (AND, B, C)). + + weak_rev_dep: + Like rev_dep, for imply. + + direct_dep: + The direct ('depends on') dependencies for the symbol, or self.kconfig.y + if there are no direct dependencies. + + This attribute includes any dependencies from surrounding menus and ifs. + Those get propagated to the direct dependencies, and the resulting direct + dependencies in turn get propagated to the conditions of all properties. + + If the symbol is defined in multiple locations, the dependencies from the + different locations get ORed together. + + referenced: + A set() with all symbols and choices referenced in the properties and + property conditions of the symbol. + + Also includes dependencies from surrounding menus and ifs, because those + get propagated to the symbol (see the 'Intro to symbol values' section in + the module docstring). + + Choices appear in the dependencies of choice symbols. + + For the following definitions, only B and not C appears in A's + 'referenced'. To get transitive references, you'll have to recursively + expand 'references' until no new items appear. + + config A + bool + depends on B + + config B + bool + depends on C + + config C + bool + + See the Symbol.direct_dep attribute if you're only interested in the + direct dependencies of the symbol (its 'depends on'). You can extract the + symbols in it with the global expr_items() function. + + env_var: + If the Symbol has an 'option env="FOO"' option, this contains the name + ("FOO") of the environment variable. None for symbols without no + 'option env'. + + 'option env="FOO"' acts like a 'default' property whose value is the + value of $FOO. + + Symbols with 'option env' are never written out to .config files, even if + they are visible. env_var corresponds to a flag called SYMBOL_AUTO in the + C implementation. + + is_allnoconfig_y: + True if the symbol has 'option allnoconfig_y' set on it. This has no + effect internally (except when printing symbols), but can be checked by + scripts. + + is_constant: + True if the symbol is a constant (quoted) symbol. + + kconfig: + The Kconfig instance this symbol is from. + """ + __slots__ = ( + "_cached_assignable", + "_cached_str_val", + "_cached_tri_val", + "_cached_vis", + "_dependents", + "_old_val", + "_visited", + "_was_set", + "_write_to_conf", + "choice", + "defaults", + "direct_dep", + "env_var", + "implies", + "is_allnoconfig_y", + "is_constant", + "kconfig", + "name", + "nodes", + "orig_type", + "ranges", + "rev_dep", + "selects", + "user_value", + "weak_rev_dep", + ) + + # + # Public interface + # + + @property + def type(self): + """ + See the class documentation. + """ + if self.orig_type is TRISTATE and \ + (self.choice and self.choice.tri_value == 2 or + not self.kconfig.modules.tri_value): + + return BOOL + + return self.orig_type + + @property + def str_value(self): + """ + See the class documentation. + """ + if self._cached_str_val is not None: + return self._cached_str_val + + if self.orig_type in _BOOL_TRISTATE: + # Also calculates the visibility, so invalidation safe + self._cached_str_val = TRI_TO_STR[self.tri_value] + return self._cached_str_val + + # As a quirk of Kconfig, undefined symbols get their name as their + # string value. This is why things like "FOO = bar" work for seeing if + # FOO has the value "bar". + if not self.orig_type: # UNKNOWN + self._cached_str_val = self.name + return self.name + + val = "" + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + + self._write_to_conf = (vis != 0) + + if self.orig_type in _INT_HEX: + # The C implementation checks the user value against the range in a + # separate code path (post-processing after loading a .config). + # Checking all values here instead makes more sense for us. It + # requires that we check for a range first. + + base = _TYPE_TO_BASE[self.orig_type] + + # Check if a range is in effect + for low_expr, high_expr, cond in self.ranges: + if expr_value(cond): + has_active_range = True + + # The zeros are from the C implementation running strtoll() + # on empty strings + low = int(low_expr.str_value, base) if \ + _is_base_n(low_expr.str_value, base) else 0 + high = int(high_expr.str_value, base) if \ + _is_base_n(high_expr.str_value, base) else 0 + + break + else: + has_active_range = False + + # Defaults are used if the symbol is invisible, lacks a user value, + # or has an out-of-range user value + use_defaults = True + + if vis and self.user_value: + user_val = int(self.user_value, base) + if has_active_range and not low <= user_val <= high: + num2str = str if base == 10 else hex + self.kconfig._warn( + "user value {} on the {} symbol {} ignored due to " + "being outside the active range ([{}, {}]) -- falling " + "back on defaults" + .format(num2str(user_val), TYPE_TO_STR[self.orig_type], + self.name_and_loc, + num2str(low), num2str(high))) + else: + # If the user value is well-formed and satisfies range + # contraints, it is stored in exactly the same form as + # specified in the assignment (with or without "0x", etc.) + val = self.user_value + use_defaults = False + + if use_defaults: + # No user value or invalid user value. Look at defaults. + + # Used to implement the warning below + has_default = False + + for sym, cond in self.defaults: + if expr_value(cond): + has_default = self._write_to_conf = True + + val = sym.str_value + + if _is_base_n(val, base): + val_num = int(val, base) + else: + val_num = 0 # strtoll() on empty string + + break + else: + val_num = 0 # strtoll() on empty string + + # This clamping procedure runs even if there's no default + if has_active_range: + clamp = None + if val_num < low: + clamp = low + elif val_num > high: + clamp = high + + if clamp is not None: + # The value is rewritten to a standard form if it is + # clamped + val = str(clamp) \ + if self.orig_type is INT else \ + hex(clamp) + + if has_default: + num2str = str if base == 10 else hex + self.kconfig._warn( + "default value {} on {} clamped to {} due to " + "being outside the active range ([{}, {}])" + .format(val_num, self.name_and_loc, + num2str(clamp), num2str(low), + num2str(high))) + + elif self.orig_type is STRING: + if vis and self.user_value is not None: + # If the symbol is visible and has a user value, use that + val = self.user_value + else: + # Otherwise, look at defaults + for sym, cond in self.defaults: + if expr_value(cond): + val = sym.str_value + self._write_to_conf = True + break + + # env_var corresponds to SYMBOL_AUTO in the C implementation, and is + # also set on the defconfig_list symbol there. Test for the + # defconfig_list symbol explicitly instead here, to avoid a nonsensical + # env_var setting and the defconfig_list symbol being printed + # incorrectly. This code is pretty cold anyway. + if self.env_var is not None or self is self.kconfig.defconfig_list: + self._write_to_conf = False + + self._cached_str_val = val + return val + + @property + def tri_value(self): + """ + See the class documentation. + """ + if self._cached_tri_val is not None: + return self._cached_tri_val + + if self.orig_type not in _BOOL_TRISTATE: + if self.orig_type: # != UNKNOWN + # Would take some work to give the location here + self.kconfig._warn( + "The {} symbol {} is being evaluated in a logical context " + "somewhere. It will always evaluate to n." + .format(TYPE_TO_STR[self.orig_type], self.name_and_loc)) + + self._cached_tri_val = 0 + return 0 + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + self._write_to_conf = (vis != 0) + + val = 0 + + if not self.choice: + # Non-choice symbol + + if vis and self.user_value is not None: + # If the symbol is visible and has a user value, use that + val = min(self.user_value, vis) + + else: + # Otherwise, look at defaults and weak reverse dependencies + # (implies) + + for default, cond in self.defaults: + dep_val = expr_value(cond) + if dep_val: + val = min(expr_value(default), dep_val) + if val: + self._write_to_conf = True + break + + # Weak reverse dependencies are only considered if our + # direct dependencies are met + dep_val = expr_value(self.weak_rev_dep) + if dep_val and expr_value(self.direct_dep): + val = max(dep_val, val) + self._write_to_conf = True + + # Reverse (select-related) dependencies take precedence + dep_val = expr_value(self.rev_dep) + if dep_val: + if expr_value(self.direct_dep) < dep_val: + self._warn_select_unsatisfied_deps() + + val = max(dep_val, val) + self._write_to_conf = True + + # m is promoted to y for (1) bool symbols and (2) symbols with a + # weak_rev_dep (from imply) of y + if val == 1 and \ + (self.type is BOOL or expr_value(self.weak_rev_dep) == 2): + val = 2 + + elif vis == 2: + # Visible choice symbol in y-mode choice. The choice mode limits + # the visibility of choice symbols, so it's sufficient to just + # check the visibility of the choice symbols themselves. + val = 2 if self.choice.selection is self else 0 + + elif vis and self.user_value: + # Visible choice symbol in m-mode choice, with set non-0 user value + val = 1 + + self._cached_tri_val = val + return val + + @property + def assignable(self): + """ + See the class documentation. + """ + if self._cached_assignable is None: + self._cached_assignable = self._assignable() + return self._cached_assignable + + @property + def visibility(self): + """ + See the class documentation. + """ + if self._cached_vis is None: + self._cached_vis = _visibility(self) + return self._cached_vis + + @property + def config_string(self): + """ + See the class documentation. + """ + # _write_to_conf is determined when the value is calculated. This is a + # hidden function call due to property magic. + val = self.str_value + if not self._write_to_conf: + return "" + + if self.orig_type in _BOOL_TRISTATE: + return "{}{}={}\n" \ + .format(self.kconfig.config_prefix, self.name, val) \ + if val != "n" else \ + "# {}{} is not set\n" \ + .format(self.kconfig.config_prefix, self.name) + + if self.orig_type in _INT_HEX: + return "{}{}={}\n" \ + .format(self.kconfig.config_prefix, self.name, val) + + # sym.orig_type is STRING + return '{}{}="{}"\n' \ + .format(self.kconfig.config_prefix, self.name, escape(val)) + + @property + def name_and_loc(self): + """ + See the class documentation. + """ + return self.name + " " + _locs(self) + + def set_value(self, value): + """ + Sets the user value of the symbol. + + Equal in effect to assigning the value to the symbol within a .config + file. For bool and tristate symbols, use the 'assignable' attribute to + check which values can currently be assigned. Setting values outside + 'assignable' will cause Symbol.user_value to differ from + Symbol.str/tri_value (be truncated down or up). + + Setting a choice symbol to 2 (y) sets Choice.user_selection to the + choice symbol in addition to setting Symbol.user_value. + Choice.user_selection is considered when the choice is in y mode (the + "normal" mode). + + Other symbols that depend (possibly indirectly) on this symbol are + automatically recalculated to reflect the assigned value. + + value: + The user value to give to the symbol. For bool and tristate symbols, + n/m/y can be specified either as 0/1/2 (the usual format for tristate + values in Kconfiglib) or as one of the strings "n", "m", or "y". For + other symbol types, pass a string. + + Note that the value for an int/hex symbol is passed as a string, e.g. + "123" or "0x0123". The format of this string is preserved in the + output. + + Values that are invalid for the type (such as "foo" or 1 (m) for a + BOOL or "0x123" for an INT) are ignored and won't be stored in + Symbol.user_value. Kconfiglib will print a warning by default for + invalid assignments, and set_value() will return False. + + Returns True if the value is valid for the type of the symbol, and + False otherwise. This only looks at the form of the value. For BOOL and + TRISTATE symbols, check the Symbol.assignable attribute to see what + values are currently in range and would actually be reflected in the + value of the symbol. For other symbol types, check whether the + visibility is non-n. + """ + if self.orig_type in _BOOL_TRISTATE and value in STR_TO_TRI: + value = STR_TO_TRI[value] + + # If the new user value matches the old, nothing changes, and we can + # avoid invalidating cached values. + # + # This optimization is skipped for choice symbols: Setting a choice + # symbol's user value to y might change the state of the choice, so it + # wouldn't be safe (symbol user values always match the values set in a + # .config file or via set_value(), and are never implicitly updated). + if value == self.user_value and not self.choice: + self._was_set = True + return True + + # Check if the value is valid for our type + if not (self.orig_type is BOOL and value in (2, 0) or + self.orig_type is TRISTATE and value in TRI_TO_STR or + value.__class__ is str and + (self.orig_type is STRING or + self.orig_type is INT and _is_base_n(value, 10) or + self.orig_type is HEX and _is_base_n(value, 16) + and int(value, 16) >= 0)): + + # Display tristate values as n, m, y in the warning + self.kconfig._warn( + "the value {} is invalid for {}, which has type {} -- " + "assignment ignored" + .format(TRI_TO_STR[value] if value in TRI_TO_STR else + "'{}'".format(value), + self.name_and_loc, TYPE_TO_STR[self.orig_type])) + + return False + + self.user_value = value + self._was_set = True + + if self.choice and value == 2: + # Setting a choice symbol to y makes it the user selection of the + # choice. Like for symbol user values, the user selection is not + # guaranteed to match the actual selection of the choice, as + # dependencies come into play. + self.choice.user_selection = self + self.choice._was_set = True + self.choice._rec_invalidate() + else: + self._rec_invalidate_if_has_prompt() + + return True + + def unset_value(self): + """ + Removes any user value from the symbol, as if the symbol had never + gotten a user value via Kconfig.load_config() or Symbol.set_value(). + """ + if self.user_value is not None: + self.user_value = None + self._rec_invalidate_if_has_prompt() + + @property + def referenced(self): + """ + See the class documentation. + """ + return {item for node in self.nodes for item in node.referenced} + + @property + def orig_defaults(self): + """ + See the class documentation. + """ + return [d for node in self.nodes for d in node.orig_defaults] + + @property + def orig_selects(self): + """ + See the class documentation. + """ + return [s for node in self.nodes for s in node.orig_selects] + + @property + def orig_implies(self): + """ + See the class documentation. + """ + return [i for node in self.nodes for i in node.orig_implies] + + @property + def orig_ranges(self): + """ + See the class documentation. + """ + return [r for node in self.nodes for r in node.orig_ranges] + + def __repr__(self): + """ + Returns a string with information about the symbol (including its name, + value, visibility, and location(s)) when it is evaluated on e.g. the + interactive Python prompt. + """ + fields = ["symbol " + self.name, TYPE_TO_STR[self.type]] + add = fields.append + + for node in self.nodes: + if node.prompt: + add('"{}"'.format(node.prompt[0])) + + # Only add quotes for non-bool/tristate symbols + add("value " + (self.str_value if self.orig_type in _BOOL_TRISTATE + else '"{}"'.format(self.str_value))) + + if not self.is_constant: + # These aren't helpful to show for constant symbols + + if self.user_value is not None: + # Only add quotes for non-bool/tristate symbols + add("user value " + (TRI_TO_STR[self.user_value] + if self.orig_type in _BOOL_TRISTATE + else '"{}"'.format(self.user_value))) + + add("visibility " + TRI_TO_STR[self.visibility]) + + if self.choice: + add("choice symbol") + + if self.is_allnoconfig_y: + add("allnoconfig_y") + + if self is self.kconfig.defconfig_list: + add("is the defconfig_list symbol") + + if self.env_var is not None: + add("from environment variable " + self.env_var) + + if self is self.kconfig.modules: + add("is the modules symbol") + + add("direct deps " + TRI_TO_STR[expr_value(self.direct_dep)]) + + if self.nodes: + for node in self.nodes: + add("{}:{}".format(node.filename, node.linenr)) + else: + add("constant" if self.is_constant else "undefined") + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the symbol when it is printed. + Matches the Kconfig format, with any parent dependencies propagated to + the 'depends on' condition. + + The string is constructed by joining the strings returned by + MenuNode.__str__() for each of the symbol's menu nodes, so symbols + defined in multiple locations will return a string with all + definitions. + + The returned string does not end in a newline. An empty string is + returned for undefined and constant symbols. + """ + return self.custom_str(standard_sc_expr_str) + + def custom_str(self, sc_expr_str_fn): + """ + Works like Symbol.__str__(), but allows a custom format to be used for + all symbol/choice references. See expr_str(). + """ + return "\n\n".join(node.custom_str(sc_expr_str_fn) + for node in self.nodes) + + # + # Private methods + # + + def __init__(self): + """ + Symbol constructor -- not intended to be called directly by Kconfiglib + clients. + """ + # These attributes are always set on the instance from outside and + # don't need defaults: + # kconfig + # direct_dep + # is_constant + # name + # rev_dep + # weak_rev_dep + + # - UNKNOWN == 0 + # - _visited is used during tree iteration and dep. loop detection + self.orig_type = self._visited = 0 + + self.nodes = [] + + self.defaults = [] + self.selects = [] + self.implies = [] + self.ranges = [] + + self.user_value = \ + self.choice = \ + self.env_var = \ + self._cached_str_val = self._cached_tri_val = self._cached_vis = \ + self._cached_assignable = None + + # _write_to_conf is calculated along with the value. If True, the + # Symbol gets a .config entry. + + self.is_allnoconfig_y = \ + self._was_set = \ + self._write_to_conf = False + + # See Kconfig._build_dep() + self._dependents = set() + + def _assignable(self): + # Worker function for the 'assignable' attribute + + if self.orig_type not in _BOOL_TRISTATE: + return () + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + if not vis: + return () + + rev_dep_val = expr_value(self.rev_dep) + + if vis == 2: + if self.choice: + return (2,) + + if not rev_dep_val: + if self.type is BOOL or expr_value(self.weak_rev_dep) == 2: + return (0, 2) + return (0, 1, 2) + + if rev_dep_val == 2: + return (2,) + + # rev_dep_val == 1 + + if self.type is BOOL or expr_value(self.weak_rev_dep) == 2: + return (2,) + return (1, 2) + + # vis == 1 + + # Must be a tristate here, because bool m visibility gets promoted to y + + if not rev_dep_val: + return (0, 1) if expr_value(self.weak_rev_dep) != 2 else (0, 2) + + if rev_dep_val == 2: + return (2,) + + # vis == rev_dep_val == 1 + + return (1,) + + def _invalidate(self): + # Marks the symbol as needing to be recalculated + + self._cached_str_val = self._cached_tri_val = self._cached_vis = \ + self._cached_assignable = None + + def _rec_invalidate(self): + # Invalidates the symbol and all items that (possibly) depend on it + + if self is self.kconfig.modules: + # Invalidating MODULES has wide-ranging effects + self.kconfig._invalidate_all() + else: + self._invalidate() + + for item in self._dependents: + # _cached_vis doubles as a flag that tells us whether 'item' + # has cached values, because it's calculated as a side effect + # of calculating all other (non-constant) cached values. + # + # If item._cached_vis is None, it means there can't be cached + # values on other items that depend on 'item', because if there + # were, some value on 'item' would have been calculated and + # item._cached_vis set as a side effect. It's therefore safe to + # stop the invalidation at symbols with _cached_vis None. + # + # This approach massively speeds up scripts that set a lot of + # values, vs simply invalidating all possibly dependent symbols + # (even when you already have a list of all the dependent + # symbols, because some symbols get huge dependency trees). + # + # This gracefully handles dependency loops too, which is nice + # for choices, where the choice depends on the choice symbols + # and vice versa. + if item._cached_vis is not None: + item._rec_invalidate() + + def _rec_invalidate_if_has_prompt(self): + # Invalidates the symbol and its dependent symbols, but only if the + # symbol has a prompt. User values never have an effect on promptless + # symbols, so we skip invalidation for them as an optimization. + # + # This also prevents constant (quoted) symbols from being invalidated + # if set_value() is called on them, which would make them lose their + # value and break things. + # + # Prints a warning if the symbol has no prompt. In some contexts (e.g. + # when loading a .config files) assignments to promptless symbols are + # normal and expected, so the warning can be disabled. + + for node in self.nodes: + if node.prompt: + self._rec_invalidate() + return + + if self.kconfig._warn_assign_no_prompt: + self.kconfig._warn(self.name_and_loc + " has no prompt, meaning " + "user values have no effect on it") + + def _str_default(self): + # write_min_config() helper function. Returns the value the symbol + # would get from defaults if it didn't have a user value. Uses exactly + # the same algorithm as the C implementation (though a bit cleaned up), + # for compatibility. + + if self.orig_type in _BOOL_TRISTATE: + val = 0 + + # Defaults, selects, and implies do not affect choice symbols + if not self.choice: + for default, cond in self.defaults: + cond_val = expr_value(cond) + if cond_val: + val = min(expr_value(default), cond_val) + break + + val = max(expr_value(self.rev_dep), + expr_value(self.weak_rev_dep), + val) + + # Transpose mod to yes if type is bool (possibly due to modules + # being disabled) + if val == 1 and self.type is BOOL: + val = 2 + + return TRI_TO_STR[val] + + if self.orig_type: # STRING/INT/HEX + for default, cond in self.defaults: + if expr_value(cond): + return default.str_value + + return "" + + def _warn_select_unsatisfied_deps(self): + # Helper for printing an informative warning when a symbol with + # unsatisfied direct dependencies (dependencies from 'depends on', ifs, + # and menus) is selected by some other symbol. Also warn if a symbol + # whose direct dependencies evaluate to m is selected to y. + + msg = "{} has direct dependencies {} with value {}, but is " \ + "currently being {}-selected by the following symbols:" \ + .format(self.name_and_loc, expr_str(self.direct_dep), + TRI_TO_STR[expr_value(self.direct_dep)], + TRI_TO_STR[expr_value(self.rev_dep)]) + + # The reverse dependencies from each select are ORed together + for select in split_expr(self.rev_dep, OR): + if expr_value(select) <= expr_value(self.direct_dep): + # Only include selects that exceed the direct dependencies + continue + + # - 'select A if B' turns into A && B + # - 'select A' just turns into A + # + # In both cases, we can split on AND and pick the first operand + selecting_sym = split_expr(select, AND)[0] + + msg += "\n - {}, with value {}, direct dependencies {} " \ + "(value: {})" \ + .format(selecting_sym.name_and_loc, + selecting_sym.str_value, + expr_str(selecting_sym.direct_dep), + TRI_TO_STR[expr_value(selecting_sym.direct_dep)]) + + if select.__class__ is tuple: + msg += ", and select condition {} (value: {})" \ + .format(expr_str(select[2]), + TRI_TO_STR[expr_value(select[2])]) + + self.kconfig._warn(msg) + + +class Choice(object): + """ + Represents a choice statement: + + choice + ... + endchoice + + The following attributes are available on Choice instances. They should be + treated as read-only, and some are implemented through @property magic (but + are still efficient to access due to internal caching). + + Note: Prompts, help texts, and locations are stored in the Choice's + MenuNode(s) rather than in the Choice itself. Check the MenuNode class and + the Choice.nodes attribute. This organization matches the C tools. + + name: + The name of the choice, e.g. "FOO" for 'choice FOO', or None if the + Choice has no name. + + type: + The type of the choice. One of BOOL, TRISTATE, UNKNOWN. UNKNOWN is for + choices defined without a type where none of the contained symbols have a + type either (otherwise the choice inherits the type of the first symbol + defined with a type). + + When running without modules (CONFIG_MODULES=n), TRISTATE choices + magically change type to BOOL. This matches the C tools, and makes sense + for menuconfig-like functionality. + + orig_type: + The type as given in the Kconfig file, without any magic applied. Used + when printing the choice. + + tri_value: + The tristate value (mode) of the choice. A choice can be in one of three + modes: + + 0 (n) - The choice is disabled and no symbols can be selected. For + visible choices, this mode is only possible for choices with + the 'optional' flag set (see kconfig-language.txt). + + 1 (m) - Any number of choice symbols can be set to m, the rest will + be n. + + 2 (y) - One symbol will be y, the rest n. + + Only tristate choices can be in m mode. The visibility of the choice is + an upper bound on the mode, and the mode in turn is an upper bound on the + visibility of the choice symbols. + + To change the mode, use Choice.set_value(). + + Implementation note: + The C tools internally represent choices as a type of symbol, with + special-casing in many code paths. This is why there is a lot of + similarity to Symbol. The value (mode) of a choice is really just a + normal symbol value, and an implicit reverse dependency forces its + lower bound to m for visible non-optional choices (the reverse + dependency is 'm && <visibility>'). + + Symbols within choices get the choice propagated as a dependency to + their properties. This turns the mode of the choice into an upper bound + on e.g. the visibility of choice symbols, and explains the gotcha + related to printing choice symbols mentioned in the module docstring. + + Kconfiglib uses a separate Choice class only because it makes the code + and interface less confusing (especially in a user-facing interface). + Corresponding attributes have the same name in the Symbol and Choice + classes, for consistency and compatibility. + + str_value: + Like choice.tri_value, but gives the value as one of the strings + "n", "m", or "y" + + user_value: + The value (mode) selected by the user through Choice.set_value(). Either + 0, 1, or 2, or None if the user hasn't selected a mode. See + Symbol.user_value. + + WARNING: Do not assign directly to this. It will break things. Use + Choice.set_value() instead. + + assignable: + See the symbol class documentation. Gives the assignable values (modes). + + selection: + The Symbol instance of the currently selected symbol. None if the Choice + is not in y mode or has no selected symbol (due to unsatisfied + dependencies on choice symbols). + + WARNING: Do not assign directly to this. It will break things. Call + sym.set_value(2) on the choice symbol you want to select instead. + + user_selection: + The symbol selected by the user (by setting it to y). Ignored if the + choice is not in y mode, but still remembered so that the choice "snaps + back" to the user selection if the mode is changed back to y. This might + differ from 'selection' due to unsatisfied dependencies. + + WARNING: Do not assign directly to this. It will break things. Call + sym.set_value(2) on the choice symbol to be selected instead. + + visibility: + See the Symbol class documentation. Acts on the value (mode). + + name_and_loc: + Holds a string like + + "<choice MY_CHOICE> (defined at foo/Kconfig:12)" + + , giving the name of the choice and its definition location(s). If the + choice has no name (isn't defined with 'choice MY_CHOICE'), then it will + be shown as "<choice>" before the list of locations (always a single one + in that case). + + syms: + List of symbols contained in the choice. + + Obscure gotcha: If a symbol depends on the previous symbol within a + choice so that an implicit menu is created, it won't be a choice symbol, + and won't be included in 'syms'. + + nodes: + A list of MenuNodes for this choice. In practice, the list will probably + always contain a single MenuNode, but it is possible to give a choice a + name and define it in multiple locations. + + defaults: + List of (symbol, cond) tuples for the choice's 'defaults' properties. For + example, 'default A if B && C' is represented as (A, (AND, B, C)). If + there is no condition, 'cond' is self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to + 'default' conditions. + + orig_defaults: + See the corresponding attribute on the MenuNode class. + + direct_dep: + See Symbol.direct_dep. + + referenced: + A set() with all symbols referenced in the properties and property + conditions of the choice. + + Also includes dependencies from surrounding menus and ifs, because those + get propagated to the choice (see the 'Intro to symbol values' section in + the module docstring). + + is_optional: + True if the choice has the 'optional' flag set on it and can be in + n mode. + + kconfig: + The Kconfig instance this choice is from. + """ + __slots__ = ( + "_cached_assignable", + "_cached_selection", + "_cached_vis", + "_dependents", + "_visited", + "_was_set", + "defaults", + "direct_dep", + "is_constant", + "is_optional", + "kconfig", + "name", + "nodes", + "orig_type", + "syms", + "user_selection", + "user_value", + ) + + # + # Public interface + # + + @property + def type(self): + """ + Returns the type of the choice. See Symbol.type. + """ + if self.orig_type is TRISTATE and not self.kconfig.modules.tri_value: + return BOOL + return self.orig_type + + @property + def str_value(self): + """ + See the class documentation. + """ + return TRI_TO_STR[self.tri_value] + + @property + def tri_value(self): + """ + See the class documentation. + """ + # This emulates a reverse dependency of 'm && visibility' for + # non-optional choices, which is how the C implementation does it + + val = 0 if self.is_optional else 1 + + if self.user_value is not None: + val = max(val, self.user_value) + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + val = min(val, self.visibility) + + # Promote m to y for boolean choices + return 2 if val == 1 and self.type is BOOL else val + + @property + def assignable(self): + """ + See the class documentation. + """ + if self._cached_assignable is None: + self._cached_assignable = self._assignable() + return self._cached_assignable + + @property + def visibility(self): + """ + See the class documentation. + """ + if self._cached_vis is None: + self._cached_vis = _visibility(self) + return self._cached_vis + + @property + def name_and_loc(self): + """ + See the class documentation. + """ + # Reuse the expression format, which is '<choice (name, if any)>'. + return standard_sc_expr_str(self) + " " + _locs(self) + + @property + def selection(self): + """ + See the class documentation. + """ + if self._cached_selection is _NO_CACHED_SELECTION: + self._cached_selection = self._selection() + return self._cached_selection + + def set_value(self, value): + """ + Sets the user value (mode) of the choice. Like for Symbol.set_value(), + the visibility might truncate the value. Choices without the 'optional' + attribute (is_optional) can never be in n mode, but 0/"n" is still + accepted since it's not a malformed value (though it will have no + effect). + + Returns True if the value is valid for the type of the choice, and + False otherwise. This only looks at the form of the value. Check the + Choice.assignable attribute to see what values are currently in range + and would actually be reflected in the mode of the choice. + """ + if value in STR_TO_TRI: + value = STR_TO_TRI[value] + + if value == self.user_value: + # We know the value must be valid if it was successfully set + # previously + self._was_set = True + return True + + if not (self.orig_type is BOOL and value in (2, 0) or + self.orig_type is TRISTATE and value in TRI_TO_STR): + + # Display tristate values as n, m, y in the warning + self.kconfig._warn( + "the value {} is invalid for {}, which has type {} -- " + "assignment ignored" + .format(TRI_TO_STR[value] if value in TRI_TO_STR else + "'{}'".format(value), + self.name_and_loc, TYPE_TO_STR[self.orig_type])) + + return False + + self.user_value = value + self._was_set = True + self._rec_invalidate() + + return True + + def unset_value(self): + """ + Resets the user value (mode) and user selection of the Choice, as if + the user had never touched the mode or any of the choice symbols. + """ + if self.user_value is not None or self.user_selection: + self.user_value = self.user_selection = None + self._rec_invalidate() + + @property + def referenced(self): + """ + See the class documentation. + """ + return {item for node in self.nodes for item in node.referenced} + + @property + def orig_defaults(self): + """ + See the class documentation. + """ + return [d for node in self.nodes for d in node.orig_defaults] + + def __repr__(self): + """ + Returns a string with information about the choice when it is evaluated + on e.g. the interactive Python prompt. + """ + fields = ["choice " + self.name if self.name else "choice", + TYPE_TO_STR[self.type]] + add = fields.append + + for node in self.nodes: + if node.prompt: + add('"{}"'.format(node.prompt[0])) + + add("mode " + self.str_value) + + if self.user_value is not None: + add('user mode {}'.format(TRI_TO_STR[self.user_value])) + + if self.selection: + add("{} selected".format(self.selection.name)) + + if self.user_selection: + user_sel_str = "{} selected by user" \ + .format(self.user_selection.name) + + if self.selection is not self.user_selection: + user_sel_str += " (overridden)" + + add(user_sel_str) + + add("visibility " + TRI_TO_STR[self.visibility]) + + if self.is_optional: + add("optional") + + for node in self.nodes: + add("{}:{}".format(node.filename, node.linenr)) + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the choice when it is printed. + Matches the Kconfig format (though without the contained choice + symbols), with any parent dependencies propagated to the 'depends on' + condition. + + The returned string does not end in a newline. + + See Symbol.__str__() as well. + """ + return self.custom_str(standard_sc_expr_str) + + def custom_str(self, sc_expr_str_fn): + """ + Works like Choice.__str__(), but allows a custom format to be used for + all symbol/choice references. See expr_str(). + """ + return "\n\n".join(node.custom_str(sc_expr_str_fn) + for node in self.nodes) + + # + # Private methods + # + + def __init__(self): + """ + Choice constructor -- not intended to be called directly by Kconfiglib + clients. + """ + # These attributes are always set on the instance from outside and + # don't need defaults: + # direct_dep + # kconfig + + # - UNKNOWN == 0 + # - _visited is used during dep. loop detection + self.orig_type = self._visited = 0 + + self.nodes = [] + + self.syms = [] + self.defaults = [] + + self.name = \ + self.user_value = self.user_selection = \ + self._cached_vis = self._cached_assignable = None + + self._cached_selection = _NO_CACHED_SELECTION + + # is_constant is checked by _depend_on(). Just set it to avoid having + # to special-case choices. + self.is_constant = self.is_optional = False + + # See Kconfig._build_dep() + self._dependents = set() + + def _assignable(self): + # Worker function for the 'assignable' attribute + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + + if not vis: + return () + + if vis == 2: + if not self.is_optional: + return (2,) if self.type is BOOL else (1, 2) + return (0, 2) if self.type is BOOL else (0, 1, 2) + + # vis == 1 + + return (0, 1) if self.is_optional else (1,) + + def _selection(self): + # Worker function for the 'selection' attribute + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + if self.tri_value != 2: + # Not in y mode, so no selection + return None + + # Use the user selection if it's visible + if self.user_selection and self.user_selection.visibility: + return self.user_selection + + # Otherwise, check if we have a default + return self._selection_from_defaults() + + def _selection_from_defaults(self): + # Check if we have a default + for sym, cond in self.defaults: + # The default symbol must be visible too + if expr_value(cond) and sym.visibility: + return sym + + # Otherwise, pick the first visible symbol, if any + for sym in self.syms: + if sym.visibility: + return sym + + # Couldn't find a selection + return None + + def _invalidate(self): + self._cached_vis = self._cached_assignable = None + self._cached_selection = _NO_CACHED_SELECTION + + def _rec_invalidate(self): + # See Symbol._rec_invalidate() + + self._invalidate() + + for item in self._dependents: + if item._cached_vis is not None: + item._rec_invalidate() + + +class MenuNode(object): + """ + Represents a menu node in the configuration. This corresponds to an entry + in e.g. the 'make menuconfig' interface, though non-visible choices, menus, + and comments also get menu nodes. If a symbol or choice is defined in + multiple locations, it gets one menu node for each location. + + The top-level menu node, corresponding to the implicit top-level menu, is + available in Kconfig.top_node. + + The menu nodes for a Symbol or Choice can be found in the + Symbol/Choice.nodes attribute. Menus and comments are represented as plain + menu nodes, with their text stored in the prompt attribute (prompt[0]). + This mirrors the C implementation. + + The following attributes are available on MenuNode instances. They should + be viewed as read-only. + + item: + Either a Symbol, a Choice, or one of the constants MENU and COMMENT. + Menus and comments are represented as plain menu nodes. Ifs are collapsed + (matching the C implementation) and do not appear in the final menu tree. + + next: + The following menu node. None if there is no following node. + + list: + The first child menu node. None if there are no children. + + Choices and menus naturally have children, but Symbols can also have + children because of menus created automatically from dependencies (see + kconfig-language.txt). + + parent: + The parent menu node. None if there is no parent. + + prompt: + A (string, cond) tuple with the prompt for the menu node and its + conditional expression (which is self.kconfig.y if there is no + condition). None if there is no prompt. + + For symbols and choices, the prompt is stored in the MenuNode rather than + the Symbol or Choice instance. For menus and comments, the prompt holds + the text. + + defaults: + The 'default' properties for this particular menu node. See + symbol.defaults. + + When evaluating defaults, you should use Symbol/Choice.defaults instead, + as it include properties from all menu nodes (a symbol/choice can have + multiple definition locations/menu nodes). MenuNode.defaults is meant for + documentation generation. + + selects: + Like MenuNode.defaults, for selects. + + implies: + Like MenuNode.defaults, for implies. + + ranges: + Like MenuNode.defaults, for ranges. + + orig_prompt: + orig_defaults: + orig_selects: + orig_implies: + orig_ranges: + These work the like the corresponding attributes without orig_*, but omit + any dependencies propagated from 'depends on' and surrounding 'if's (the + direct dependencies, stored in MenuNode.dep). + + One use for this is generating less cluttered documentation, by only + showing the direct dependencies in one place. + + help: + The help text for the menu node for Symbols and Choices. None if there is + no help text. Always stored in the node rather than the Symbol or Choice. + It is possible to have a separate help text at each location if a symbol + is defined in multiple locations. + + Trailing whitespace (including a final newline) is stripped from the help + text. This was not the case before Kconfiglib 10.21.0, where the format + was undocumented. + + dep: + The direct ('depends on') dependencies for the menu node, or + self.kconfig.y if there are no direct dependencies. + + This attribute includes any dependencies from surrounding menus and ifs. + Those get propagated to the direct dependencies, and the resulting direct + dependencies in turn get propagated to the conditions of all properties. + + If a symbol or choice is defined in multiple locations, only the + properties defined at a particular location get the corresponding + MenuNode.dep dependencies propagated to them. + + visibility: + The 'visible if' dependencies for the menu node (which must represent a + menu), or self.kconfig.y if there are no 'visible if' dependencies. + 'visible if' dependencies are recursively propagated to the prompts of + symbols and choices within the menu. + + referenced: + A set() with all symbols and choices referenced in the properties and + property conditions of the menu node. + + Also includes dependencies inherited from surrounding menus and ifs. + Choices appear in the dependencies of choice symbols. + + is_menuconfig: + Set to True if the children of the menu node should be displayed in a + separate menu. This is the case for the following items: + + - Menus (node.item == MENU) + + - Choices + + - Symbols defined with the 'menuconfig' keyword. The children come from + implicitly created submenus, and should be displayed in a separate + menu rather than being indented. + + 'is_menuconfig' is just a hint on how to display the menu node. It's + ignored internally by Kconfiglib, except when printing symbols. + + filename/linenr: + The location where the menu node appears. The filename is relative to + $srctree (or to the current directory if $srctree isn't set), except + absolute paths are used for paths outside $srctree. + + include_path: + A tuple of (filename, linenr) tuples, giving the locations of the + 'source' statements via which the Kconfig file containing this menu node + was included. The first element is the location of the 'source' statement + in the top-level Kconfig file passed to Kconfig.__init__(), etc. + + Note that the Kconfig file of the menu node itself isn't included. Check + 'filename' and 'linenr' for that. + + kconfig: + The Kconfig instance the menu node is from. + """ + __slots__ = ( + "dep", + "filename", + "help", + "include_path", + "is_menuconfig", + "item", + "kconfig", + "linenr", + "list", + "next", + "parent", + "prompt", + "visibility", + + # Properties + "defaults", + "selects", + "implies", + "ranges", + ) + + def __init__(self): + # Properties defined on this particular menu node. A local 'depends on' + # only applies to these, in case a symbol is defined in multiple + # locations. + self.defaults = [] + self.selects = [] + self.implies = [] + self.ranges = [] + + @property + def orig_prompt(self): + """ + See the class documentation. + """ + if not self.prompt: + return None + return (self.prompt[0], self._strip_dep(self.prompt[1])) + + @property + def orig_defaults(self): + """ + See the class documentation. + """ + return [(default, self._strip_dep(cond)) + for default, cond in self.defaults] + + @property + def orig_selects(self): + """ + See the class documentation. + """ + return [(select, self._strip_dep(cond)) + for select, cond in self.selects] + + @property + def orig_implies(self): + """ + See the class documentation. + """ + return [(imply, self._strip_dep(cond)) + for imply, cond in self.implies] + + @property + def orig_ranges(self): + """ + See the class documentation. + """ + return [(low, high, self._strip_dep(cond)) + for low, high, cond in self.ranges] + + @property + def referenced(self): + """ + See the class documentation. + """ + # self.dep is included to catch dependencies from a lone 'depends on' + # when there are no properties to propagate it to + res = expr_items(self.dep) + + if self.prompt: + res |= expr_items(self.prompt[1]) + + if self.item is MENU: + res |= expr_items(self.visibility) + + for value, cond in self.defaults: + res |= expr_items(value) + res |= expr_items(cond) + + for value, cond in self.selects: + res.add(value) + res |= expr_items(cond) + + for value, cond in self.implies: + res.add(value) + res |= expr_items(cond) + + for low, high, cond in self.ranges: + res.add(low) + res.add(high) + res |= expr_items(cond) + + return res + + def __repr__(self): + """ + Returns a string with information about the menu node when it is + evaluated on e.g. the interactive Python prompt. + """ + fields = [] + add = fields.append + + if self.item.__class__ is Symbol: + add("menu node for symbol " + self.item.name) + + elif self.item.__class__ is Choice: + s = "menu node for choice" + if self.item.name is not None: + s += " " + self.item.name + add(s) + + elif self.item is MENU: + add("menu node for menu") + + else: # self.item is COMMENT + add("menu node for comment") + + if self.prompt: + add('prompt "{}" (visibility {})'.format( + self.prompt[0], TRI_TO_STR[expr_value(self.prompt[1])])) + + if self.item.__class__ is Symbol and self.is_menuconfig: + add("is menuconfig") + + add("deps " + TRI_TO_STR[expr_value(self.dep)]) + + if self.item is MENU: + add("'visible if' deps " + TRI_TO_STR[expr_value(self.visibility)]) + + if self.item.__class__ in _SYMBOL_CHOICE and self.help is not None: + add("has help") + + if self.list: + add("has child") + + if self.next: + add("has next") + + add("{}:{}".format(self.filename, self.linenr)) + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the menu node. Matches the Kconfig + format, with any parent dependencies propagated to the 'depends on' + condition. + + The output could (almost) be fed back into a Kconfig parser to redefine + the object associated with the menu node. See the module documentation + for a gotcha related to choice symbols. + + For symbols and choices with multiple menu nodes (multiple definition + locations), properties that aren't associated with a particular menu + node are shown on all menu nodes ('option env=...', 'optional' for + choices, etc.). + + The returned string does not end in a newline. + """ + return self.custom_str(standard_sc_expr_str) + + def custom_str(self, sc_expr_str_fn): + """ + Works like MenuNode.__str__(), but allows a custom format to be used + for all symbol/choice references. See expr_str(). + """ + return self._menu_comment_node_str(sc_expr_str_fn) \ + if self.item in _MENU_COMMENT else \ + self._sym_choice_node_str(sc_expr_str_fn) + + def _menu_comment_node_str(self, sc_expr_str_fn): + s = '{} "{}"'.format("menu" if self.item is MENU else "comment", + self.prompt[0]) + + if self.dep is not self.kconfig.y: + s += "\n\tdepends on {}".format(expr_str(self.dep, sc_expr_str_fn)) + + if self.item is MENU and self.visibility is not self.kconfig.y: + s += "\n\tvisible if {}".format(expr_str(self.visibility, + sc_expr_str_fn)) + + return s + + def _sym_choice_node_str(self, sc_expr_str_fn): + def indent_add(s): + lines.append("\t" + s) + + def indent_add_cond(s, cond): + if cond is not self.kconfig.y: + s += " if " + expr_str(cond, sc_expr_str_fn) + indent_add(s) + + sc = self.item + + if sc.__class__ is Symbol: + lines = [("menuconfig " if self.is_menuconfig else "config ") + + sc.name] + else: + lines = ["choice " + sc.name if sc.name else "choice"] + + if sc.orig_type and not self.prompt: # sc.orig_type != UNKNOWN + # If there's a prompt, we'll use the '<type> "prompt"' shorthand + # instead + indent_add(TYPE_TO_STR[sc.orig_type]) + + if self.prompt: + if sc.orig_type: + prefix = TYPE_TO_STR[sc.orig_type] + else: + # Symbol defined without a type (which generates a warning) + prefix = "prompt" + + indent_add_cond(prefix + ' "{}"'.format(escape(self.prompt[0])), + self.orig_prompt[1]) + + if sc.__class__ is Symbol: + if sc.is_allnoconfig_y: + indent_add("option allnoconfig_y") + + if sc is sc.kconfig.defconfig_list: + indent_add("option defconfig_list") + + if sc.env_var is not None: + indent_add('option env="{}"'.format(sc.env_var)) + + if sc is sc.kconfig.modules: + indent_add("option modules") + + for low, high, cond in self.orig_ranges: + indent_add_cond( + "range {} {}".format(sc_expr_str_fn(low), + sc_expr_str_fn(high)), + cond) + + for default, cond in self.orig_defaults: + indent_add_cond("default " + expr_str(default, sc_expr_str_fn), + cond) + + if sc.__class__ is Choice and sc.is_optional: + indent_add("optional") + + if sc.__class__ is Symbol: + for select, cond in self.orig_selects: + indent_add_cond("select " + sc_expr_str_fn(select), cond) + + for imply, cond in self.orig_implies: + indent_add_cond("imply " + sc_expr_str_fn(imply), cond) + + if self.dep is not sc.kconfig.y: + indent_add("depends on " + expr_str(self.dep, sc_expr_str_fn)) + + if self.help is not None: + indent_add("help") + for line in self.help.splitlines(): + indent_add(" " + line) + + return "\n".join(lines) + + def _strip_dep(self, expr): + # Helper function for removing MenuNode.dep from 'expr'. Uses two + # pieces of internal knowledge: (1) Expressions are reused rather than + # copied, and (2) the direct dependencies always appear at the end. + + # ... if dep -> ... if y + if self.dep is expr: + return self.kconfig.y + + # (AND, X, dep) -> X + if expr.__class__ is tuple and expr[0] is AND and expr[2] is self.dep: + return expr[1] + + return expr + + +class Variable(object): + """ + Represents a preprocessor variable/function. + + The following attributes are available: + + name: + The name of the variable. + + value: + The unexpanded value of the variable. + + expanded_value: + The expanded value of the variable. For simple variables (those defined + with :=), this will equal 'value'. Accessing this property will raise a + KconfigError if the expansion seems to be stuck in a loop. + + Accessing this field is the same as calling expanded_value_w_args() with + no arguments. I hadn't considered function arguments when adding it. It + is retained for backwards compatibility though. + + is_recursive: + True if the variable is recursive (defined with =). + """ + __slots__ = ( + "_n_expansions", + "is_recursive", + "kconfig", + "name", + "value", + ) + + @property + def expanded_value(self): + """ + See the class documentation. + """ + return self.expanded_value_w_args() + + def expanded_value_w_args(self, *args): + """ + Returns the expanded value of the variable/function. Any arguments + passed will be substituted for $(1), $(2), etc. + + Raises a KconfigError if the expansion seems to be stuck in a loop. + """ + return self.kconfig._fn_val((self.name,) + args) + + def __repr__(self): + return "<variable {}, {}, value '{}'>" \ + .format(self.name, + "recursive" if self.is_recursive else "immediate", + self.value) + + +class KconfigError(Exception): + """ + Exception raised for Kconfig-related errors. + + KconfigError and KconfigSyntaxError are the same class. The + KconfigSyntaxError alias is only maintained for backwards compatibility. + """ + +KconfigSyntaxError = KconfigError # Backwards compatibility + + +class InternalError(Exception): + "Never raised. Kept around for backwards compatibility." + + +# Workaround: +# +# If 'errno' and 'strerror' are set on IOError, then __str__() always returns +# "[Errno <errno>] <strerror>", ignoring any custom message passed to the +# constructor. By defining our own subclass, we can use a custom message while +# also providing 'errno', 'strerror', and 'filename' to scripts. +class _KconfigIOError(IOError): + def __init__(self, ioerror, msg): + self.msg = msg + super(_KconfigIOError, self).__init__( + ioerror.errno, ioerror.strerror, ioerror.filename) + + def __str__(self): + return self.msg + + +# +# Public functions +# + + +def expr_value(expr): + """ + Evaluates the expression 'expr' to a tristate value. Returns 0 (n), 1 (m), + or 2 (y). + + 'expr' must be an already-parsed expression from a Symbol, Choice, or + MenuNode property. To evaluate an expression represented as a string, use + Kconfig.eval_string(). + + Passing subexpressions of expressions to this function works as expected. + """ + if expr.__class__ is not tuple: + return expr.tri_value + + if expr[0] is AND: + v1 = expr_value(expr[1]) + # Short-circuit the n case as an optimization (~5% faster + # allnoconfig.py and allyesconfig.py, as of writing) + return 0 if not v1 else min(v1, expr_value(expr[2])) + + if expr[0] is OR: + v1 = expr_value(expr[1]) + # Short-circuit the y case as an optimization + return 2 if v1 == 2 else max(v1, expr_value(expr[2])) + + if expr[0] is NOT: + return 2 - expr_value(expr[1]) + + # Relation + # + # Implements <, <=, >, >= comparisons as well. These were added to + # kconfig in 31847b67 (kconfig: allow use of relations other than + # (in)equality). + + rel, v1, v2 = expr + + # If both operands are strings... + if v1.orig_type is STRING and v2.orig_type is STRING: + # ...then compare them lexicographically + comp = _strcmp(v1.str_value, v2.str_value) + else: + # Otherwise, try to compare them as numbers + try: + comp = _sym_to_num(v1) - _sym_to_num(v2) + except ValueError: + # Fall back on a lexicographic comparison if the operands don't + # parse as numbers + comp = _strcmp(v1.str_value, v2.str_value) + + return 2*(comp == 0 if rel is EQUAL else + comp != 0 if rel is UNEQUAL else + comp < 0 if rel is LESS else + comp <= 0 if rel is LESS_EQUAL else + comp > 0 if rel is GREATER else + comp >= 0) + + +def standard_sc_expr_str(sc): + """ + Standard symbol/choice printing function. Uses plain Kconfig syntax, and + displays choices as <choice> (or <choice NAME>, for named choices). + + See expr_str(). + """ + if sc.__class__ is Symbol: + if sc.is_constant and sc.name not in STR_TO_TRI: + return '"{}"'.format(escape(sc.name)) + return sc.name + + return "<choice {}>".format(sc.name) if sc.name else "<choice>" + + +def expr_str(expr, sc_expr_str_fn=standard_sc_expr_str): + """ + Returns the string representation of the expression 'expr', as in a Kconfig + file. + + Passing subexpressions of expressions to this function works as expected. + + sc_expr_str_fn (default: standard_sc_expr_str): + This function is called for every symbol/choice (hence "sc") appearing in + the expression, with the symbol/choice as the argument. It is expected to + return a string to be used for the symbol/choice. + + This can be used e.g. to turn symbols/choices into links when generating + documentation, or for printing the value of each symbol/choice after it. + + Note that quoted values are represented as constants symbols + (Symbol.is_constant == True). + """ + if expr.__class__ is not tuple: + return sc_expr_str_fn(expr) + + if expr[0] is AND: + return "{} && {}".format(_parenthesize(expr[1], OR, sc_expr_str_fn), + _parenthesize(expr[2], OR, sc_expr_str_fn)) + + if expr[0] is OR: + # This turns A && B || C && D into "(A && B) || (C && D)", which is + # redundant, but more readable + return "{} || {}".format(_parenthesize(expr[1], AND, sc_expr_str_fn), + _parenthesize(expr[2], AND, sc_expr_str_fn)) + + if expr[0] is NOT: + if expr[1].__class__ is tuple: + return "!({})".format(expr_str(expr[1], sc_expr_str_fn)) + return "!" + sc_expr_str_fn(expr[1]) # Symbol + + # Relation + # + # Relation operands are always symbols (quoted strings are constant + # symbols) + return "{} {} {}".format(sc_expr_str_fn(expr[1]), REL_TO_STR[expr[0]], + sc_expr_str_fn(expr[2])) + + +def expr_items(expr): + """ + Returns a set() of all items (symbols and choices) that appear in the + expression 'expr'. + + Passing subexpressions of expressions to this function works as expected. + """ + res = set() + + def rec(subexpr): + if subexpr.__class__ is tuple: + # AND, OR, NOT, or relation + + rec(subexpr[1]) + + # NOTs only have a single operand + if subexpr[0] is not NOT: + rec(subexpr[2]) + + else: + # Symbol or choice + res.add(subexpr) + + rec(expr) + return res + + +def split_expr(expr, op): + """ + Returns a list containing the top-level AND or OR operands in the + expression 'expr', in the same (left-to-right) order as they appear in + the expression. + + This can be handy e.g. for splitting (weak) reverse dependencies + from 'select' and 'imply' into individual selects/implies. + + op: + Either AND to get AND operands, or OR to get OR operands. + + (Having this as an operand might be more future-safe than having two + hardcoded functions.) + + + Pseudo-code examples: + + split_expr( A , OR ) -> [A] + split_expr( A && B , OR ) -> [A && B] + split_expr( A || B , OR ) -> [A, B] + split_expr( A || B , AND ) -> [A || B] + split_expr( A || B || (C && D) , OR ) -> [A, B, C && D] + + # Second || is not at the top level + split_expr( A || (B && (C || D)) , OR ) -> [A, B && (C || D)] + + # Parentheses don't matter as long as we stay at the top level (don't + # encounter any non-'op' nodes) + split_expr( (A || B) || C , OR ) -> [A, B, C] + split_expr( A || (B || C) , OR ) -> [A, B, C] + """ + res = [] + + def rec(subexpr): + if subexpr.__class__ is tuple and subexpr[0] is op: + rec(subexpr[1]) + rec(subexpr[2]) + else: + res.append(subexpr) + + rec(expr) + return res + + +def escape(s): + r""" + Escapes the string 's' in the same fashion as is done for display in + Kconfig format and when writing strings to a .config file. " and \ are + replaced by \" and \\, respectively. + """ + # \ must be escaped before " to avoid double escaping + return s.replace("\\", r"\\").replace('"', r'\"') + + +def unescape(s): + r""" + Unescapes the string 's'. \ followed by any character is replaced with just + that character. Used internally when reading .config files. + """ + return _unescape_sub(r"\1", s) + +# unescape() helper +_unescape_sub = re.compile(r"\\(.)").sub + + +def standard_kconfig(description=None): + """ + Argument parsing helper for tools that take a single optional Kconfig file + argument (default: Kconfig). Returns the Kconfig instance for the parsed + configuration. Uses argparse internally. + + Exits with sys.exit() (which raises SystemExit) on errors. + + description (default: None): + The 'description' passed to argparse.ArgumentParser(). + argparse.RawDescriptionHelpFormatter is used, so formatting is preserved. + """ + import argparse + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=description) + + parser.add_argument( + "kconfig", + metavar="KCONFIG", + default="Kconfig", + nargs="?", + help="Top-level Kconfig file (default: Kconfig)") + + return Kconfig(parser.parse_args().kconfig, suppress_traceback=True) + + +def standard_config_filename(): + """ + Helper for tools. Returns the value of KCONFIG_CONFIG (which specifies the + .config file to load/save) if it is set, and ".config" otherwise. + + Calling load_config() with filename=None might give the behavior you want, + without having to use this function. + """ + return os.getenv("KCONFIG_CONFIG", ".config") + + +def load_allconfig(kconf, filename): + """ + Use Kconfig.load_allconfig() instead, which was added in Kconfiglib 13.4.0. + Supported for backwards compatibility. Might be removed at some point after + a long period of deprecation warnings. + """ + allconfig = os.getenv("KCONFIG_ALLCONFIG") + if allconfig is None: + return + + def std_msg(e): + # "Upcasts" a _KconfigIOError to an IOError, removing the custom + # __str__() message. The standard message is better here. + # + # This might also convert an OSError to an IOError in obscure cases, + # but it's probably not a big deal. The distinction is shaky (see + # PEP-3151). + return IOError(e.errno, e.strerror, e.filename) + + old_warn_assign_override = kconf.warn_assign_override + old_warn_assign_redun = kconf.warn_assign_redun + kconf.warn_assign_override = kconf.warn_assign_redun = False + + if allconfig in ("", "1"): + try: + print(kconf.load_config(filename, False)) + except EnvironmentError as e1: + try: + print(kconf.load_config("all.config", False)) + except EnvironmentError as e2: + sys.exit("error: KCONFIG_ALLCONFIG is set, but neither {} " + "nor all.config could be opened: {}, {}" + .format(filename, std_msg(e1), std_msg(e2))) + else: + try: + print(kconf.load_config(allconfig, False)) + except EnvironmentError as e: + sys.exit("error: KCONFIG_ALLCONFIG is set to '{}', which " + "could not be opened: {}" + .format(allconfig, std_msg(e))) + + kconf.warn_assign_override = old_warn_assign_override + kconf.warn_assign_redun = old_warn_assign_redun + + +# +# Internal functions +# + + +def _visibility(sc): + # Symbols and Choices have a "visibility" that acts as an upper bound on + # the values a user can set for them, corresponding to the visibility in + # e.g. 'make menuconfig'. This function calculates the visibility for the + # Symbol or Choice 'sc' -- the logic is nearly identical. + + vis = 0 + + for node in sc.nodes: + if node.prompt: + vis = max(vis, expr_value(node.prompt[1])) + + if sc.__class__ is Symbol and sc.choice: + if sc.choice.orig_type is TRISTATE and \ + sc.orig_type is not TRISTATE and sc.choice.tri_value != 2: + # Non-tristate choice symbols are only visible in y mode + return 0 + + if sc.orig_type is TRISTATE and vis == 1 and sc.choice.tri_value == 2: + # Choice symbols with m visibility are not visible in y mode + return 0 + + # Promote m to y if we're dealing with a non-tristate (possibly due to + # modules being disabled) + if vis == 1 and sc.type is not TRISTATE: + return 2 + + return vis + + +def _depend_on(sc, expr): + # Adds 'sc' (symbol or choice) as a "dependee" to all symbols in 'expr'. + # Constant symbols in 'expr' are skipped as they can never change value + # anyway. + + if expr.__class__ is tuple: + # AND, OR, NOT, or relation + + _depend_on(sc, expr[1]) + + # NOTs only have a single operand + if expr[0] is not NOT: + _depend_on(sc, expr[2]) + + elif not expr.is_constant: + # Non-constant symbol, or choice + expr._dependents.add(sc) + + +def _parenthesize(expr, type_, sc_expr_str_fn): + # expr_str() helper. Adds parentheses around expressions of type 'type_'. + + if expr.__class__ is tuple and expr[0] is type_: + return "({})".format(expr_str(expr, sc_expr_str_fn)) + return expr_str(expr, sc_expr_str_fn) + + +def _ordered_unique(lst): + # Returns 'lst' with any duplicates removed, preserving order. This hacky + # version seems to be a common idiom. It relies on short-circuit evaluation + # and set.add() returning None, which is falsy. + + seen = set() + seen_add = seen.add + return [x for x in lst if x not in seen and not seen_add(x)] + + +def _is_base_n(s, n): + try: + int(s, n) + return True + except ValueError: + return False + + +def _strcmp(s1, s2): + # strcmp()-alike that returns -1, 0, or 1 + + return (s1 > s2) - (s1 < s2) + + +def _sym_to_num(sym): + # expr_value() helper for converting a symbol to a number. Raises + # ValueError for symbols that can't be converted. + + # For BOOL and TRISTATE, n/m/y count as 0/1/2. This mirrors 9059a3493ef + # ("kconfig: fix relational operators for bool and tristate symbols") in + # the C implementation. + return sym.tri_value if sym.orig_type in _BOOL_TRISTATE else \ + int(sym.str_value, _TYPE_TO_BASE[sym.orig_type]) + + +def _touch_dep_file(path, sym_name): + # If sym_name is MY_SYM_NAME, touches my/sym/name.h. See the sync_deps() + # docstring. + + sym_path = path + os.sep + sym_name.lower().replace("_", os.sep) + ".h" + sym_path_dir = dirname(sym_path) + if not exists(sym_path_dir): + os.makedirs(sym_path_dir, 0o755) + + # A kind of truncating touch, mirroring the C tools + os.close(os.open( + sym_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644)) + + +def _save_old(path): + # See write_config() + + def copy(src, dst): + # Import as needed, to save some startup time + import shutil + shutil.copyfile(src, dst) + + if islink(path): + # Preserve symlinks + copy_fn = copy + elif hasattr(os, "replace"): + # Python 3 (3.3+) only. Best choice when available, because it + # removes <filename>.old on both *nix and Windows. + copy_fn = os.replace + elif os.name == "posix": + # Removes <filename>.old on POSIX systems + copy_fn = os.rename + else: + # Fall back on copying + copy_fn = copy + + try: + copy_fn(path, path + ".old") + except Exception: + # Ignore errors from 'path' missing as well as other errors. + # <filename>.old file is usually more of a nice-to-have, and not worth + # erroring out over e.g. if <filename>.old happens to be a directory or + # <filename> is something like /dev/null. + pass + + +def _locs(sc): + # Symbol/Choice.name_and_loc helper. Returns the "(defined at ...)" part of + # the string. 'sc' is a Symbol or Choice. + + if sc.nodes: + return "(defined at {})".format( + ", ".join("{0.filename}:{0.linenr}".format(node) + for node in sc.nodes)) + + return "(undefined)" + + +# Menu manipulation + + +def _expr_depends_on(expr, sym): + # Reimplementation of expr_depends_symbol() from mconf.c. Used to determine + # if a submenu should be implicitly created. This also influences which + # items inside choice statements are considered choice items. + + if expr.__class__ is not tuple: + return expr is sym + + if expr[0] in _EQUAL_UNEQUAL: + # Check for one of the following: + # sym = m/y, m/y = sym, sym != n, n != sym + + left, right = expr[1:] + + if right is sym: + left, right = right, left + elif left is not sym: + return False + + return (expr[0] is EQUAL and right is sym.kconfig.m or + right is sym.kconfig.y) or \ + (expr[0] is UNEQUAL and right is sym.kconfig.n) + + return expr[0] is AND and \ + (_expr_depends_on(expr[1], sym) or + _expr_depends_on(expr[2], sym)) + + +def _auto_menu_dep(node1, node2): + # Returns True if node2 has an "automatic menu dependency" on node1. If + # node2 has a prompt, we check its condition. Otherwise, we look directly + # at node2.dep. + + return _expr_depends_on(node2.prompt[1] if node2.prompt else node2.dep, + node1.item) + + +def _flatten(node): + # "Flattens" menu nodes without prompts (e.g. 'if' nodes and non-visible + # symbols with children from automatic menu creation) so that their + # children appear after them instead. This gives a clean menu structure + # with no unexpected "jumps" in the indentation. + # + # Do not flatten promptless choices (which can appear "legitimately" if a + # named choice is defined in multiple locations to add on symbols). It + # looks confusing, and the menuconfig already shows all choice symbols if + # you enter the choice at some location with a prompt. + + while node: + if node.list and not node.prompt and \ + node.item.__class__ is not Choice: + + last_node = node.list + while 1: + last_node.parent = node.parent + if not last_node.next: + break + last_node = last_node.next + + last_node.next = node.next + node.next = node.list + node.list = None + + node = node.next + + +def _remove_ifs(node): + # Removes 'if' nodes (which can be recognized by MenuNode.item being None), + # which are assumed to already have been flattened. The C implementation + # doesn't bother to do this, but we expose the menu tree directly, and it + # makes it nicer to work with. + + cur = node.list + while cur and not cur.item: + cur = cur.next + + node.list = cur + + while cur: + next = cur.next + while next and not next.item: + next = next.next + + # Equivalent to + # + # cur.next = next + # cur = next + # + # due to tricky Python semantics. The order matters. + cur.next = cur = next + + +def _finalize_choice(node): + # Finalizes a choice, marking each symbol whose menu node has the choice as + # the parent as a choice symbol, and automatically determining types if not + # specified. + + choice = node.item + + cur = node.list + while cur: + if cur.item.__class__ is Symbol: + cur.item.choice = choice + choice.syms.append(cur.item) + cur = cur.next + + # If no type is specified for the choice, its type is that of + # the first choice item with a specified type + if not choice.orig_type: + for item in choice.syms: + if item.orig_type: + choice.orig_type = item.orig_type + break + + # Each choice item of UNKNOWN type gets the type of the choice + for sym in choice.syms: + if not sym.orig_type: + sym.orig_type = choice.orig_type + + +def _check_dep_loop_sym(sym, ignore_choice): + # Detects dependency loops using depth-first search on the dependency graph + # (which is calculated earlier in Kconfig._build_dep()). + # + # Algorithm: + # + # 1. Symbols/choices start out with _visited = 0, meaning unvisited. + # + # 2. When a symbol/choice is first visited, _visited is set to 1, meaning + # "visited, potentially part of a dependency loop". The recursive + # search then continues from the symbol/choice. + # + # 3. If we run into a symbol/choice X with _visited already set to 1, + # there's a dependency loop. The loop is found on the call stack by + # recording symbols while returning ("on the way back") until X is seen + # again. + # + # 4. Once a symbol/choice and all its dependencies (or dependents in this + # case) have been checked recursively without detecting any loops, its + # _visited is set to 2, meaning "visited, not part of a dependency + # loop". + # + # This saves work if we run into the symbol/choice again in later calls + # to _check_dep_loop_sym(). We just return immediately. + # + # Choices complicate things, as every choice symbol depends on every other + # choice symbol in a sense. When a choice is "entered" via a choice symbol + # X, we visit all choice symbols from the choice except X, and prevent + # immediately revisiting the choice with a flag (ignore_choice). + # + # Maybe there's a better way to handle this (different flags or the + # like...) + + if not sym._visited: + # sym._visited == 0, unvisited + + sym._visited = 1 + + for dep in sym._dependents: + # Choices show up in Symbol._dependents when the choice has the + # symbol in a 'prompt' or 'default' condition (e.g. + # 'default ... if SYM'). + # + # Since we aren't entering the choice via a choice symbol, all + # choice symbols need to be checked, hence the None. + loop = _check_dep_loop_choice(dep, None) \ + if dep.__class__ is Choice \ + else _check_dep_loop_sym(dep, False) + + if loop: + # Dependency loop found + return _found_dep_loop(loop, sym) + + if sym.choice and not ignore_choice: + loop = _check_dep_loop_choice(sym.choice, sym) + if loop: + # Dependency loop found + return _found_dep_loop(loop, sym) + + # The symbol is not part of a dependency loop + sym._visited = 2 + + # No dependency loop found + return None + + if sym._visited == 2: + # The symbol was checked earlier and is already known to not be part of + # a dependency loop + return None + + # sym._visited == 1, found a dependency loop. Return the symbol as the + # first element in it. + return (sym,) + + +def _check_dep_loop_choice(choice, skip): + if not choice._visited: + # choice._visited == 0, unvisited + + choice._visited = 1 + + # Check for loops involving choice symbols. If we came here via a + # choice symbol, skip that one, as we'd get a false positive + # '<sym FOO> -> <choice> -> <sym FOO>' loop otherwise. + for sym in choice.syms: + if sym is not skip: + # Prevent the choice from being immediately re-entered via the + # "is a choice symbol" path by passing True + loop = _check_dep_loop_sym(sym, True) + if loop: + # Dependency loop found + return _found_dep_loop(loop, choice) + + # The choice is not part of a dependency loop + choice._visited = 2 + + # No dependency loop found + return None + + if choice._visited == 2: + # The choice was checked earlier and is already known to not be part of + # a dependency loop + return None + + # choice._visited == 1, found a dependency loop. Return the choice as the + # first element in it. + return (choice,) + + +def _found_dep_loop(loop, cur): + # Called "on the way back" when we know we have a loop + + # Is the symbol/choice 'cur' where the loop started? + if cur is not loop[0]: + # Nope, it's just a part of the loop + return loop + (cur,) + + # Yep, we have the entire loop. Throw an exception that shows it. + + msg = "\nDependency loop\n" \ + "===============\n\n" + + for item in loop: + if item is not loop[0]: + msg += "...depends on " + if item.__class__ is Symbol and item.choice: + msg += "the choice symbol " + + msg += "{}, with definition...\n\n{}\n\n" \ + .format(item.name_and_loc, item) + + # Small wart: Since we reuse the already calculated + # Symbol/Choice._dependents sets for recursive dependency detection, we + # lose information on whether a dependency came from a 'select'/'imply' + # condition or e.g. a 'depends on'. + # + # This might cause selecting symbols to "disappear". For example, + # a symbol B having 'select A if C' gives a direct dependency from A to + # C, since it corresponds to a reverse dependency of B && C. + # + # Always print reverse dependencies for symbols that have them to make + # sure information isn't lost. I wonder if there's some neat way to + # improve this. + + if item.__class__ is Symbol: + if item.rev_dep is not item.kconfig.n: + msg += "(select-related dependencies: {})\n\n" \ + .format(expr_str(item.rev_dep)) + + if item.weak_rev_dep is not item.kconfig.n: + msg += "(imply-related dependencies: {})\n\n" \ + .format(expr_str(item.rev_dep)) + + msg += "...depends again on " + loop[0].name_and_loc + + raise KconfigError(msg) + + +def _decoding_error(e, filename, macro_linenr=None): + # Gives the filename and context for UnicodeDecodeError's, which are a pain + # to debug otherwise. 'e' is the UnicodeDecodeError object. + # + # If the decoding error is for the output of a $(shell,...) command, + # macro_linenr holds the line number where it was run (the exact line + # number isn't available for decoding errors in files). + + raise KconfigError( + "\n" + "Malformed {} in {}\n" + "Context: {}\n" + "Problematic data: {}\n" + "Reason: {}".format( + e.encoding, + "'{}'".format(filename) if macro_linenr is None else + "output from macro at {}:{}".format(filename, macro_linenr), + e.object[max(e.start - 40, 0):e.end + 40], + e.object[e.start:e.end], + e.reason)) + + +def _warn_verbose_deprecated(fn_name): + sys.stderr.write( + "Deprecation warning: {0}()'s 'verbose' argument has no effect. Since " + "Kconfiglib 12.0.0, the message is returned from {0}() instead, " + "and is always generated. Do e.g. print(kconf.{0}()) if you want to " + "want to show a message like \"Loaded configuration '.config'\" on " + "stdout. The old API required ugly hacks to reuse messages in " + "configuration interfaces.\n".format(fn_name)) + + +# Predefined preprocessor functions + + +def _filename_fn(kconf, _): + return kconf.filename + + +def _lineno_fn(kconf, _): + return str(kconf.linenr) + + +def _info_fn(kconf, _, msg): + print("{}:{}: {}".format(kconf.filename, kconf.linenr, msg)) + + return "" + + +def _warning_if_fn(kconf, _, cond, msg): + if cond == "y": + kconf._warn(msg, kconf.filename, kconf.linenr) + + return "" + + +def _error_if_fn(kconf, _, cond, msg): + if cond == "y": + raise KconfigError("{}:{}: {}".format( + kconf.filename, kconf.linenr, msg)) + + return "" + + +def _shell_fn(kconf, _, command): + import subprocess # Only import as needed, to save some startup time + + stdout, stderr = subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ).communicate() + + if not _IS_PY2: + try: + stdout = stdout.decode(kconf._encoding) + stderr = stderr.decode(kconf._encoding) + except UnicodeDecodeError as e: + _decoding_error(e, kconf.filename, kconf.linenr) + + if stderr: + kconf._warn("'{}' wrote to stderr: {}".format( + command, "\n".join(stderr.splitlines())), + kconf.filename, kconf.linenr) + + # Universal newlines with splitlines() (to prevent e.g. stray \r's in + # command output on Windows), trailing newline removal, and + # newline-to-space conversion. + # + # On Python 3 versions before 3.6, it's not possible to specify the + # encoding when passing universal_newlines=True to Popen() (the 'encoding' + # parameter was added in 3.6), so we do this manual version instead. + return "\n".join(stdout.splitlines()).rstrip("\n").replace("\n", " ") + +# +# Global constants +# + +TRI_TO_STR = { + 0: "n", + 1: "m", + 2: "y", +} + +STR_TO_TRI = { + "n": 0, + "m": 1, + "y": 2, +} + +# Constant representing that there's no cached choice selection. This is +# distinct from a cached None (no selection). Any object that's not None or a +# Symbol will do. We test this with 'is'. +_NO_CACHED_SELECTION = 0 + +# Are we running on Python 2? +_IS_PY2 = sys.version_info[0] < 3 + +try: + _UNAME_RELEASE = os.uname()[2] +except AttributeError: + # Only import as needed, to save some startup time + import platform + _UNAME_RELEASE = platform.uname()[2] + +# The token and type constants below are safe to test with 'is', which is a bit +# faster (~30% faster on my machine, and a few % faster for total parsing +# time), even without assuming Python's small integer optimization (which +# caches small integer objects). The constants end up pointing to unique +# integer objects, and since we consistently refer to them via the names below, +# we always get the same object. +# +# Client code should use == though. + +# Tokens, with values 1, 2, ... . Avoiding 0 simplifies some checks by making +# all tokens except empty strings truthy. +( + _T_ALLNOCONFIG_Y, + _T_AND, + _T_BOOL, + _T_CHOICE, + _T_CLOSE_PAREN, + _T_COMMENT, + _T_CONFIG, + _T_DEFAULT, + _T_DEFCONFIG_LIST, + _T_DEF_BOOL, + _T_DEF_HEX, + _T_DEF_INT, + _T_DEF_STRING, + _T_DEF_TRISTATE, + _T_DEPENDS, + _T_ENDCHOICE, + _T_ENDIF, + _T_ENDMENU, + _T_ENV, + _T_EQUAL, + _T_GREATER, + _T_GREATER_EQUAL, + _T_HELP, + _T_HEX, + _T_IF, + _T_IMPLY, + _T_INT, + _T_LESS, + _T_LESS_EQUAL, + _T_MAINMENU, + _T_MENU, + _T_MENUCONFIG, + _T_MODULES, + _T_NOT, + _T_ON, + _T_OPEN_PAREN, + _T_OPTION, + _T_OPTIONAL, + _T_OR, + _T_ORSOURCE, + _T_OSOURCE, + _T_PROMPT, + _T_RANGE, + _T_RSOURCE, + _T_SELECT, + _T_SOURCE, + _T_STRING, + _T_TRISTATE, + _T_UNEQUAL, + _T_VISIBLE, +) = range(1, 51) + +# Keyword to token map, with the get() method assigned directly as a small +# optimization +_get_keyword = { + "---help---": _T_HELP, + "allnoconfig_y": _T_ALLNOCONFIG_Y, + "bool": _T_BOOL, + "boolean": _T_BOOL, + "choice": _T_CHOICE, + "comment": _T_COMMENT, + "config": _T_CONFIG, + "def_bool": _T_DEF_BOOL, + "def_hex": _T_DEF_HEX, + "def_int": _T_DEF_INT, + "def_string": _T_DEF_STRING, + "def_tristate": _T_DEF_TRISTATE, + "default": _T_DEFAULT, + "defconfig_list": _T_DEFCONFIG_LIST, + "depends": _T_DEPENDS, + "endchoice": _T_ENDCHOICE, + "endif": _T_ENDIF, + "endmenu": _T_ENDMENU, + "env": _T_ENV, + "grsource": _T_ORSOURCE, # Backwards compatibility + "gsource": _T_OSOURCE, # Backwards compatibility + "help": _T_HELP, + "hex": _T_HEX, + "if": _T_IF, + "imply": _T_IMPLY, + "int": _T_INT, + "mainmenu": _T_MAINMENU, + "menu": _T_MENU, + "menuconfig": _T_MENUCONFIG, + "modules": _T_MODULES, + "on": _T_ON, + "option": _T_OPTION, + "optional": _T_OPTIONAL, + "orsource": _T_ORSOURCE, + "osource": _T_OSOURCE, + "prompt": _T_PROMPT, + "range": _T_RANGE, + "rsource": _T_RSOURCE, + "select": _T_SELECT, + "source": _T_SOURCE, + "string": _T_STRING, + "tristate": _T_TRISTATE, + "visible": _T_VISIBLE, +}.get + +# The constants below match the value of the corresponding tokens to remove the +# need for conversion + +# Node types +MENU = _T_MENU +COMMENT = _T_COMMENT + +# Expression types +AND = _T_AND +OR = _T_OR +NOT = _T_NOT +EQUAL = _T_EQUAL +UNEQUAL = _T_UNEQUAL +LESS = _T_LESS +LESS_EQUAL = _T_LESS_EQUAL +GREATER = _T_GREATER +GREATER_EQUAL = _T_GREATER_EQUAL + +REL_TO_STR = { + EQUAL: "=", + UNEQUAL: "!=", + LESS: "<", + LESS_EQUAL: "<=", + GREATER: ">", + GREATER_EQUAL: ">=", +} + +# Symbol/choice types. UNKNOWN is 0 (falsy) to simplify some checks. +# Client code shouldn't rely on it though, as it was non-zero in +# older versions. +UNKNOWN = 0 +BOOL = _T_BOOL +TRISTATE = _T_TRISTATE +STRING = _T_STRING +INT = _T_INT +HEX = _T_HEX + +TYPE_TO_STR = { + UNKNOWN: "unknown", + BOOL: "bool", + TRISTATE: "tristate", + STRING: "string", + INT: "int", + HEX: "hex", +} + +# Used in comparisons. 0 means the base is inferred from the format of the +# string. +_TYPE_TO_BASE = { + HEX: 16, + INT: 10, + STRING: 0, + UNKNOWN: 0, +} + +# def_bool -> BOOL, etc. +_DEF_TOKEN_TO_TYPE = { + _T_DEF_BOOL: BOOL, + _T_DEF_HEX: HEX, + _T_DEF_INT: INT, + _T_DEF_STRING: STRING, + _T_DEF_TRISTATE: TRISTATE, +} + +# Tokens after which strings are expected. This is used to tell strings from +# constant symbol references during tokenization, both of which are enclosed in +# quotes. +# +# Identifier-like lexemes ("missing quotes") are also treated as strings after +# these tokens. _T_CHOICE is included to avoid symbols being registered for +# named choices. +_STRING_LEX = frozenset({ + _T_BOOL, + _T_CHOICE, + _T_COMMENT, + _T_HEX, + _T_INT, + _T_MAINMENU, + _T_MENU, + _T_ORSOURCE, + _T_OSOURCE, + _T_PROMPT, + _T_RSOURCE, + _T_SOURCE, + _T_STRING, + _T_TRISTATE, +}) + +# Various sets for quick membership tests. Gives a single global lookup and +# avoids creating temporary dicts/tuples. + +_TYPE_TOKENS = frozenset({ + _T_BOOL, + _T_TRISTATE, + _T_INT, + _T_HEX, + _T_STRING, +}) + +_SOURCE_TOKENS = frozenset({ + _T_SOURCE, + _T_RSOURCE, + _T_OSOURCE, + _T_ORSOURCE, +}) + +_REL_SOURCE_TOKENS = frozenset({ + _T_RSOURCE, + _T_ORSOURCE, +}) + +# Obligatory (non-optional) sources +_OBL_SOURCE_TOKENS = frozenset({ + _T_SOURCE, + _T_RSOURCE, +}) + +_BOOL_TRISTATE = frozenset({ + BOOL, + TRISTATE, +}) + +_BOOL_TRISTATE_UNKNOWN = frozenset({ + BOOL, + TRISTATE, + UNKNOWN, +}) + +_INT_HEX = frozenset({ + INT, + HEX, +}) + +_SYMBOL_CHOICE = frozenset({ + Symbol, + Choice, +}) + +_MENU_COMMENT = frozenset({ + MENU, + COMMENT, +}) + +_EQUAL_UNEQUAL = frozenset({ + EQUAL, + UNEQUAL, +}) + +_RELATIONS = frozenset({ + EQUAL, + UNEQUAL, + LESS, + LESS_EQUAL, + GREATER, + GREATER_EQUAL, +}) + +# Helper functions for getting compiled regular expressions, with the needed +# matching function returned directly as a small optimization. +# +# Use ASCII regex matching on Python 3. It's already the default on Python 2. + + +def _re_match(regex): + return re.compile(regex, 0 if _IS_PY2 else re.ASCII).match + + +def _re_search(regex): + return re.compile(regex, 0 if _IS_PY2 else re.ASCII).search + + +# Various regular expressions used during parsing + +# The initial token on a line. Also eats leading and trailing whitespace, so +# that we can jump straight to the next token (or to the end of the line if +# there is only one token). +# +# This regex will also fail to match for empty lines and comment lines. +# +# '$' is included to detect preprocessor variable assignments with macro +# expansions in the left-hand side. +_command_match = _re_match(r"\s*([A-Za-z0-9_$-]+)\s*") + +# An identifier/keyword after the first token. Also eats trailing whitespace. +# '$' is included to detect identifiers containing macro expansions. +_id_keyword_match = _re_match(r"([A-Za-z0-9_$/.-]+)\s*") + +# A fragment in the left-hand side of a preprocessor variable assignment. These +# are the portions between macro expansions ($(foo)). Macros are supported in +# the LHS (variable name). +_assignment_lhs_fragment_match = _re_match("[A-Za-z0-9_-]*") + +# The assignment operator and value (right-hand side) in a preprocessor +# variable assignment +_assignment_rhs_match = _re_match(r"\s*(=|:=|\+=)\s*(.*)") + +# Special characters/strings while expanding a macro ('(', ')', ',', and '$(') +_macro_special_search = _re_search(r"\(|\)|,|\$\(") + +# Special characters/strings while expanding a string (quotes, '\', and '$(') +_string_special_search = _re_search(r'"|\'|\\|\$\(') + +# Special characters/strings while expanding a symbol name. Also includes +# end-of-line, in case the macro is the last thing on the line. +_name_special_search = _re_search(r'[^A-Za-z0-9_$/.-]|\$\(|$') + +# A valid right-hand side for an assignment to a string symbol in a .config +# file, including escaped characters. Extracts the contents. +_conf_string_match = _re_match(r'"((?:[^\\"]|\\.)*)"') diff --git a/roms/u-boot/tools/buildman/main.py b/roms/u-boot/tools/buildman/main.py new file mode 100755 index 000000000..2b714739a --- /dev/null +++ b/roms/u-boot/tools/buildman/main.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2012 The Chromium OS Authors. +# + +"""See README for more information""" + +import doctest +import multiprocessing +import os +import re +import sys +import unittest + +# Bring in the patman libraries +our_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(1, os.path.join(our_path, '..')) + +# Our modules +from buildman import board +from buildman import bsettings +from buildman import builder +from buildman import cmdline +from buildman import control +from buildman import toolchain +from patman import patchstream +from patman import gitutil +from patman import terminal + +def RunTests(skip_net_tests): + import func_test + import test + import doctest + + result = unittest.TestResult() + for module in ['buildman.toolchain', 'patman.gitutil']: + suite = doctest.DocTestSuite(module) + suite.run(result) + + sys.argv = [sys.argv[0]] + if skip_net_tests: + test.use_network = False + for module in (test.TestBuild, func_test.TestFunctional): + suite = unittest.TestLoader().loadTestsFromTestCase(module) + suite.run(result) + + print(result) + for test, err in result.errors: + print(err) + for test, err in result.failures: + print(err) + + +options, args = cmdline.ParseArgs() + +# Run our meagre tests +if options.test: + RunTests(options.skip_net_tests) + +# Build selected commits for selected boards +else: + bsettings.Setup(options.config_file) + ret_code = control.DoBuildman(options, args) + sys.exit(ret_code) diff --git a/roms/u-boot/tools/buildman/test.py b/roms/u-boot/tools/buildman/test.py new file mode 100644 index 000000000..b9c65c0d3 --- /dev/null +++ b/roms/u-boot/tools/buildman/test.py @@ -0,0 +1,628 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. +# + +import os +import shutil +import sys +import tempfile +import time +import unittest + +from buildman import board +from buildman import bsettings +from buildman import builder +from buildman import control +from buildman import toolchain +from patman import commit +from patman import command +from patman import terminal +from patman import test_util +from patman import tools + +use_network = True + +settings_data = ''' +# Buildman settings file + +[toolchain] +main: /usr/sbin + +[toolchain-alias] +x86: i386 x86_64 +''' + +migration = '''===================== WARNING ====================== +This board does not use CONFIG_DM. CONFIG_DM will be +compulsory starting with the v2020.01 release. +Failure to update may result in board removal. +See doc/driver-model/migration.rst for more info. +==================================================== +''' + +errors = [ + '''main.c: In function 'main_loop': +main.c:260:6: warning: unused variable 'joe' [-Wunused-variable] +''', + '''main.c: In function 'main_loop2': +main.c:295:2: error: 'fred' undeclared (first use in this function) +main.c:295:2: note: each undeclared identifier is reported only once for each function it appears in +make[1]: *** [main.o] Error 1 +make: *** [common/libcommon.o] Error 2 +Make failed +''', + '''arch/arm/dts/socfpga_arria10_socdk_sdmmc.dtb: Warning \ +(avoid_unnecessary_addr_size): /clocks: unnecessary #address-cells/#size-cells \ +without "ranges" or child "reg" property +''', + '''powerpc-linux-ld: warning: dot moved backwards before `.bss' +powerpc-linux-ld: warning: dot moved backwards before `.bss' +powerpc-linux-ld: u-boot: section .text lma 0xfffc0000 overlaps previous sections +powerpc-linux-ld: u-boot: section .rodata lma 0xfffef3ec overlaps previous sections +powerpc-linux-ld: u-boot: section .reloc lma 0xffffa400 overlaps previous sections +powerpc-linux-ld: u-boot: section .data lma 0xffffcd38 overlaps previous sections +powerpc-linux-ld: u-boot: section .u_boot_cmd lma 0xffffeb40 overlaps previous sections +powerpc-linux-ld: u-boot: section .bootpg lma 0xfffff198 overlaps previous sections +''', + '''In file included from %(basedir)sarch/sandbox/cpu/cpu.c:9:0: +%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default] +%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition +%(basedir)sarch/sandbox/cpu/cpu.c: In function 'do_reset': +%(basedir)sarch/sandbox/cpu/cpu.c:27:1: error: unknown type name 'blah' +%(basedir)sarch/sandbox/cpu/cpu.c:28:12: error: expected declaration specifiers or '...' before numeric constant +make[2]: *** [arch/sandbox/cpu/cpu.o] Error 1 +make[1]: *** [arch/sandbox/cpu] Error 2 +make[1]: *** Waiting for unfinished jobs.... +In file included from %(basedir)scommon/board_f.c:55:0: +%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default] +%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition +make: *** [sub-make] Error 2 +''' +] + + +# hash, subject, return code, list of errors/warnings +commits = [ + ['1234', 'upstream/master, migration warning', 0, []], + ['5678', 'Second commit, a warning', 0, errors[0:1]], + ['9012', 'Third commit, error', 1, errors[0:2]], + ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]], + ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]], + ['abcd', 'Sixth commit, fixes all errors', 0, []], + ['ef01', 'Seventh commit, fix migration, check directory suppression', 1, + [errors[4]]], +] + +boards = [ + ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], + ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], + ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], + ['Active', 'powerpc', 'mpc83xx', '', 'Tester', 'PowerPC board 2', 'board3', ''], + ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''], +] + +BASE_DIR = 'base' + +OUTCOME_OK, OUTCOME_WARN, OUTCOME_ERR = range(3) + +class Options: + """Class that holds build options""" + pass + +class TestBuild(unittest.TestCase): + """Test buildman + + TODO: Write tests for the rest of the functionality + """ + def setUp(self): + # Set up commits to build + self.commits = [] + sequence = 0 + for commit_info in commits: + comm = commit.Commit(commit_info[0]) + comm.subject = commit_info[1] + comm.return_code = commit_info[2] + comm.error_list = commit_info[3] + if sequence < 6: + comm.error_list += [migration] + comm.sequence = sequence + sequence += 1 + self.commits.append(comm) + + # Set up boards to build + self.boards = board.Boards() + for brd in boards: + self.boards.AddBoard(board.Board(*brd)) + self.boards.SelectBoards([]) + + # Add some test settings + bsettings.Setup(None) + bsettings.AddFile(settings_data) + + # Set up the toolchains + self.toolchains = toolchain.Toolchains() + self.toolchains.Add('arm-linux-gcc', test=False) + self.toolchains.Add('sparc-linux-gcc', test=False) + self.toolchains.Add('powerpc-linux-gcc', test=False) + self.toolchains.Add('gcc', test=False) + + # Avoid sending any output + terminal.SetPrintTestMode() + self._col = terminal.Color() + + self.base_dir = tempfile.mkdtemp() + if not os.path.isdir(self.base_dir): + os.mkdir(self.base_dir) + + def tearDown(self): + shutil.rmtree(self.base_dir) + + def Make(self, commit, brd, stage, *args, **kwargs): + result = command.CommandResult() + boardnum = int(brd.target[-1]) + result.return_code = 0 + result.stderr = '' + result.stdout = ('This is the test output for board %s, commit %s' % + (brd.target, commit.hash)) + if ((boardnum >= 1 and boardnum >= commit.sequence) or + boardnum == 4 and commit.sequence == 6): + result.return_code = commit.return_code + result.stderr = (''.join(commit.error_list) + % {'basedir' : self.base_dir + '/.bm-work/00/'}) + elif commit.sequence < 6: + result.stderr = migration + + result.combined = result.stdout + result.stderr + return result + + def assertSummary(self, text, arch, plus, boards, outcome=OUTCOME_ERR): + col = self._col + expected_colour = (col.GREEN if outcome == OUTCOME_OK else + col.YELLOW if outcome == OUTCOME_WARN else col.RED) + expect = '%10s: ' % arch + # TODO(sjg@chromium.org): If plus is '', we shouldn't need this + expect += ' ' + col.Color(expected_colour, plus) + expect += ' ' + for board in boards: + expect += col.Color(expected_colour, ' %s' % board) + self.assertEqual(text, expect) + + def _SetupTest(self, echo_lines=False, threads=1, **kwdisplay_args): + """Set up the test by running a build and summary + + Args: + echo_lines: True to echo lines to the terminal to aid test + development + kwdisplay_args: Dict of arguemnts to pass to + Builder.SetDisplayOptions() + + Returns: + Iterator containing the output lines, each a PrintLine() object + """ + build = builder.Builder(self.toolchains, self.base_dir, None, threads, + 2, checkout=False, show_unknown=False) + build.do_make = self.Make + board_selected = self.boards.GetSelectedDict() + + # Build the boards for the pre-defined commits and warnings/errors + # associated with each. This calls our Make() to inject the fake output. + build.BuildBoards(self.commits, board_selected, keep_outputs=False, + verbose=False) + lines = terminal.GetPrintTestLines() + count = 0 + for line in lines: + if line.text.strip(): + count += 1 + + # We should get two starting messages, an update for every commit built + # and a summary message + self.assertEqual(count, len(commits) * len(boards) + 3) + build.SetDisplayOptions(**kwdisplay_args); + build.ShowSummary(self.commits, board_selected) + if echo_lines: + terminal.EchoPrintTestLines() + return iter(terminal.GetPrintTestLines()) + + def _CheckOutput(self, lines, list_error_boards=False, + filter_dtb_warnings=False, + filter_migration_warnings=False): + """Check for expected output from the build summary + + Args: + lines: Iterator containing the lines returned from the summary + list_error_boards: Adjust the check for output produced with the + --list-error-boards flag + filter_dtb_warnings: Adjust the check for output produced with the + --filter-dtb-warnings flag + """ + def add_line_prefix(prefix, boards, error_str, colour): + """Add a prefix to each line of a string + + The training \n in error_str is removed before processing + + Args: + prefix: String prefix to add + error_str: Error string containing the lines + colour: Expected colour for the line. Note that the board list, + if present, always appears in magenta + + Returns: + New string where each line has the prefix added + """ + lines = error_str.strip().splitlines() + new_lines = [] + for line in lines: + if boards: + expect = self._col.Color(colour, prefix + '(') + expect += self._col.Color(self._col.MAGENTA, boards, + bright=False) + expect += self._col.Color(colour, ') %s' % line) + else: + expect = self._col.Color(colour, prefix + line) + new_lines.append(expect) + return '\n'.join(new_lines) + + col = terminal.Color() + boards01234 = ('board0 board1 board2 board3 board4' + if list_error_boards else '') + boards1234 = 'board1 board2 board3 board4' if list_error_boards else '' + boards234 = 'board2 board3 board4' if list_error_boards else '' + boards34 = 'board3 board4' if list_error_boards else '' + boards4 = 'board4' if list_error_boards else '' + + # Upstream commit: migration warnings only + self.assertEqual(next(lines).text, '01: %s' % commits[0][1]) + + if not filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', 'w+', + ['board0', 'board1'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) + + self.assertEqual(next(lines).text, + add_line_prefix('+', boards01234, migration, col.RED)) + + # Second commit: all archs should fail with warnings + self.assertEqual(next(lines).text, '02: %s' % commits[1][1]) + + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', 'w+', + ['board1'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) + + # Second commit: The warnings should be listed + self.assertEqual(next(lines).text, + add_line_prefix('w+', boards1234, errors[0], col.YELLOW)) + + # Third commit: Still fails + self.assertEqual(next(lines).text, '03: %s' % commits[2][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', '', + ['board1'], outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'powerpc', '+', + ['board2', 'board3']) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) + + # Expect a compiler error + self.assertEqual(next(lines).text, + add_line_prefix('+', boards234, errors[1], col.RED)) + + # Fourth commit: Compile errors are fixed, just have warning for board3 + self.assertEqual(next(lines).text, '04: %s' % commits[3][1]) + if filter_migration_warnings: + expect = '%10s: ' % 'powerpc' + expect += ' ' + col.Color(col.GREEN, '') + expect += ' ' + expect += col.Color(col.GREEN, ' %s' % 'board2') + expect += ' ' + col.Color(col.YELLOW, 'w+') + expect += ' ' + expect += col.Color(col.YELLOW, ' %s' % 'board3') + self.assertEqual(next(lines).text, expect) + else: + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) + + # Compile error fixed + self.assertEqual(next(lines).text, + add_line_prefix('-', boards234, errors[1], col.GREEN)) + + if not filter_dtb_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('w+', boards34, errors[2], col.YELLOW)) + + # Fifth commit + self.assertEqual(next(lines).text, '05: %s' % commits[4][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'powerpc', '', ['board3'], + outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) + + # The second line of errors[3] is a duplicate, so buildman will drop it + expect = errors[3].rstrip().split('\n') + expect = [expect[0]] + expect[2:] + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('+', boards4, expect, col.RED)) + + if not filter_dtb_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('w-', boards34, errors[2], col.CYAN)) + + # Sixth commit + self.assertEqual(next(lines).text, '06: %s' % commits[5][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'sandbox', '', ['board4'], + outcome=OUTCOME_OK) + else: + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) + + # The second line of errors[3] is a duplicate, so buildman will drop it + expect = errors[3].rstrip().split('\n') + expect = [expect[0]] + expect[2:] + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('-', boards4, expect, col.GREEN)) + self.assertEqual(next(lines).text, + add_line_prefix('w-', boards4, errors[0], col.CYAN)) + + # Seventh commit + self.assertEqual(next(lines).text, '07: %s' % commits[6][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) + else: + self.assertSummary(next(lines).text, 'arm', '', ['board0', 'board1'], + outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'powerpc', '', + ['board2', 'board3'], outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) + + # Pick out the correct error lines + expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n') + expect = expect_str[3:8] + [expect_str[-1]] + expect = '\n'.join(expect) + if not filter_migration_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('-', boards01234, migration, col.GREEN)) + + self.assertEqual(next(lines).text, + add_line_prefix('+', boards4, expect, col.RED)) + + # Now the warnings lines + expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]] + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('w+', boards4, expect, col.YELLOW)) + + def testOutput(self): + """Test basic builder operation and output + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True) + self._CheckOutput(lines, list_error_boards=False, + filter_dtb_warnings=False) + + def testErrorBoards(self): + """Test output with --list-error-boards + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, list_error_boards=True) + self._CheckOutput(lines, list_error_boards=True) + + def testFilterDtb(self): + """Test output with --filter-dtb-warnings + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, filter_dtb_warnings=True) + self._CheckOutput(lines, filter_dtb_warnings=True) + + def testFilterMigration(self): + """Test output with --filter-migration-warnings + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, + filter_migration_warnings=True) + self._CheckOutput(lines, filter_migration_warnings=True) + + def testSingleThread(self): + """Test operation without threading""" + lines = self._SetupTest(show_errors=True, threads=0) + self._CheckOutput(lines, list_error_boards=False, + filter_dtb_warnings=False) + + def _testGit(self): + """Test basic builder operation by building a branch""" + options = Options() + options.git = os.getcwd() + options.summary = False + options.jobs = None + options.dry_run = False + #options.git = os.path.join(self.base_dir, 'repo') + options.branch = 'test-buildman' + options.force_build = False + options.list_tool_chains = False + options.count = -1 + options.git_dir = None + options.threads = None + options.show_unknown = False + options.quick = False + options.show_errors = False + options.keep_outputs = False + args = ['tegra20'] + control.DoBuildman(options, args) + + def testBoardSingle(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['sandbox']), + ({'all': ['board4'], 'sandbox': ['board4']}, [])) + + def testBoardArch(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['arm']), + ({'all': ['board0', 'board1'], + 'arm': ['board0', 'board1']}, [])) + + def testBoardArchSingle(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['arm sandbox']), + ({'sandbox': ['board4'], + 'all': ['board0', 'board1', 'board4'], + 'arm': ['board0', 'board1']}, [])) + + + def testBoardArchSingleMultiWord(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['arm', 'sandbox']), + ({'sandbox': ['board4'], + 'all': ['board0', 'board1', 'board4'], + 'arm': ['board0', 'board1']}, [])) + + def testBoardSingleAnd(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['Tester & arm']), + ({'Tester&arm': ['board0', 'board1'], + 'all': ['board0', 'board1']}, [])) + + def testBoardTwoAnd(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['Tester', '&', 'arm', + 'Tester' '&', 'powerpc', + 'sandbox']), + ({'sandbox': ['board4'], + 'all': ['board0', 'board1', 'board2', 'board3', + 'board4'], + 'Tester&powerpc': ['board2', 'board3'], + 'Tester&arm': ['board0', 'board1']}, [])) + + def testBoardAll(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards([]), + ({'all': ['board0', 'board1', 'board2', 'board3', + 'board4']}, [])) + + def testBoardRegularExpression(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['T.*r&^Po']), + ({'all': ['board2', 'board3'], + 'T.*r&^Po': ['board2', 'board3']}, [])) + + def testBoardDuplicate(self): + """Test single board selection""" + self.assertEqual(self.boards.SelectBoards(['sandbox sandbox', + 'sandbox']), + ({'all': ['board4'], 'sandbox': ['board4']}, [])) + def CheckDirs(self, build, dirname): + self.assertEqual('base%s' % dirname, build._GetOutputDir(1)) + self.assertEqual('base%s/fred' % dirname, + build.GetBuildDir(1, 'fred')) + self.assertEqual('base%s/fred/done' % dirname, + build.GetDoneFile(1, 'fred')) + self.assertEqual('base%s/fred/u-boot.sizes' % dirname, + build.GetFuncSizesFile(1, 'fred', 'u-boot')) + self.assertEqual('base%s/fred/u-boot.objdump' % dirname, + build.GetObjdumpFile(1, 'fred', 'u-boot')) + self.assertEqual('base%s/fred/err' % dirname, + build.GetErrFile(1, 'fred')) + + def testOutputDir(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False) + build.commits = self.commits + build.commit_count = len(self.commits) + subject = self.commits[1].subject.translate(builder.trans_valid_chars) + dirname ='/%02d_g%s_%s' % (2, commits[1][0], subject[:20]) + self.CheckDirs(build, dirname) + + def testOutputDirCurrent(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False) + build.commits = None + build.commit_count = 0 + self.CheckDirs(build, '/current') + + def testOutputDirNoSubdirs(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False, + no_subdirs=True) + build.commits = None + build.commit_count = 0 + self.CheckDirs(build, '') + + def testToolchainAliases(self): + self.assertTrue(self.toolchains.Select('arm') != None) + with self.assertRaises(ValueError): + self.toolchains.Select('no-arch') + with self.assertRaises(ValueError): + self.toolchains.Select('x86') + + self.toolchains = toolchain.Toolchains() + self.toolchains.Add('x86_64-linux-gcc', test=False) + self.assertTrue(self.toolchains.Select('x86') != None) + + self.toolchains = toolchain.Toolchains() + self.toolchains.Add('i386-linux-gcc', test=False) + self.assertTrue(self.toolchains.Select('x86') != None) + + def testToolchainDownload(self): + """Test that we can download toolchains""" + if use_network: + with test_util.capture_sys_output() as (stdout, stderr): + url = self.toolchains.LocateArchUrl('arm') + self.assertRegexpMatches(url, 'https://www.kernel.org/pub/tools/' + 'crosstool/files/bin/x86_64/.*/' + 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz') + + def testGetEnvArgs(self): + """Test the GetEnvArgs() function""" + tc = self.toolchains.Select('arm') + self.assertEqual('arm-linux-', + tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_PATH)) + self.assertEqual('arm', + tc.GetEnvArgs(toolchain.VAR_ARCH)) + self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + + self.toolchains.Add('/path/to/x86_64-linux-gcc', test=False) + tc = self.toolchains.Select('x86') + self.assertEqual('/path/to', + tc.GetEnvArgs(toolchain.VAR_PATH)) + tc.override_toolchain = 'clang' + self.assertEqual('HOSTCC=clang CC=clang', + tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + + def testPrepareOutputSpace(self): + def _Touch(fname): + tools.WriteFile(os.path.join(base_dir, fname), b'') + + base_dir = tempfile.mkdtemp() + + # Add various files that we want removed and left alone + to_remove = ['01_g0982734987_title', '102_g92bf_title', + '01_g2938abd8_title'] + to_leave = ['something_else', '01-something.patch', '01_another'] + for name in to_remove + to_leave: + _Touch(name) + + build = builder.Builder(self.toolchains, base_dir, None, 1, 2) + build.commits = self.commits + build.commit_count = len(commits) + result = set(build._GetOutputSpaceRemovals()) + expected = set([os.path.join(base_dir, f) for f in to_remove]) + self.assertEqual(expected, result) + +if __name__ == "__main__": + unittest.main() diff --git a/roms/u-boot/tools/buildman/toolchain.py b/roms/u-boot/tools/buildman/toolchain.py new file mode 100644 index 000000000..fd137f730 --- /dev/null +++ b/roms/u-boot/tools/buildman/toolchain.py @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. +# + +import re +import glob +from html.parser import HTMLParser +import os +import sys +import tempfile +import urllib.request, urllib.error, urllib.parse + +from buildman import bsettings +from patman import command +from patman import terminal +from patman import tools + +(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, + PRIORITY_CALC) = list(range(4)) + +(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) + +# Simple class to collect links from a page +class MyHTMLParser(HTMLParser): + def __init__(self, arch): + """Create a new parser + + After the parser runs, self.links will be set to a list of the links + to .xz archives found in the page, and self.arch_link will be set to + the one for the given architecture (or None if not found). + + Args: + arch: Architecture to search for + """ + HTMLParser.__init__(self) + self.arch_link = None + self.links = [] + self.re_arch = re.compile('[-_]%s-' % arch) + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for tag, value in attrs: + if tag == 'href': + if value and value.endswith('.xz'): + self.links.append(value) + if self.re_arch.search(value): + self.arch_link = value + + +class Toolchain: + """A single toolchain + + Public members: + gcc: Full path to C compiler + path: Directory path containing C compiler + cross: Cross compile string, e.g. 'arm-linux-' + arch: Architecture of toolchain as determined from the first + component of the filename. E.g. arm-linux-gcc becomes arm + priority: Toolchain priority (0=highest, 20=lowest) + override_toolchain: Toolchain to use for sandbox, overriding the normal + one + """ + def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, + arch=None, override_toolchain=None): + """Create a new toolchain object. + + Args: + fname: Filename of the gcc component + test: True to run the toolchain to test it + verbose: True to print out the information + priority: Priority to use for this toolchain, or PRIORITY_CALC to + calculate it + """ + self.gcc = fname + self.path = os.path.dirname(fname) + self.override_toolchain = override_toolchain + + # Find the CROSS_COMPILE prefix to use for U-Boot. For example, + # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. + basename = os.path.basename(fname) + pos = basename.rfind('-') + self.cross = basename[:pos + 1] if pos != -1 else '' + + # The architecture is the first part of the name + pos = self.cross.find('-') + if arch: + self.arch = arch + else: + self.arch = self.cross[:pos] if pos != -1 else 'sandbox' + if self.arch == 'sandbox' and override_toolchain: + self.gcc = override_toolchain + + env = self.MakeEnvironment(False) + + # As a basic sanity check, run the C compiler with --version + cmd = [fname, '--version'] + if priority == PRIORITY_CALC: + self.priority = self.GetPriority(fname) + else: + self.priority = priority + if test: + result = command.RunPipe([cmd], capture=True, env=env, + raise_on_error=False) + self.ok = result.return_code == 0 + if verbose: + print('Tool chain test: ', end=' ') + if self.ok: + print("OK, arch='%s', priority %d" % (self.arch, + self.priority)) + else: + print('BAD') + print('Command: ', cmd) + print(result.stdout) + print(result.stderr) + else: + self.ok = True + + def GetPriority(self, fname): + """Return the priority of the toolchain. + + Toolchains are ranked according to their suitability by their + filename prefix. + + Args: + fname: Filename of toolchain + Returns: + Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. + """ + priority_list = ['-elf', '-unknown-linux-gnu', '-linux', + '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', + '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', + '-linux-gnueabihf', '-le-linux', '-uclinux'] + for prio in range(len(priority_list)): + if priority_list[prio] in fname: + return PRIORITY_CALC + prio + return PRIORITY_CALC + prio + + def GetWrapper(self, show_warning=True): + """Get toolchain wrapper from the setting file. + """ + value = '' + for name, value in bsettings.GetItems('toolchain-wrapper'): + if not value: + print("Warning: Wrapper not found") + if value: + value = value + ' ' + + return value + + def GetEnvArgs(self, which): + """Get an environment variable/args value based on the the toolchain + + Args: + which: VAR_... value to get + + Returns: + Value of that environment variable or arguments + """ + wrapper = self.GetWrapper() + if which == VAR_CROSS_COMPILE: + return wrapper + os.path.join(self.path, self.cross) + elif which == VAR_PATH: + return self.path + elif which == VAR_ARCH: + return self.arch + elif which == VAR_MAKE_ARGS: + args = self.MakeArgs() + if args: + return ' '.join(args) + return '' + else: + raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) + + def MakeEnvironment(self, full_path): + """Returns an environment for using the toolchain. + + Thie takes the current environment and adds CROSS_COMPILE so that + the tool chain will operate correctly. This also disables localized + output and possibly unicode encoded output of all build tools by + adding LC_ALL=C. + + Note that os.environb is used to obtain the environment, since in some + cases the environment many contain non-ASCII characters and we see + errors like: + + UnicodeEncodeError: 'utf-8' codec can't encode characters in position + 569-570: surrogates not allowed + + Args: + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + Returns: + Dict containing the (bytes) environment to use. This is based on the + current environment, with changes as needed to CROSS_COMPILE, PATH + and LC_ALL. + """ + env = dict(os.environb) + wrapper = self.GetWrapper() + + if self.override_toolchain: + # We'll use MakeArgs() to provide this + pass + elif full_path: + env[b'CROSS_COMPILE'] = tools.ToBytes( + wrapper + os.path.join(self.path, self.cross)) + else: + env[b'CROSS_COMPILE'] = tools.ToBytes(wrapper + self.cross) + env[b'PATH'] = tools.ToBytes(self.path) + b':' + env[b'PATH'] + + env[b'LC_ALL'] = b'C' + + return env + + def MakeArgs(self): + """Create the 'make' arguments for a toolchain + + This is only used when the toolchain is being overridden. Since the + U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the + environment (and MakeEnvironment()) to override these values. This + function returns the arguments to accomplish this. + + Returns: + List of arguments to pass to 'make' + """ + if self.override_toolchain: + return ['HOSTCC=%s' % self.override_toolchain, + 'CC=%s' % self.override_toolchain] + return [] + + +class Toolchains: + """Manage a list of toolchains for building U-Boot + + We select one toolchain for each architecture type + + Public members: + toolchains: Dict of Toolchain objects, keyed by architecture name + prefixes: Dict of prefixes to check, keyed by architecture. This can + be a full path and toolchain prefix, for example + {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of + something on the search path, for example + {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. + paths: List of paths to check for toolchains (may contain wildcards) + """ + + def __init__(self, override_toolchain=None): + self.toolchains = {} + self.prefixes = {} + self.paths = [] + self.override_toolchain = override_toolchain + self._make_flags = dict(bsettings.GetItems('make-flags')) + + def GetPathList(self, show_warning=True): + """Get a list of available toolchain paths + + Args: + show_warning: True to show a warning if there are no tool chains. + + Returns: + List of strings, each a path to a toolchain mentioned in the + [toolchain] section of the settings file. + """ + toolchains = bsettings.GetItems('toolchain') + if show_warning and not toolchains: + print(("Warning: No tool chains. Please run 'buildman " + "--fetch-arch all' to download all available toolchains, or " + "add a [toolchain] section to your buildman config file " + "%s. See README for details" % + bsettings.config_fname)) + + paths = [] + for name, value in toolchains: + if '*' in value: + paths += glob.glob(value) + else: + paths.append(value) + return paths + + def GetSettings(self, show_warning=True): + """Get toolchain settings from the settings file. + + Args: + show_warning: True to show a warning if there are no tool chains. + """ + self.prefixes = bsettings.GetItems('toolchain-prefix') + self.paths += self.GetPathList(show_warning) + + def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, + arch=None): + """Add a toolchain to our list + + We select the given toolchain as our preferred one for its + architecture if it is a higher priority than the others. + + Args: + fname: Filename of toolchain's gcc driver + test: True to run the toolchain to test it + priority: Priority to use for this toolchain + arch: Toolchain architecture, or None if not known + """ + toolchain = Toolchain(fname, test, verbose, priority, arch, + self.override_toolchain) + add_it = toolchain.ok + if toolchain.arch in self.toolchains: + add_it = (toolchain.priority < + self.toolchains[toolchain.arch].priority) + if add_it: + self.toolchains[toolchain.arch] = toolchain + elif verbose: + print(("Toolchain '%s' at priority %d will be ignored because " + "another toolchain for arch '%s' has priority %d" % + (toolchain.gcc, toolchain.priority, toolchain.arch, + self.toolchains[toolchain.arch].priority))) + + def ScanPath(self, path, verbose): + """Scan a path for a valid toolchain + + Args: + path: Path to scan + verbose: True to print out progress information + Returns: + Filename of C compiler if found, else None + """ + fnames = [] + for subdir in ['.', 'bin', 'usr/bin']: + dirname = os.path.join(path, subdir) + if verbose: print(" - looking in '%s'" % dirname) + for fname in glob.glob(dirname + '/*gcc'): + if verbose: print(" - found '%s'" % fname) + fnames.append(fname) + return fnames + + def ScanPathEnv(self, fname): + """Scan the PATH environment variable for a given filename. + + Args: + fname: Filename to scan for + Returns: + List of matching pathanames, or [] if none + """ + pathname_list = [] + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + pathname = os.path.join(path, fname) + if os.path.exists(pathname): + pathname_list.append(pathname) + return pathname_list + + def Scan(self, verbose): + """Scan for available toolchains and select the best for each arch. + + We look for all the toolchains we can file, figure out the + architecture for each, and whether it works. Then we select the + highest priority toolchain for each arch. + + Args: + verbose: True to print out progress information + """ + if verbose: print('Scanning for tool chains') + for name, value in self.prefixes: + if verbose: print(" - scanning prefix '%s'" % value) + if os.path.exists(value): + self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) + continue + fname = value + 'gcc' + if os.path.exists(fname): + self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) + continue + fname_list = self.ScanPathEnv(fname) + for f in fname_list: + self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) + if not fname_list: + raise ValueError("No tool chain found for prefix '%s'" % + value) + for path in self.paths: + if verbose: print(" - scanning path '%s'" % path) + fnames = self.ScanPath(path, verbose) + for fname in fnames: + self.Add(fname, True, verbose) + + def List(self): + """List out the selected toolchains for each architecture""" + col = terminal.Color() + print(col.Color(col.BLUE, 'List of available toolchains (%d):' % + len(self.toolchains))) + if len(self.toolchains): + for key, value in sorted(self.toolchains.items()): + print('%-10s: %s' % (key, value.gcc)) + else: + print('None') + + def Select(self, arch): + """Returns the toolchain for a given architecture + + Args: + args: Name of architecture (e.g. 'arm', 'ppc_8xx') + + returns: + toolchain object, or None if none found + """ + for tag, value in bsettings.GetItems('toolchain-alias'): + if arch == tag: + for alias in value.split(): + if alias in self.toolchains: + return self.toolchains[alias] + + if not arch in self.toolchains: + raise ValueError("No tool chain found for arch '%s'" % arch) + return self.toolchains[arch] + + def ResolveReferences(self, var_dict, args): + """Resolve variable references in a string + + This converts ${blah} within the string to the value of blah. + This function works recursively. + + Args: + var_dict: Dictionary containing variables and their values + args: String containing make arguments + Returns: + Resolved string + + >>> bsettings.Setup() + >>> tcs = Toolchains() + >>> tcs.Add('fred', False) + >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ + 'second' : '2nd'} + >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') + 'this=OBLIQUE_set' + >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') + 'this=OBLIQUE_setfi2ndrstnd' + """ + re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') + + while True: + m = re_var.search(args) + if not m: + break + lookup = m.group(0)[2:-1] + value = var_dict.get(lookup, '') + args = args[:m.start(0)] + value + args[m.end(0):] + return args + + def GetMakeArguments(self, board): + """Returns 'make' arguments for a given board + + The flags are in a section called 'make-flags'. Flags are named + after the target they represent, for example snapper9260=TESTING=1 + will pass TESTING=1 to make when building the snapper9260 board. + + References to other boards can be added in the string also. For + example: + + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443 + + This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. + + A special 'target' variable is set to the board target. + + Args: + board: Board object for the board to check. + Returns: + 'make' flags for that board, or '' if none + """ + self._make_flags['target'] = board.target + arg_str = self.ResolveReferences(self._make_flags, + self._make_flags.get(board.target, '')) + args = re.findall("(?:\".*?\"|\S)+", arg_str) + i = 0 + while i < len(args): + args[i] = args[i].replace('"', '') + if not args[i]: + del args[i] + else: + i += 1 + return args + + def LocateArchUrl(self, fetch_arch): + """Find a toolchain available online + + Look in standard places for available toolchains. At present the + only standard place is at kernel.org. + + Args: + arch: Architecture to look for, or 'list' for all + Returns: + If fetch_arch is 'list', a tuple: + Machine architecture (e.g. x86_64) + List of toolchains + else + URL containing this toolchain, if avaialble, else None + """ + arch = command.OutputOneLine('uname', '-m') + if arch == 'aarch64': + arch = 'arm64' + base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' + versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4'] + links = [] + for version in versions: + url = '%s/%s/%s/' % (base, arch, version) + print('Checking: %s' % url) + response = urllib.request.urlopen(url) + html = tools.ToString(response.read()) + parser = MyHTMLParser(fetch_arch) + parser.feed(html) + if fetch_arch == 'list': + links += parser.links + elif parser.arch_link: + return url + parser.arch_link + if fetch_arch == 'list': + return arch, links + return None + + def Download(self, url): + """Download a file to a temporary directory + + Args: + url: URL to download + Returns: + Tuple: + Temporary directory name + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + """ + print('Downloading: %s' % url) + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp('.buildman') + response = urllib.request.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.get('Content-Length')) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print(chr(8) * (len(status) + 1), '\r', end=' ') + break + + done += len(buffer) + fd.write(buffer) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) + status = status + chr(8) * (len(status) + 1) + print(status, end=' ') + sys.stdout.flush() + fd.close() + if done != size: + print('Error, failed to download') + os.remove(fname) + fname = None + return tmpdir, fname + + def Unpack(self, fname, dest): + """Unpack a tar file + + Args: + fname: Filename to unpack + dest: Destination directory + Returns: + Directory name of the first entry in the archive, without the + trailing / + """ + stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) + dirs = stdout.splitlines()[1].split('/')[:2] + return '/'.join(dirs) + + def TestSettingsHasPath(self, path): + """Check if buildman will find this toolchain + + Returns: + True if the path is in settings, False if not + """ + paths = self.GetPathList(False) + return path in paths + + def ListArchs(self): + """List architectures with available toolchains to download""" + host_arch, archives = self.LocateArchUrl('list') + re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') + arch_set = set() + for archive in archives: + # Remove the host architecture from the start + arch = re_arch.match(archive[len(host_arch):]) + if arch: + if arch.group(1) != '2.0' and arch.group(1) != '64': + arch_set.add(arch.group(1)) + return sorted(arch_set) + + def FetchAndInstall(self, arch): + """Fetch and install a new toolchain + + arch: + Architecture to fetch, or 'list' to list + """ + # Fist get the URL for this architecture + col = terminal.Color() + print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) + url = self.LocateArchUrl(arch) + if not url: + print(("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch)) + return 2 + home = os.environ['HOME'] + dest = os.path.join(home, '.buildman-toolchains') + if not os.path.exists(dest): + os.mkdir(dest) + + # Download the tar file for this toolchain and unpack it + tmpdir, tarfile = self.Download(url) + if not tarfile: + return 1 + print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ') + sys.stdout.flush() + path = self.Unpack(tarfile, dest) + os.remove(tarfile) + os.rmdir(tmpdir) + print() + + # Check that the toolchain works + print(col.Color(col.GREEN, 'Testing')) + dirpath = os.path.join(dest, path) + compiler_fname_list = self.ScanPath(dirpath, True) + if not compiler_fname_list: + print('Could not locate C compiler - fetch failed.') + return 1 + if len(compiler_fname_list) != 1: + print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % + ', '.join(compiler_fname_list))) + toolchain = Toolchain(compiler_fname_list[0], True, True) + + # Make sure that it will be found by buildman + if not self.TestSettingsHasPath(dirpath): + print(("Adding 'download' to config file '%s'" % + bsettings.config_fname)) + bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) + return 0 |