diff options
Diffstat (limited to 'tests/docker')
61 files changed, 3684 insertions, 0 deletions
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include new file mode 100644 index 000000000..f1a0c5db7 --- /dev/null +++ b/tests/docker/Makefile.include @@ -0,0 +1,342 @@ +# Makefile for Docker tests + +.PHONY: docker docker-help docker-test docker-clean docker-image docker-qemu-src + +NULL := +SPACE := $(NULL) # +COMMA := , + +HOST_ARCH = $(if $(ARCH),$(ARCH),$(shell uname -m)) + +DOCKER_SUFFIX := .docker +DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles +# we don't run tests on intermediate images (used as base by another image) +DOCKER_PARTIAL_IMAGES := debian10 debian11 +# we don't directly build virtual images (they are used to build other images) +DOCKER_VIRTUAL_IMAGES := debian-bootstrap debian-toolchain empty +DOCKER_IMAGES := $(sort $(filter-out $(DOCKER_VIRTUAL_IMAGES), $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker))))) +DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES)) +# Use a global constant ccache directory to speed up repetitive builds +DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache +ifeq ($(HOST_ARCH),x86_64) +DOCKER_DEFAULT_REGISTRY := registry.gitlab.com/qemu-project/qemu +endif +DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),$(DOCKER_DEFAULT_REGISTRY)) + +DOCKER_TESTS := $(notdir $(shell \ + find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f)) + +ENGINE := auto + +DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py --engine $(ENGINE) + +TESTS ?= % +IMAGES ?= % + +CUR_TIME := $(shell date +%Y-%m-%d-%H.%M.%S.$$$$) +DOCKER_SRC_COPY := $(BUILD_DIR)/docker-src.$(CUR_TIME) + +.DELETE_ON_ERROR: $(DOCKER_SRC_COPY) +$(DOCKER_SRC_COPY): + @mkdir $@ + $(if $(SRC_ARCHIVE), \ + $(call quiet-command, cp "$(SRC_ARCHIVE)" $@/qemu.tar, \ + "CP", "$@/qemu.tar"), \ + $(call quiet-command, cd $(SRC_PATH) && scripts/archive-source.sh $@/qemu.tar, \ + "GEN", "$@/qemu.tar")) + $(call quiet-command, cp $(SRC_PATH)/tests/docker/run $@/run, \ + "COPY","RUNNER") + +docker-qemu-src: $(DOCKER_SRC_COPY) + +docker-image: ${DOCKER_TARGETS} + +# General rule for building docker images. If we are a sub-make +# invoked with SKIP_DOCKER_BUILD we still check the image is up to date +# though +ifdef SKIP_DOCKER_BUILD +docker-image-%: $(DOCKER_FILES_DIR)/%.docker + $(call quiet-command, \ + $(DOCKER_SCRIPT) check --quiet qemu/$* $<, \ + "CHECK", "$*") +else +docker-image-%: $(DOCKER_FILES_DIR)/%.docker + $(call quiet-command,\ + $(DOCKER_SCRIPT) build -t qemu/$* -f $< \ + $(if $V,,--quiet) \ + $(if $(NOCACHE),--no-cache, \ + $(if $(DOCKER_REGISTRY),--registry $(DOCKER_REGISTRY))) \ + $(if $(NOUSER),,--add-current-user) \ + $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\ + $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ + "BUILD","$*") + +# Special rule for debootstraped binfmt linux-user images +docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker + $(if $(EXECUTABLE),,\ + $(error EXECUTABLE not set, debootstrap of debian-$* would fail)) + $(if $(DEB_ARCH),,\ + $(error DEB_ARCH not set, debootstrap of debian-$* would fail)) + $(if $(DEB_TYPE),,\ + $(error DEB_TYPE not set, debootstrap of debian-$* would fail)) + $(if $(wildcard $(EXECUTABLE)), \ + $(call quiet-command, \ + DEB_ARCH=$(DEB_ARCH) \ + DEB_TYPE=$(DEB_TYPE) \ + $(if $(DEB_URL),DEB_URL=$(DEB_URL),) \ + $(DOCKER_SCRIPT) build -t qemu/debian-$* -f $< \ + $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ + $(if $(NOUSER),,--add-current-user) \ + $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES)) \ + $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)), \ + "BUILD","binfmt debian-$* (debootstrapped)"), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) check --quiet qemu/debian-$* $< || \ + { echo "You will need to build $(EXECUTABLE)"; exit 1;},\ + "CHECK", "debian-$* exists")) + +# These are test targets +USER_TCG_TARGETS=$(patsubst %-linux-user,qemu-%,$(filter %-linux-user,$(TARGET_DIRS))) +EXEC_COPY_TESTS=$(patsubst %,docker-exec-copy-test-%, $(USER_TCG_TARGETS)) + +$(EXEC_COPY_TESTS): docker-exec-copy-test-%: $(DOCKER_FILES_DIR)/empty.docker + $(call quiet-command, \ + $(DOCKER_SCRIPT) build -t qemu/exec-copy-test-$* -f $< \ + $(if $V,,--quiet) --no-cache \ + --include-executable=$* \ + --skip-binfmt, \ + "TEST","copy $* to container") + $(call quiet-command, \ + $(DOCKER_SCRIPT) run qemu/exec-copy-test-$* \ + /$* -version > tests/docker-exec-copy-test-$*.out, \ + "TEST","check $* works in container") + +docker-exec-copy-test: $(EXEC_COPY_TESTS) + +endif + +# Enforce dependencies for composite images +ifeq ($(HOST_ARCH),x86_64) +docker-image-debian-amd64: docker-image-debian10 +DOCKER_PARTIAL_IMAGES += debian-amd64-cross +else +docker-image-debian-amd64-cross: docker-image-debian10 +DOCKER_PARTIAL_IMAGES += debian-amd64 +endif + +# For non-x86 hosts not all cross-compilers have been packaged +ifneq ($(HOST_ARCH),x86_64) +DOCKER_PARTIAL_IMAGES += debian-mips-cross debian-mipsel-cross debian-mips64el-cross +DOCKER_PARTIAL_IMAGES += debian-ppc64el-cross +DOCKER_PARTIAL_IMAGES += debian-s390x-cross +DOCKER_PARTIAL_IMAGES += fedora +endif + +docker-image-debian-alpha-cross: docker-image-debian10 +docker-image-debian-arm64-cross: docker-image-debian10 +docker-image-debian-armel-cross: docker-image-debian10 +docker-image-debian-armhf-cross: docker-image-debian10 +docker-image-debian-hppa-cross: docker-image-debian10 +docker-image-debian-m68k-cross: docker-image-debian10 +docker-image-debian-mips-cross: docker-image-debian10 +docker-image-debian-mips64-cross: docker-image-debian10 +docker-image-debian-mips64el-cross: docker-image-debian10 +docker-image-debian-mipsel-cross: docker-image-debian10 +docker-image-debian-ppc64el-cross: docker-image-debian10 +docker-image-debian-s390x-cross: docker-image-debian10 +docker-image-debian-sh4-cross: docker-image-debian10 +docker-image-debian-sparc64-cross: docker-image-debian10 + +# The native build should never use the registry +docker-image-debian-native: DOCKER_REGISTRY= + +# base images should not add a local user +docker-image-debian10: NOUSER=1 +docker-image-debian11: NOUSER=1 + +# +# The build rule for hexagon-cross is special in so far for most of +# the time we don't want to build it. While dockers caching does avoid +# this most of the time sometimes we want to force the issue. +# +docker-image-debian-hexagon-cross: $(DOCKER_FILES_DIR)/debian-hexagon-cross.docker + $(if $(NOCACHE), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) build -t qemu/debian-hexagon-cross -f $< \ + $(if $V,,--quiet) --no-cache \ + --registry $(DOCKER_REGISTRY) --extra-files \ + $(DOCKER_FILES_DIR)/debian-hexagon-cross.docker.d/build-toolchain.sh, \ + "BUILD", "debian-hexagon-cross"), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) fetch $(if $V,,--quiet) \ + qemu/debian-hexagon-cross $(DOCKER_REGISTRY), \ + "FETCH", "debian-hexagon-cross") \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) update $(if $V,,--quiet) \ + qemu/debian-hexagon-cross --add-current-user, \ + "PREPARE", "debian-hexagon-cross")) + +debian-toolchain-run = \ + $(if $(NOCACHE), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) build -t qemu/$1 -f $< \ + $(if $V,,--quiet) --no-cache \ + --registry $(DOCKER_REGISTRY) --extra-files \ + $(DOCKER_FILES_DIR)/$1.d/build-toolchain.sh, \ + "BUILD", $1), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) fetch $(if $V,,--quiet) \ + qemu/$1 $(DOCKER_REGISTRY), \ + "FETCH", $1) \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) update $(if $V,,--quiet) \ + qemu/$1 \ + $(if $(NOUSER),,--add-current-user) \ + "PREPARE", $1)) +debian-toolchain = $(call debian-toolchain-run,$(patsubst docker-image-%,%,$1)) + +docker-image-debian-microblaze-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ + $(DOCKER_FILES_DIR)/debian-microblaze-cross.d/build-toolchain.sh + $(call debian-toolchain, $@) + +docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ + $(DOCKER_FILES_DIR)/debian-nios2-cross.d/build-toolchain.sh + $(call debian-toolchain, $@) + +# Specialist build images, sometimes very limited tools +docker-image-debian-tricore-cross: docker-image-debian10 +docker-image-debian-all-test-cross: docker-image-debian10 +docker-image-debian-arm64-test-cross: docker-image-debian11 +docker-image-debian-microblaze-cross: docker-image-debian10 +docker-image-debian-nios2-cross: docker-image-debian10 +docker-image-debian-powerpc-test-cross: docker-image-debian11 + +# These images may be good enough for building tests but not for test builds +DOCKER_PARTIAL_IMAGES += debian-alpha-cross +DOCKER_PARTIAL_IMAGES += debian-arm64-test-cross +DOCKER_PARTIAL_IMAGES += debian-powerpc-test-cross +DOCKER_PARTIAL_IMAGES += debian-hppa-cross +DOCKER_PARTIAL_IMAGES += debian-m68k-cross debian-mips64-cross +DOCKER_PARTIAL_IMAGES += debian-microblaze-cross +DOCKER_PARTIAL_IMAGES += debian-nios2-cross +DOCKER_PARTIAL_IMAGES += debian-sh4-cross debian-sparc64-cross +DOCKER_PARTIAL_IMAGES += debian-tricore-cross +DOCKER_PARTIAL_IMAGES += debian-xtensa-cross +DOCKER_PARTIAL_IMAGES += fedora-cris-cross + +# Rules for building linux-user powered images +# +# These are slower than using native cross compiler setups but can +# work around issues with poorly working multi-arch systems and broken +# packages. + +# Expand all the pre-requistes for each docker image and test combination +$(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES) $(DOCKER_VIRTUAL_IMAGES),$(DOCKER_IMAGES)), \ + $(foreach t,$(DOCKER_TESTS), \ + $(eval .PHONY: docker-$t@$i) \ + $(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \ + ) \ + $(foreach t,$(DOCKER_TESTS), \ + $(eval docker-all-tests: docker-$t@$i) \ + $(eval docker-$t: docker-$t@$i) \ + ) \ +) + +docker: + @echo 'Build QEMU and run tests inside Docker or Podman containers' + @echo + @echo 'Available targets:' + @echo + @echo ' docker: Print this help.' + @echo ' docker-all-tests: Run all image/test combinations.' + @echo ' docker-TEST: Run "TEST" on all image combinations.' + @echo ' docker-clean: Kill and remove residual docker testing containers.' + @echo ' docker-TEST@IMAGE: Run "TEST" in container "IMAGE".' + @echo ' Note: "TEST" is one of the listed test name,' + @echo ' or a script name under $$QEMU_SRC/tests/docker/;' + @echo ' "IMAGE" is one of the listed container name.' + @echo ' docker-image: Build all images.' + @echo ' docker-image-IMAGE: Build image "IMAGE".' + @echo ' docker-run: For manually running a "TEST" with "IMAGE".' + @echo + @echo 'Available container images:' + @echo ' $(DOCKER_IMAGES)' +ifneq ($(DOCKER_USER_IMAGES),) + @echo + @echo 'Available linux-user images (docker-binfmt-image-debian-%):' + @echo ' $(DOCKER_USER_IMAGES)' +endif + @echo + @echo 'Available tests:' + @echo ' $(DOCKER_TESTS)' + @echo + @echo 'Special variables:' + @echo ' TARGET_LIST=a,b,c Override target list in builds.' + @echo ' EXTRA_CONFIGURE_OPTS="..."' + @echo ' Extra configure options.' + @echo ' IMAGES="a b c ..": Filters which images to build or run.' + @echo ' TESTS="x y z .." Filters which tests to run (for docker-test).' + @echo ' J=[0..9]* Overrides the -jN parameter for make commands' + @echo ' (default is 1)' + @echo ' DEBUG=1 Stop and drop to shell in the created container' + @echo ' before running the command.' + @echo ' NETWORK=1 Enable virtual network interface with default backend.' + @echo ' NETWORK=$$BACKEND Enable virtual network interface with $$BACKEND.' + @echo ' NOUSER=1 Define to disable adding current user to containers passwd.' + @echo ' NOCACHE=1 Ignore cache when build images.' + @echo ' EXECUTABLE=<path> Include executable in image.' + @echo ' EXTRA_FILES="<path> [... <path>]"' + @echo ' Include extra files in image.' + @echo ' ENGINE=auto/docker/podman' + @echo ' Specify which container engine to run.' + @echo ' REGISTRY=url Cache builds from registry (default:$(DOCKER_REGISTRY))' + +docker-help: docker + +# This rule if for directly running against an arbitrary docker target. +# It is called by the expanded docker targets (e.g. make +# docker-test-foo@bar) which will do additional verification. +# +# For example: make docker-run TEST="test-quick" IMAGE="debian:arm64" EXECUTABLE=./aarch64-linux-user/qemu-aarch64 +# +docker-run: docker-qemu-src + @mkdir -p "$(DOCKER_CCACHE_DIR)" + @if test -z "$(IMAGE)" || test -z "$(TEST)"; \ + then echo "Invalid target $(IMAGE)/$(TEST)"; exit 1; \ + fi + $(if $(EXECUTABLE), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) update \ + $(IMAGE) --executable $(EXECUTABLE), \ + " COPYING $(EXECUTABLE) to $(IMAGE)")) + $(call quiet-command, \ + $(DOCKER_SCRIPT) run \ + $(if $(NOUSER),,--run-as-current-user) \ + --security-opt seccomp=unconfined \ + $(if $(DEBUG),-ti,) \ + $(if $(NETWORK),$(if $(subst $(NETWORK),,1),--net=$(NETWORK)),--net=none) \ + -e TARGET_LIST=$(subst $(SPACE),$(COMMA),$(TARGET_LIST)) \ + -e EXTRA_CONFIGURE_OPTS="$(EXTRA_CONFIGURE_OPTS)" \ + -e V=$V -e J=$J -e DEBUG=$(DEBUG) \ + -e SHOW_ENV=$(SHOW_ENV) \ + $(if $(NOUSER),, \ + -e CCACHE_DIR=/var/tmp/ccache \ + -v $(DOCKER_CCACHE_DIR):/var/tmp/ccache:z \ + ) \ + -v $$(readlink -e $(DOCKER_SRC_COPY)):/var/tmp/qemu:z$(COMMA)ro \ + $(IMAGE) \ + /var/tmp/qemu/run \ + $(TEST), " RUN $(TEST) in ${IMAGE}") + $(call quiet-command, rm -r $(DOCKER_SRC_COPY), \ + " CLEANUP $(DOCKER_SRC_COPY)") + +# Run targets: +# +# Of the form docker-TEST-FOO@IMAGE-BAR which will then be expanded into a call to "make docker-run" +docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/') +docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/') +docker-run-%: + @$(MAKE) docker-run TEST=$(CMD) IMAGE=qemu/$(IMAGE) + +docker-clean: + $(call quiet-command, $(DOCKER_SCRIPT) clean) diff --git a/tests/docker/common.rc b/tests/docker/common.rc new file mode 100755 index 000000000..e6f8cee0d --- /dev/null +++ b/tests/docker/common.rc @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Common routines for docker test scripts. +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +# This might be set by ENV of a docker container... it is always +# overriden by TARGET_LIST if the user sets it. We special case +# "none" to allow for other options like --disable-tcg to restrict the +# builds we eventually do. +if test "$DEF_TARGET_LIST" = "none"; then + DEF_TARGET_LIST="" +else + DEF_TARGET_LIST=${DEF_TARGET_LIST:-"x86_64-softmmu,aarch64-softmmu"} +fi + +requires_binary() +{ + found=0 + for c in $@; do + for d in /bin /usr/bin /usr/local/bin + do + if test -f "$d/$c" + then + found=1 + fi + done + done + if test "$found" != "1" + then + echo "Prerequisite '$c' not present, skip" + exit 0 + fi +} + +configure_qemu() +{ + config_opts="--enable-werror \ + ${TARGET_LIST:+--target-list=${TARGET_LIST}} \ + --prefix=$INSTALL_DIR \ + $QEMU_CONFIGURE_OPTS $EXTRA_CONFIGURE_OPTS \ + $@" + echo "Configure options:" + echo $config_opts + $QEMU_SRC/configure $config_opts || \ + { cat config.log && test_fail "Failed to run 'configure'"; } +} + +build_qemu() +{ + configure_qemu $@ + make $MAKEFLAGS +} + +check_qemu() +{ + # default to make check unless the caller specifies + if [ $# = 0 ]; then + INVOCATION="check" + else + INVOCATION="$@" + fi + + make $MAKEFLAGS $INVOCATION +} + +test_fail() +{ + echo "$@" + exit 1 +} + +prep_fail() +{ + echo "$@" + exit 2 +} + +install_qemu() +{ + make install $MAKEFLAGS DESTDIR=$PWD/=destdir + ret=$? + rm -rf $PWD/=destdir + return $ret +} diff --git a/tests/docker/docker.py b/tests/docker/docker.py new file mode 100755 index 000000000..78dd13171 --- /dev/null +++ b/tests/docker/docker.py @@ -0,0 +1,758 @@ +#!/usr/bin/env python3 +# +# Docker controlling module +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +import os +import sys +import subprocess +import json +import hashlib +import atexit +import uuid +import argparse +import enum +import tempfile +import re +import signal +from tarfile import TarFile, TarInfo +from io import StringIO, BytesIO +from shutil import copy, rmtree +from pwd import getpwuid +from datetime import datetime, timedelta + + +FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] + + +DEVNULL = open(os.devnull, 'wb') + +class EngineEnum(enum.IntEnum): + AUTO = 1 + DOCKER = 2 + PODMAN = 3 + + def __str__(self): + return self.name.lower() + + def __repr__(self): + return str(self) + + @staticmethod + def argparse(s): + try: + return EngineEnum[s.upper()] + except KeyError: + return s + + +USE_ENGINE = EngineEnum.AUTO + +def _bytes_checksum(bytes): + """Calculate a digest string unique to the text content""" + return hashlib.sha1(bytes).hexdigest() + +def _text_checksum(text): + """Calculate a digest string unique to the text content""" + return _bytes_checksum(text.encode('utf-8')) + +def _read_dockerfile(path): + return open(path, 'rt', encoding='utf-8').read() + +def _file_checksum(filename): + return _bytes_checksum(open(filename, 'rb').read()) + + +def _guess_engine_command(): + """ Guess a working engine command or raise exception if not found""" + commands = [] + + if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]: + commands += [["podman"]] + if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]: + commands += [["docker"], ["sudo", "-n", "docker"]] + for cmd in commands: + try: + # docker version will return the client details in stdout + # but still report a status of 1 if it can't contact the daemon + if subprocess.call(cmd + ["version"], + stdout=DEVNULL, stderr=DEVNULL) == 0: + return cmd + except OSError: + pass + commands_txt = "\n".join([" " + " ".join(x) for x in commands]) + raise Exception("Cannot find working engine command. Tried:\n%s" % + commands_txt) + + +def _copy_with_mkdir(src, root_dir, sub_path='.', name=None): + """Copy src into root_dir, creating sub_path as needed.""" + dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) + try: + os.makedirs(dest_dir) + except OSError: + # we can safely ignore already created directories + pass + + dest_file = "%s/%s" % (dest_dir, name if name else os.path.basename(src)) + + try: + copy(src, dest_file) + except FileNotFoundError: + print("Couldn't copy %s to %s" % (src, dest_file)) + pass + + +def _get_so_libs(executable): + """Return a list of libraries associated with an executable. + + The paths may be symbolic links which would need to be resolved to + ensure the right data is copied.""" + + libs = [] + ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)") + try: + ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8') + for line in ldd_output.split("\n"): + search = ldd_re.search(line) + if search: + try: + libs.append(search.group(1)) + except IndexError: + pass + except subprocess.CalledProcessError: + print("%s had no associated libraries (static build?)" % (executable)) + + return libs + + +def _copy_binary_with_libs(src, bin_dest, dest_dir): + """Maybe copy a binary and all its dependent libraries. + + If bin_dest isn't set we only copy the support libraries because + we don't need qemu in the docker path to run (due to persistent + mapping). Indeed users may get confused if we aren't running what + is in the image. + + This does rely on the host file-system being fairly multi-arch + aware so the file don't clash with the guests layout. + """ + + if bin_dest: + _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest)) + else: + print("only copying support libraries for %s" % (src)) + + libs = _get_so_libs(src) + if libs: + for l in libs: + so_path = os.path.dirname(l) + name = os.path.basename(l) + real_l = os.path.realpath(l) + _copy_with_mkdir(real_l, dest_dir, so_path, name) + + +def _check_binfmt_misc(executable): + """Check binfmt_misc has entry for executable in the right place. + + The details of setting up binfmt_misc are outside the scope of + this script but we should at least fail early with a useful + message if it won't work. + + Returns the configured binfmt path and a valid flag. For + persistent configurations we will still want to copy and dependent + libraries. + """ + + binary = os.path.basename(executable) + binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary) + + if not os.path.exists(binfmt_entry): + print ("No binfmt_misc entry for %s" % (binary)) + return None, False + + with open(binfmt_entry) as x: entry = x.read() + + if re.search("flags:.*F.*\n", entry): + print("binfmt_misc for %s uses persistent(F) mapping to host binary" % + (binary)) + return None, True + + m = re.search("interpreter (\S+)\n", entry) + interp = m.group(1) + if interp and interp != executable: + print("binfmt_misc for %s does not point to %s, using %s" % + (binary, executable, interp)) + + return interp, True + + +def _read_qemu_dockerfile(img_name): + # special case for Debian linux-user images + if img_name.startswith("debian") and img_name.endswith("user"): + img_name = "debian-bootstrap" + + df = os.path.join(os.path.dirname(__file__), "dockerfiles", + img_name + ".docker") + return _read_dockerfile(df) + + +def _dockerfile_preprocess(df): + out = "" + for l in df.splitlines(): + if len(l.strip()) == 0 or l.startswith("#"): + continue + from_pref = "FROM qemu/" + if l.startswith(from_pref): + # TODO: Alternatively we could replace this line with "FROM $ID" + # where $ID is the image's hex id obtained with + # $ docker images $IMAGE --format="{{.Id}}" + # but unfortunately that's not supported by RHEL 7. + inlining = _read_qemu_dockerfile(l[len(from_pref):]) + out += _dockerfile_preprocess(inlining) + continue + out += l + "\n" + return out + + +class Docker(object): + """ Running Docker commands """ + def __init__(self): + self._command = _guess_engine_command() + + if ("docker" in self._command and + "TRAVIS" not in os.environ and + "GITLAB_CI" not in os.environ): + os.environ["DOCKER_BUILDKIT"] = "1" + self._buildkit = True + else: + self._buildkit = False + + self._instance = None + atexit.register(self._kill_instances) + signal.signal(signal.SIGTERM, self._kill_instances) + signal.signal(signal.SIGHUP, self._kill_instances) + + def _do(self, cmd, quiet=True, **kwargs): + if quiet: + kwargs["stdout"] = DEVNULL + return subprocess.call(self._command + cmd, **kwargs) + + def _do_check(self, cmd, quiet=True, **kwargs): + if quiet: + kwargs["stdout"] = DEVNULL + return subprocess.check_call(self._command + cmd, **kwargs) + + def _do_kill_instances(self, only_known, only_active=True): + cmd = ["ps", "-q"] + if not only_active: + cmd.append("-a") + + filter = "--filter=label=com.qemu.instance.uuid" + if only_known: + if self._instance: + filter += "=%s" % (self._instance) + else: + # no point trying to kill, we finished + return + + print("filter=%s" % (filter)) + cmd.append(filter) + for i in self._output(cmd).split(): + self._do(["rm", "-f", i]) + + def clean(self): + self._do_kill_instances(False, False) + return 0 + + def _kill_instances(self, *args, **kwargs): + return self._do_kill_instances(True) + + def _output(self, cmd, **kwargs): + try: + return subprocess.check_output(self._command + cmd, + stderr=subprocess.STDOUT, + encoding='utf-8', + **kwargs) + except TypeError: + # 'encoding' argument was added in 3.6+ + return subprocess.check_output(self._command + cmd, + stderr=subprocess.STDOUT, + **kwargs).decode('utf-8') + + + def inspect_tag(self, tag): + try: + return self._output(["inspect", tag]) + except subprocess.CalledProcessError: + return None + + def get_image_creation_time(self, info): + return json.loads(info)[0]["Created"] + + def get_image_dockerfile_checksum(self, tag): + resp = self.inspect_tag(tag) + labels = json.loads(resp)[0]["Config"].get("Labels", {}) + return labels.get("com.qemu.dockerfile-checksum", "") + + def build_image(self, tag, docker_dir, dockerfile, + quiet=True, user=False, argv=None, registry=None, + extra_files_cksum=[]): + if argv is None: + argv = [] + + # pre-calculate the docker checksum before any + # substitutions we make for caching + checksum = _text_checksum(_dockerfile_preprocess(dockerfile)) + + if registry is not None: + sources = re.findall("FROM qemu\/(.*)", dockerfile) + # Fetch any cache layers we can, may fail + for s in sources: + pull_args = ["pull", "%s/qemu/%s" % (registry, s)] + if self._do(pull_args, quiet=quiet) != 0: + registry = None + break + # Make substitutions + if registry is not None: + dockerfile = dockerfile.replace("FROM qemu/", + "FROM %s/qemu/" % + (registry)) + + tmp_df = tempfile.NamedTemporaryFile(mode="w+t", + encoding='utf-8', + dir=docker_dir, suffix=".docker") + tmp_df.write(dockerfile) + + if user: + uid = os.getuid() + uname = getpwuid(uid).pw_name + tmp_df.write("\n") + tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % + (uname, uid, uname)) + + tmp_df.write("\n") + tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s\n" % (checksum)) + for f, c in extra_files_cksum: + tmp_df.write("LABEL com.qemu.%s-checksum=%s\n" % (f, c)) + + tmp_df.flush() + + build_args = ["build", "-t", tag, "-f", tmp_df.name] + if self._buildkit: + build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"] + + if registry is not None: + pull_args = ["pull", "%s/%s" % (registry, tag)] + self._do(pull_args, quiet=quiet) + cache = "%s/%s" % (registry, tag) + build_args += ["--cache-from", cache] + build_args += argv + build_args += [docker_dir] + + self._do_check(build_args, + quiet=quiet) + + def update_image(self, tag, tarball, quiet=True): + "Update a tagged image using " + + self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball) + + def image_matches_dockerfile(self, tag, dockerfile): + try: + checksum = self.get_image_dockerfile_checksum(tag) + except Exception: + return False + return checksum == _text_checksum(_dockerfile_preprocess(dockerfile)) + + def run(self, cmd, keep, quiet, as_user=False): + label = uuid.uuid4().hex + if not keep: + self._instance = label + + if as_user: + uid = os.getuid() + cmd = [ "-u", str(uid) ] + cmd + # podman requires a bit more fiddling + if self._command[0] == "podman": + cmd.insert(0, '--userns=keep-id') + + ret = self._do_check(["run", "--rm", "--label", + "com.qemu.instance.uuid=" + label] + cmd, + quiet=quiet) + if not keep: + self._instance = None + return ret + + def command(self, cmd, argv, quiet): + return self._do([cmd] + argv, quiet=quiet) + + +class SubCommand(object): + """A SubCommand template base class""" + name = None # Subcommand name + + def shared_args(self, parser): + parser.add_argument("--quiet", action="store_true", + help="Run quietly unless an error occurred") + + def args(self, parser): + """Setup argument parser""" + pass + + def run(self, args, argv): + """Run command. + args: parsed argument by argument parser. + argv: remaining arguments from sys.argv. + """ + pass + + +class RunCommand(SubCommand): + """Invoke docker run and take care of cleaning up""" + name = "run" + + def args(self, parser): + parser.add_argument("--keep", action="store_true", + help="Don't remove image when command completes") + parser.add_argument("--run-as-current-user", action="store_true", + help="Run container using the current user's uid") + + def run(self, args, argv): + return Docker().run(argv, args.keep, quiet=args.quiet, + as_user=args.run_as_current_user) + + +class BuildCommand(SubCommand): + """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>""" + name = "build" + + def args(self, parser): + parser.add_argument("--include-executable", "-e", + help="""Specify a binary that will be copied to the + container together with all its dependent + libraries""") + parser.add_argument("--skip-binfmt", + action="store_true", + help="""Skip binfmt entry check (used for testing)""") + parser.add_argument("--extra-files", nargs='*', + help="""Specify files that will be copied in the + Docker image, fulfilling the ADD directive from the + Dockerfile""") + parser.add_argument("--add-current-user", "-u", dest="user", + action="store_true", + help="Add the current user to image's passwd") + parser.add_argument("--registry", "-r", + help="cache from docker registry") + parser.add_argument("-t", dest="tag", + help="Image Tag") + parser.add_argument("-f", dest="dockerfile", + help="Dockerfile name") + + def run(self, args, argv): + dockerfile = _read_dockerfile(args.dockerfile) + tag = args.tag + + dkr = Docker() + if "--no-cache" not in argv and \ + dkr.image_matches_dockerfile(tag, dockerfile): + if not args.quiet: + print("Image is up to date.") + else: + # Create a docker context directory for the build + docker_dir = tempfile.mkdtemp(prefix="docker_build") + + # Validate binfmt_misc will work + if args.skip_binfmt: + qpath = args.include_executable + elif args.include_executable: + qpath, enabled = _check_binfmt_misc(args.include_executable) + if not enabled: + return 1 + + # Is there a .pre file to run in the build context? + docker_pre = os.path.splitext(args.dockerfile)[0]+".pre" + if os.path.exists(docker_pre): + stdout = DEVNULL if args.quiet else None + rc = subprocess.call(os.path.realpath(docker_pre), + cwd=docker_dir, stdout=stdout) + if rc == 3: + print("Skip") + return 0 + elif rc != 0: + print("%s exited with code %d" % (docker_pre, rc)) + return 1 + + # Copy any extra files into the Docker context. These can be + # included by the use of the ADD directive in the Dockerfile. + cksum = [] + if args.include_executable: + # FIXME: there is no checksum of this executable and the linked + # libraries, once the image built any change of this executable + # or any library won't trigger another build. + _copy_binary_with_libs(args.include_executable, + qpath, docker_dir) + + for filename in args.extra_files or []: + _copy_with_mkdir(filename, docker_dir) + cksum += [(filename, _file_checksum(filename))] + + argv += ["--build-arg=" + k.lower() + "=" + v + for k, v in os.environ.items() + if k.lower() in FILTERED_ENV_NAMES] + dkr.build_image(tag, docker_dir, dockerfile, + quiet=args.quiet, user=args.user, + argv=argv, registry=args.registry, + extra_files_cksum=cksum) + + rmtree(docker_dir) + + return 0 + +class FetchCommand(SubCommand): + """ Fetch a docker image from the registry. Args: <tag> <registry>""" + name = "fetch" + + def args(self, parser): + parser.add_argument("tag", + help="Local tag for image") + parser.add_argument("registry", + help="Docker registry") + + def run(self, args, argv): + dkr = Docker() + dkr.command(cmd="pull", quiet=args.quiet, + argv=["%s/%s" % (args.registry, args.tag)]) + dkr.command(cmd="tag", quiet=args.quiet, + argv=["%s/%s" % (args.registry, args.tag), args.tag]) + + +class UpdateCommand(SubCommand): + """ Update a docker image. Args: <tag> <actions>""" + name = "update" + + def args(self, parser): + parser.add_argument("tag", + help="Image Tag") + parser.add_argument("--executable", + help="Executable to copy") + parser.add_argument("--add-current-user", "-u", dest="user", + action="store_true", + help="Add the current user to image's passwd") + + def run(self, args, argv): + # Create a temporary tarball with our whole build context and + # dockerfile for the update + tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") + tmp_tar = TarFile(fileobj=tmp, mode='w') + + # Create a Docker buildfile + df = StringIO() + df.write(u"FROM %s\n" % args.tag) + + if args.executable: + # Add the executable to the tarball, using the current + # configured binfmt_misc path. If we don't get a path then we + # only need the support libraries copied + ff, enabled = _check_binfmt_misc(args.executable) + + if not enabled: + print("binfmt_misc not enabled, update disabled") + return 1 + + if ff: + tmp_tar.add(args.executable, arcname=ff) + + # Add any associated libraries + libs = _get_so_libs(args.executable) + if libs: + for l in libs: + so_path = os.path.dirname(l) + name = os.path.basename(l) + real_l = os.path.realpath(l) + try: + tmp_tar.add(real_l, arcname="%s/%s" % (so_path, name)) + except FileNotFoundError: + print("Couldn't add %s/%s to archive" % (so_path, name)) + pass + + df.write(u"ADD . /\n") + + if args.user: + uid = os.getuid() + uname = getpwuid(uid).pw_name + df.write("\n") + df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % + (uname, uid, uname)) + + df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8")) + + df_tar = TarInfo(name="Dockerfile") + df_tar.size = df_bytes.getbuffer().nbytes + tmp_tar.addfile(df_tar, fileobj=df_bytes) + + tmp_tar.close() + + # reset the file pointers + tmp.flush() + tmp.seek(0) + + # Run the build with our tarball context + dkr = Docker() + dkr.update_image(args.tag, tmp, quiet=args.quiet) + + return 0 + + +class CleanCommand(SubCommand): + """Clean up docker instances""" + name = "clean" + + def run(self, args, argv): + Docker().clean() + return 0 + + +class ImagesCommand(SubCommand): + """Run "docker images" command""" + name = "images" + + def run(self, args, argv): + return Docker().command("images", argv, args.quiet) + + +class ProbeCommand(SubCommand): + """Probe if we can run docker automatically""" + name = "probe" + + def run(self, args, argv): + try: + docker = Docker() + if docker._command[0] == "docker": + print("docker") + elif docker._command[0] == "sudo": + print("sudo docker") + elif docker._command[0] == "podman": + print("podman") + except Exception: + print("no") + + return + + +class CcCommand(SubCommand): + """Compile sources with cc in images""" + name = "cc" + + def args(self, parser): + parser.add_argument("--image", "-i", required=True, + help="The docker image in which to run cc") + parser.add_argument("--cc", default="cc", + help="The compiler executable to call") + parser.add_argument("--source-path", "-s", nargs="*", dest="paths", + help="""Extra paths to (ro) mount into container for + reading sources""") + + def run(self, args, argv): + if argv and argv[0] == "--": + argv = argv[1:] + cwd = os.getcwd() + cmd = ["-w", cwd, + "-v", "%s:%s:rw" % (cwd, cwd)] + if args.paths: + for p in args.paths: + cmd += ["-v", "%s:%s:ro,z" % (p, p)] + cmd += [args.image, args.cc] + cmd += argv + return Docker().run(cmd, False, quiet=args.quiet, + as_user=True) + + +class CheckCommand(SubCommand): + """Check if we need to re-build a docker image out of a dockerfile. + Arguments: <tag> <dockerfile>""" + name = "check" + + def args(self, parser): + parser.add_argument("tag", + help="Image Tag") + parser.add_argument("dockerfile", default=None, + help="Dockerfile name", nargs='?') + parser.add_argument("--checktype", choices=["checksum", "age"], + default="checksum", help="check type") + parser.add_argument("--olderthan", default=60, type=int, + help="number of minutes") + + def run(self, args, argv): + tag = args.tag + + try: + dkr = Docker() + except subprocess.CalledProcessError: + print("Docker not set up") + return 1 + + info = dkr.inspect_tag(tag) + if info is None: + print("Image does not exist") + return 1 + + if args.checktype == "checksum": + if not args.dockerfile: + print("Need a dockerfile for tag:%s" % (tag)) + return 1 + + dockerfile = _read_dockerfile(args.dockerfile) + + if dkr.image_matches_dockerfile(tag, dockerfile): + if not args.quiet: + print("Image is up to date") + return 0 + else: + print("Image needs updating") + return 1 + elif args.checktype == "age": + timestr = dkr.get_image_creation_time(info).split(".")[0] + created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S") + past = datetime.now() - timedelta(minutes=args.olderthan) + if created < past: + print ("Image created @ %s more than %d minutes old" % + (timestr, args.olderthan)) + return 1 + else: + if not args.quiet: + print ("Image less than %d minutes old" % (args.olderthan)) + return 0 + + +def main(): + global USE_ENGINE + + parser = argparse.ArgumentParser(description="A Docker helper", + usage="%s <subcommand> ..." % + os.path.basename(sys.argv[0])) + parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum), + help="specify which container engine to use") + subparsers = parser.add_subparsers(title="subcommands", help=None) + for cls in SubCommand.__subclasses__(): + cmd = cls() + subp = subparsers.add_parser(cmd.name, help=cmd.__doc__) + cmd.shared_args(subp) + cmd.args(subp) + subp.set_defaults(cmdobj=cmd) + args, argv = parser.parse_known_args() + if args.engine: + USE_ENGINE = args.engine + return args.cmdobj.run(args, argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker new file mode 100644 index 000000000..7e6997e30 --- /dev/null +++ b/tests/docker/dockerfiles/alpine.docker @@ -0,0 +1,59 @@ + +FROM alpine:edge + +RUN apk update +RUN apk upgrade + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + alsa-lib-dev \ + bash \ + binutils \ + ccache \ + coreutils \ + curl-dev \ + g++ \ + gcc \ + git \ + glib-dev \ + glib-static \ + gnutls-dev \ + gtk+3.0-dev \ + libaio-dev \ + libbpf-dev \ + libcap-ng-dev \ + libffi-dev \ + libjpeg-turbo-dev \ + libnfs-dev \ + libpng-dev \ + libseccomp-dev \ + libssh-dev \ + libusb-dev \ + libxml2-dev \ + lzo-dev \ + make \ + mesa-dev \ + mesa-egl \ + mesa-gbm \ + meson \ + ncurses-dev \ + ninja \ + perl \ + pulseaudio-dev \ + python3 \ + py3-sphinx \ + py3-sphinx_rtd_theme \ + shadow \ + snappy-dev \ + spice-dev \ + texinfo \ + usbredir-dev \ + util-linux-dev \ + vde2-dev \ + virglrenderer-dev \ + vte3-dev \ + xfsprogs-dev \ + zlib-dev \ + zlib-static + +RUN apk add $PACKAGES diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker new file mode 100644 index 000000000..7f135f8e8 --- /dev/null +++ b/tests/docker/dockerfiles/centos8.docker @@ -0,0 +1,112 @@ +FROM docker.io/centos:8 + +RUN dnf -y update +ENV PACKAGES \ + SDL2-devel \ + alsa-lib-devel \ + bc \ + brlapi-devel \ + bzip2 \ + bzip2-devel \ + ca-certificates \ + capstone-devel \ + ccache \ + clang \ + ctags \ + cyrus-sasl-devel \ + daxctl-devel \ + dbus-daemon \ + device-mapper-multipath-devel \ + diffutils \ + findutils \ + gcc \ + gcc-c++ \ + genisoimage \ + gettext \ + git \ + glib2-devel \ + glibc-langpack-en \ + glibc-static \ + glusterfs-api-devel \ + gnutls-devel \ + gtk3-devel \ + hostname \ + jemalloc-devel \ + libaio-devel \ + libasan \ + libattr-devel \ + libbpf-devel \ + libcacard-devel \ + libcap-ng-devel \ + libcurl-devel \ + libdrm-devel \ + libepoxy-devel \ + libfdt-devel \ + libffi-devel \ + libgcrypt-devel \ + libiscsi-devel \ + libjpeg-devel \ + libnfs-devel \ + libpmem-devel \ + libpng-devel \ + librbd-devel \ + libseccomp-devel \ + libselinux-devel \ + libslirp-devel \ + libssh-devel \ + libtasn1-devel \ + libubsan \ + libudev-devel \ + libusbx-devel \ + libxml2-devel \ + libzstd-devel \ + llvm \ + lzo-devel \ + make \ + mesa-libgbm-devel \ + ncurses-devel \ + nettle-devel \ + ninja-build \ + nmap-ncat \ + numactl-devel \ + openssh-clients \ + pam-devel \ + perl \ + perl-Test-Harness \ + pixman-devel \ + pkgconfig \ + pulseaudio-libs-devel \ + python3 \ + python3-PyYAML \ + python3-numpy \ + python3-pillow \ + python3-pip \ + python3-setuptools \ + python3-sphinx \ + python3-sphinx_rtd_theme \ + python3-virtualenv \ + python3-wheel \ + rdma-core-devel \ + rpm \ + sed \ + snappy-devel \ + spice-protocol \ + spice-server-devel \ + systemd-devel \ + systemtap-sdt-devel \ + tar \ + texinfo \ + usbredir-devel \ + util-linux \ + virglrenderer-devel \ + vte291-devel \ + which \ + xfsprogs-devel \ + zlib-devel + +RUN dnf install -y dnf-plugins-core && \ + dnf config-manager --set-enabled powertools && \ + dnf install -y centos-release-advanced-virtualization && \ + dnf install -y epel-release && \ + dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker new file mode 100644 index 000000000..dedcea58b --- /dev/null +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -0,0 +1,53 @@ +# +# Docker all cross-compiler target (tests only) +# +# While the normal cross builds take care to setup proper multiarch +# build environments which can cross build QEMU this just installs the +# basic compilers for as many targets as possible. We shall use this +# to build and run linux-user tests on GitLab +# +FROM qemu/debian10 + +# What we need to build QEMU itself +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy qemu + +# Add the foreign architecture we want and install dependencies +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + gcc-alpha-linux-gnu \ + libc6.1-dev-alpha-cross \ + gcc-arm-linux-gnueabihf \ + libc6-dev-armhf-cross \ + gcc-hppa-linux-gnu \ + libc6-dev-hppa-cross \ + gcc-m68k-linux-gnu \ + libc6-dev-m68k-cross \ + gcc-mips-linux-gnu \ + libc6-dev-mips-cross \ + gcc-mips64-linux-gnuabi64 \ + libc6-dev-mips64-cross \ + gcc-mips64el-linux-gnuabi64 \ + libc6-dev-mips64el-cross \ + gcc-mipsel-linux-gnu \ + libc6-dev-mipsel-cross \ + gcc-powerpc-linux-gnu \ + libc6-dev-powerpc-cross \ + gcc-powerpc64-linux-gnu \ + libc6-dev-ppc64-cross \ + gcc-powerpc64le-linux-gnu \ + libc6-dev-ppc64el-cross \ + gcc-riscv64-linux-gnu \ + libc6-dev-riscv64-cross \ + gcc-s390x-linux-gnu \ + libc6-dev-s390x-cross \ + gcc-sh4-linux-gnu \ + libc6-dev-sh4-cross \ + gcc-sparc64-linux-gnu \ + libc6-dev-sparc64-cross + +ENV QEMU_CONFIGURE_OPTS --disable-system --disable-docs --disable-tools +ENV DEF_TARGET_LIST aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user diff --git a/tests/docker/dockerfiles/debian-alpha-cross.docker b/tests/docker/dockerfiles/debian-alpha-cross.docker new file mode 100644 index 000000000..10fe30df0 --- /dev/null +++ b/tests/docker/dockerfiles/debian-alpha-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-alpha-linux-gnu \ + libc6.1-dev-alpha-cross diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker new file mode 100644 index 000000000..870109ef6 --- /dev/null +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -0,0 +1,22 @@ +# +# Docker x86_64 cross target +# +# This docker target is used on non-x86_64 machines which need the +# x86_64 cross compilers installed. +# +FROM qemu/debian10 +MAINTAINER Alex Bennée <alex.bennee@linaro.org> + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture amd64 +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + crossbuild-essential-amd64 +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a amd64 --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=x86_64-linux-gnu- +ENV DEF_TARGET_LIST x86_64-softmmu,x86_64-linux-user,i386-softmmu,i386-linux-user diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker new file mode 100644 index 000000000..ed546edcd --- /dev/null +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -0,0 +1,59 @@ +# +# Docker x86_64 target +# +# This docker target builds on the Debian Buster base image. Further +# libraries which are not widely available are installed by hand. +# +FROM qemu/debian10 +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy qemu + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + cscope \ + genisoimage \ + exuberant-ctags \ + global \ + libbz2-dev \ + liblzo2-dev \ + libgcrypt20-dev \ + libfdt-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsnappy-dev \ + libvte-dev \ + netcat-openbsd \ + openssh-client \ + python3-numpy \ + python3-opencv \ + python3-venv + +# virgl +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libegl1-mesa-dev \ + libepoxy-dev \ + libgbm-dev +RUN git clone https://gitlab.freedesktop.org/virgl/virglrenderer.git /usr/src/virglrenderer && \ + cd /usr/src/virglrenderer && git checkout virglrenderer-0.8.0 +RUN cd /usr/src/virglrenderer && ./autogen.sh && ./configure --disable-tests && make install + +# netmap +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + linux-headers-amd64 +RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap +RUN cd /usr/src/netmap && git checkout v11.3 +RUN cd /usr/src/netmap/LINUX && ./configure --no-drivers --no-apps --kernel-dir=$(ls -d /usr/src/linux-headers-*-amd64) && make install +ENV QEMU_CONFIGURE_OPTS --enable-netmap + +RUN ldconfig + +# gcrypt +ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-gcrypt diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker new file mode 100644 index 000000000..166e24df1 --- /dev/null +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -0,0 +1,32 @@ +# +# Docker arm64 cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture arm64 +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + crossbuild-essential-arm64 +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a arm64 --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu- +ENV DEF_TARGET_LIST aarch64-softmmu,aarch64-linux-user + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:arm64 \ + liblzo2-dev:arm64 \ + librdmacm-dev:arm64 \ + libsnappy-dev:arm64 \ + libxen-dev:arm64 + +# nettle +ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle diff --git a/tests/docker/dockerfiles/debian-arm64-test-cross.docker b/tests/docker/dockerfiles/debian-arm64-test-cross.docker new file mode 100644 index 000000000..53a9012be --- /dev/null +++ b/tests/docker/dockerfiles/debian-arm64-test-cross.docker @@ -0,0 +1,13 @@ +# +# Docker arm64 cross-compiler target (tests only) +# +# This docker target builds on the debian Bullseye base image. +# +FROM qemu/debian11 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture arm64 +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + crossbuild-essential-arm64 gcc-10-aarch64-linux-gnu diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker new file mode 100644 index 000000000..b7b1a3585 --- /dev/null +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -0,0 +1,26 @@ +# +# Docker armel cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# +FROM qemu/debian10 +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture armel && \ + apt update && \ + apt install -yy crossbuild-essential-armel && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a armel --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=arm-linux-gnueabi- +ENV DEF_TARGET_LIST arm-softmmu,arm-linux-user,armeb-linux-user + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:armel \ + liblzo2-dev:armel \ + librdmacm-dev:armel \ + libsnappy-dev:armel diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker new file mode 100644 index 000000000..25d761883 --- /dev/null +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -0,0 +1,29 @@ +# +# Docker armhf cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# +FROM qemu/debian10 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture armhf +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + crossbuild-essential-armhf +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a armhf --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=arm-linux-gnueabihf- +ENV DEF_TARGET_LIST arm-softmmu,arm-linux-user,armeb-linux-user + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:armhf \ + liblzo2-dev:armhf \ + librdmacm-dev:armhf \ + libsnappy-dev:armhf \ + libxen-dev:armhf diff --git a/tests/docker/dockerfiles/debian-bootstrap.docker b/tests/docker/dockerfiles/debian-bootstrap.docker new file mode 100644 index 000000000..e13c26a7e --- /dev/null +++ b/tests/docker/dockerfiles/debian-bootstrap.docker @@ -0,0 +1,20 @@ +# Create Debian Bootstrap Image +# +# This is intended to be pre-poluated by: +# - a first stage debootstrap (see debian-bootstrap.pre) +# - a native qemu-$arch that binfmt_misc will run +FROM scratch + +# Add everything from the context into the container +ADD . / + +# Patch all mounts as docker already has stuff set up +# (this is not needed for later debootstraps but is harmless atm) +RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions + +# Run stage 2 +RUN /debootstrap/debootstrap --second-stage + +# At this point we can install additional packages if we want +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/deb/deb-src/" >> /etc/apt/sources.list diff --git a/tests/docker/dockerfiles/debian-bootstrap.pre b/tests/docker/dockerfiles/debian-bootstrap.pre new file mode 100755 index 000000000..35c85f7db --- /dev/null +++ b/tests/docker/dockerfiles/debian-bootstrap.pre @@ -0,0 +1,130 @@ +#!/bin/sh +# +# Simple wrapper for debootstrap, run in the docker build context +# +FAKEROOT=$(which fakeroot 2> /dev/null) +# debootstrap < 1.0.67 generates empty sources.list, see Debian#732255 +MIN_DEBOOTSTRAP_VERSION=1.0.67 + +exit_and_skip() +{ + exit 3 +} + +# +# fakeroot is needed to run the bootstrap stage +# +if [ -z $FAKEROOT ]; then + echo "Please install fakeroot to enable bootstraping" >&2 + exit_and_skip + +fi + +if [ -z "${DEB_ARCH}" ]; then + echo "Please set DEB_ARCH to choose an architecture (e.g. armhf)" >&2 + exit_and_skip + +fi + +if [ -z "${DEB_TYPE}" ]; then + echo "Please set DEB_TYPE to a Debian archive name (e.g. testing)" >&2 + exit_and_skip + +fi + +# The following allow finer grain control over the defaults +if [ -z "${DEB_VARIANT}" ]; then + DEB_VARIANT=buildd +fi + +if [ -z "${DEB_URL}" ]; then + DEB_URL="http://httpredir.debian.org/debian" +fi + +# We check in order for +# +# - DEBOOTSTRAP_DIR pointing at a development checkout +# - PATH for the debootstrap script (installed) +# +# If neither option works then we checkout debootstrap from its +# upstream SCM and run it from there. +# + +if [ -z $DEBOOTSTRAP_DIR ]; then + NEED_DEBOOTSTRAP=false + DEBOOTSTRAP=$(which debootstrap 2> /dev/null) + if [ -z $DEBOOTSTRAP ]; then + echo "No debootstrap installed, attempting to install from SCM" + NEED_DEBOOTSTRAP=true + else + INSTALLED_VERSION=$(${DEBOOTSTRAP} --version | sed 's/debootstrap \([0-9\.]*\)[^0-9\.]*.*/\1/') + if ! (echo "${MIN_DEBOOTSTRAP_VERSION}" ; echo "${INSTALLED_VERSION}") \ + | sort -t . -n -k 1,1 -k 2,2 -k 3,3 -C ; then + echo "debootstrap too old, attempting to install from SCM" + NEED_DEBOOTSTRAP=true + fi + fi + if $NEED_DEBOOTSTRAP; then + DEBOOTSTRAP_SOURCE=https://salsa.debian.org/installer-team/debootstrap.git + git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git + export DEBOOTSTRAP_DIR=./debootstrap.git + DEBOOTSTRAP=./debootstrap.git/debootstrap + (cd "${DEBOOTSTRAP_DIR}" && "${FAKEROOT}" make ) + fi +else + DEBOOTSTRAP=${DEBOOTSTRAP_DIR}/debootstrap + if [ ! -f $DEBOOTSTRAP ]; then + echo "Couldn't find script at ${DEBOOTSTRAP}" >&2 + exit_and_skip + fi +fi + +# +# Add optional args +# +if [ -n "${DEB_KEYRING}" ]; then + DEBOOTSTRAP="${DEBOOTSTRAP} --keyring=${DEB_KEYRING}" +fi + +# +# Finally check to see if any qemu's are installed +# +BINFMT_DIR=/proc/sys/fs/binfmt_misc +if [ ! -e $BINFMT_DIR ]; then + echo "binfmt_misc needs enabling for a QEMU bootstrap to work" >&2 + exit_and_skip +else + # DEB_ARCH and QEMU arch names are not totally aligned + case "${DEB_ARCH}" in + amd64) + QEMU=qemu-i386 + ;; + armel|armhf) + QEMU=qemu-arm + ;; + arm64) + QEMU=qemu-aarch64 + ;; + powerpc) + QEMU=qemu-ppc + ;; + ppc64el) + QEMU=qemu-ppc64le + ;; + s390) + QEMU=qemu-s390x + ;; + *) + QEMU=qemu-${DEB_ARCH} + ;; + esac + if [ ! -e "${BINFMT_DIR}/$QEMU" ]; then + echo "No binfmt_misc rule to run $QEMU, can't bootstrap" >&2 + exit_and_skip + fi +fi + +echo "Building a rootfs using ${FAKEROOT} and ${DEBOOTSTRAP} ${DEB_ARCH}/${DEB_TYPE}" + +${FAKEROOT} ${DEBOOTSTRAP} --variant=$DEB_VARIANT --foreign --arch=$DEB_ARCH $DEB_TYPE . $DEB_URL || exit 1 +exit 0 diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker new file mode 100644 index 000000000..d5dc299dc --- /dev/null +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker @@ -0,0 +1,45 @@ +# +# Docker Hexagon cross-compiler target +# +# This docker target is used for building hexagon tests. As it also +# needs to be able to build QEMU itself in CI we include it's +# build-deps. It is also a "stand-alone" image so as not to be +# triggered by re-builds on other base images given it takes a long +# time to build. +# +FROM qemu/debian10 + +# Install common build utilities +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bison \ + cmake \ + flex \ + lld \ + rsync \ + wget + +ENV TOOLCHAIN_INSTALL /usr/local +ENV ROOTFS /usr/local + +ENV LLVM_URL https://github.com/llvm/llvm-project/archive/bfcd21876adc3498065e4da92799f613e730d475.tar.gz +ENV MUSL_URL https://github.com/quic/musl/archive/aff74b395fbf59cd7e93b3691905aa1af6c0778c.tar.gz +ENV LINUX_URL https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.6.18.tar.xz + +ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh + +RUN cd /root/hexagon-toolchain && ./build-toolchain.sh + +FROM debian:buster-slim +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list +# Install QEMU build deps for use in CI +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy git ninja-build && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy --arch-only qemu +COPY --from=0 /usr/local /usr/local +ENV PATH $PATH:/usr/local/bin/ diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh b/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh new file mode 100755 index 000000000..19b1c9f83 --- /dev/null +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +set -e + +BASE=$(readlink -f ${PWD}) + +TOOLCHAIN_INSTALL=$(readlink -f "$TOOLCHAIN_INSTALL") +ROOTFS=$(readlink -f "$ROOTFS") + +TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin +HEX_SYSROOT=${TOOLCHAIN_INSTALL}/hexagon-unknown-linux-musl +HEX_TOOLS_TARGET_BASE=${HEX_SYSROOT}/usr + +function cdp() { + DIR="$1" + mkdir -p "$DIR" + cd "$DIR" +} + +function fetch() { + DIR="$1" + URL="$2" + TEMP="$(readlink -f "$PWD/tmp.tar.gz")" + wget --quiet "$URL" -O "$TEMP" + cdp "$DIR" + tar xaf "$TEMP" --strip-components=1 + rm "$TEMP" + cd - +} + +build_llvm_clang() { + fetch "$BASE/llvm-project" "$LLVM_URL" + cdp "$BASE/build-llvm" + + cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${TOOLCHAIN_INSTALL} \ + -DLLVM_ENABLE_LLD=ON \ + -DLLVM_TARGETS_TO_BUILD="Hexagon" \ + -DLLVM_ENABLE_PROJECTS="clang;lld" \ + "$BASE/llvm-project/llvm" + ninja all install + cd ${TOOLCHAIN_BIN} + ln -sf clang hexagon-unknown-linux-musl-clang + ln -sf clang++ hexagon-unknown-linux-musl-clang++ + ln -sf llvm-ar hexagon-unknown-linux-musl-ar + ln -sf llvm-objdump hexagon-unknown-linux-musl-objdump + ln -sf llvm-objcopy hexagon-unknown-linux-musl-objcopy + ln -sf llvm-readelf hexagon-unknown-linux-musl-readelf + ln -sf llvm-ranlib hexagon-unknown-linux-musl-ranlib + + # workaround for now: + cat <<EOF > hexagon-unknown-linux-musl.cfg +-G0 --sysroot=${HEX_SYSROOT} +EOF +} + +build_clang_rt() { + cdp "$BASE/build-clang_rt" + cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_CONFIG_PATH="$BASE/build-llvm/bin/llvm-config" \ + -DCMAKE_ASM_FLAGS="-G0 -mlong-calls -fno-pic --target=hexagon-unknown-linux-musl " \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_C_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \ + -DCMAKE_ASM_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \ + -DCMAKE_INSTALL_PREFIX=${HEX_TOOLS_TARGET_BASE} \ + -DCMAKE_CROSSCOMPILING=ON \ + -DCMAKE_C_COMPILER_FORCED=ON \ + -DCMAKE_CXX_COMPILER_FORCED=ON \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DCOMPILER_RT_BUILTINS_ENABLE_PIC=OFF \ + -DCMAKE_SIZEOF_VOID_P=4 \ + -DCOMPILER_RT_OS_DIR= \ + -DCAN_TARGET_hexagon=1 \ + -DCAN_TARGET_x86_64=0 \ + -DCOMPILER_RT_SUPPORTED_ARCH=hexagon \ + -DLLVM_ENABLE_PROJECTS="compiler-rt" \ + "$BASE/llvm-project/compiler-rt" + ninja install-compiler-rt +} + +build_musl_headers() { + fetch "$BASE/musl" "$MUSL_URL" + cd "$BASE/musl" + make clean + CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \ + CROSS_COMPILE=hexagon-unknown-linux-musl \ + LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \ + CROSS_CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \ + ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE} + PATH=${TOOLCHAIN_BIN}:$PATH make CROSS_COMPILE= install-headers + + cd ${HEX_SYSROOT}/.. + ln -sf hexagon-unknown-linux-musl hexagon +} + +build_kernel_headers() { + fetch "$BASE/linux" "$LINUX_URL" + mkdir -p "$BASE/build-linux" + cd "$BASE/linux" + make O=../build-linux ARCH=hexagon \ + KBUILD_CFLAGS_KERNEL="-mlong-calls" \ + CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \ + LD=${TOOLCHAIN_BIN}/ld.lld \ + KBUILD_VERBOSE=1 comet_defconfig + make mrproper + + cd "$BASE/build-linux" + make \ + ARCH=hexagon \ + CC=${TOOLCHAIN_BIN}/clang \ + INSTALL_HDR_PATH=${HEX_TOOLS_TARGET_BASE} \ + V=1 \ + headers_install +} + +build_musl() { + cd "$BASE/musl" + make clean + CROSS_COMPILE=hexagon-unknown-linux-musl- \ + AR=llvm-ar \ + RANLIB=llvm-ranlib \ + STRIP=llvm-strip \ + CC=clang \ + LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \ + CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \ + ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE} + PATH=${TOOLCHAIN_BIN}/:$PATH make CROSS_COMPILE= install + cd ${HEX_TOOLS_TARGET_BASE}/lib + ln -sf libc.so ld-musl-hexagon.so + ln -sf ld-musl-hexagon.so ld-musl-hexagon.so.1 + cdp ${HEX_TOOLS_TARGET_BASE}/../lib + ln -sf ../usr/lib/ld-musl-hexagon.so.1 +} + +build_llvm_clang +build_kernel_headers +build_musl_headers +build_clang_rt +build_musl diff --git a/tests/docker/dockerfiles/debian-hppa-cross.docker b/tests/docker/dockerfiles/debian-hppa-cross.docker new file mode 100644 index 000000000..3d6c65a3e --- /dev/null +++ b/tests/docker/dockerfiles/debian-hppa-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-hppa-linux-gnu \ + libc6-dev-hppa-cross diff --git a/tests/docker/dockerfiles/debian-m68k-cross.docker b/tests/docker/dockerfiles/debian-m68k-cross.docker new file mode 100644 index 000000000..fcb10e353 --- /dev/null +++ b/tests/docker/dockerfiles/debian-m68k-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-m68k-linux-gnu \ + libc6-dev-m68k-cross diff --git a/tests/docker/dockerfiles/debian-microblaze-cross.d/build-toolchain.sh b/tests/docker/dockerfiles/debian-microblaze-cross.d/build-toolchain.sh new file mode 100755 index 000000000..23ec0aa9a --- /dev/null +++ b/tests/docker/dockerfiles/debian-microblaze-cross.d/build-toolchain.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -e + +TARGET=microblaze-linux-musl +LINUX_ARCH=microblaze + +J=$(expr $(nproc) / 2) +TOOLCHAIN_INSTALL=/usr/local +TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin +CROSS_SYSROOT=${TOOLCHAIN_INSTALL}/$TARGET/sys-root + +export PATH=${TOOLCHAIN_BIN}:$PATH + +# +# Grab all of the source for the toolchain bootstrap. +# + +wget https://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.xz +wget https://ftp.gnu.org/gnu/gcc/gcc-11.2.0/gcc-11.2.0.tar.xz +wget https://www.musl-libc.org/releases/musl-1.2.2.tar.gz +wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.70.tar.xz + +tar axf binutils-2.37.tar.xz +tar axf gcc-11.2.0.tar.xz +tar axf musl-1.2.2.tar.gz +tar axf linux-5.10.70.tar.xz + +mv binutils-2.37 src-binu +mv gcc-11.2.0 src-gcc +mv musl-1.2.2 src-musl +mv linux-5.10.70 src-linux + +mkdir -p bld-hdr bld-binu bld-gcc bld-musl +mkdir -p ${CROSS_SYSROOT}/usr/include + +# +# Install kernel headers +# + +cd src-linux +make headers_install ARCH=${LINUX_ARCH} INSTALL_HDR_PATH=${CROSS_SYSROOT}/usr +cd .. + +# +# Build binutils +# + +cd bld-binu +../src-binu/configure --disable-werror \ + --prefix=${TOOLCHAIN_INSTALL} --with-sysroot --target=${TARGET} +make -j${J} +make install +cd .. + +# +# Build gcc, just the compiler so far. +# + +cd bld-gcc +../src-gcc/configure --disable-werror --disable-shared \ + --prefix=${TOOLCHAIN_INSTALL} --with-sysroot --target=${TARGET} \ + --enable-languages=c --disable-libssp --disable-libsanitizer \ + --disable-libatomic --disable-libgomp --disable-libquadmath +make -j${J} all-gcc +make install-gcc +cd .. + +# +# Build musl. +# We won't go through the extra step of building shared libraries +# because we don't actually use them in QEMU docker testing. +# + +cd bld-musl +../src-musl/configure --prefix=/usr --host=${TARGET} --disable-shared +make -j${j} +make install DESTDIR=${CROSS_SYSROOT} +cd .. + +# +# Go back and build the compiler runtime +# + +cd bld-gcc +make -j${j} +make install +cd .. diff --git a/tests/docker/dockerfiles/debian-mips-cross.docker b/tests/docker/dockerfiles/debian-mips-cross.docker new file mode 100644 index 000000000..26c154014 --- /dev/null +++ b/tests/docker/dockerfiles/debian-mips-cross.docker @@ -0,0 +1,32 @@ +# +# Docker mips cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture mips +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-mips-linux-gnu + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a mips --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=mips-linux-gnu- +ENV DEF_TARGET_LIST mips-softmmu,mipsel-linux-user + +# Install extra libraries to increase code coverage +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:mips \ + liblzo2-dev:mips \ + librdmacm-dev:mips \ + libsnappy-dev:mips diff --git a/tests/docker/dockerfiles/debian-mips64-cross.docker b/tests/docker/dockerfiles/debian-mips64-cross.docker new file mode 100644 index 000000000..09c2ba584 --- /dev/null +++ b/tests/docker/dockerfiles/debian-mips64-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-mips64-linux-gnuabi64 \ + libc6-dev-mips64-cross diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker new file mode 100644 index 000000000..c990b683b --- /dev/null +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -0,0 +1,33 @@ +# +# Docker mips64el cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# + +FROM qemu/debian10 + +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture mips64el && \ + apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-mips64el-linux-gnuabi64 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a mips64el --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=mips64el-linux-gnuabi64- +ENV DEF_TARGET_LIST mips64el-softmmu,mips64el-linux-user + +# Install extra libraries to increase code coverage +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:mips64el \ + liblzo2-dev:mips64el \ + librdmacm-dev:mips64el \ + libsnappy-dev:mips64el diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker new file mode 100644 index 000000000..0e5dd42d3 --- /dev/null +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -0,0 +1,31 @@ +# +# Docker mipsel cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# +FROM qemu/debian10 + +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture mipsel +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-mipsel-linux-gnu + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a mipsel --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=mipsel-linux-gnu- + +# Install extra libraries to increase code coverage +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:mipsel \ + liblzo2-dev:mipsel \ + librdmacm-dev:mipsel \ + libsnappy-dev:mipsel diff --git a/tests/docker/dockerfiles/debian-native.docker b/tests/docker/dockerfiles/debian-native.docker new file mode 100644 index 000000000..efd55cb6e --- /dev/null +++ b/tests/docker/dockerfiles/debian-native.docker @@ -0,0 +1,49 @@ +# +# Docker Debian Native +# +# This this intended to build QEMU on native host systems. Debian is +# chosen due to the broadest range on supported host systems for QEMU. +# +# This docker target is based on the docker.io Debian Bullseye base +# image rather than QEMU's base because we would otherwise confuse the +# build grabbing stuff from the registry built for other +# architectures. +# +FROM docker.io/library/debian:bullseye-slim +MAINTAINER Alex Bennée <alex.bennee@linaro.org> + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy --arch-only qemu + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + cscope \ + genisoimage \ + exuberant-ctags \ + global \ + libbz2-dev \ + liblzo2-dev \ + libgcrypt20-dev \ + libfdt-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsnappy-dev \ + libvte-dev \ + netcat-openbsd \ + ninja-build \ + openssh-client \ + python3-numpy \ + python3-opencv \ + python3-venv + +ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS +ENV DEF_TARGET_LIST "none" diff --git a/tests/docker/dockerfiles/debian-nios2-cross.d/build-toolchain.sh b/tests/docker/dockerfiles/debian-nios2-cross.d/build-toolchain.sh new file mode 100755 index 000000000..ba3c9d8af --- /dev/null +++ b/tests/docker/dockerfiles/debian-nios2-cross.d/build-toolchain.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -e + +TARGET=nios2-linux-gnu +LINUX_ARCH=nios2 + +J=$(expr $(nproc) / 2) +TOOLCHAIN_INSTALL=/usr/local +TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin +CROSS_SYSROOT=${TOOLCHAIN_INSTALL}/$TARGET/sys-root + +export PATH=${TOOLCHAIN_BIN}:$PATH + +# +# Grab all of the source for the toolchain bootstrap. +# + +wget https://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.xz +wget https://ftp.gnu.org/gnu/gcc/gcc-11.2.0/gcc-11.2.0.tar.xz +wget https://ftp.gnu.org/gnu/glibc/glibc-2.34.tar.xz +wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.70.tar.xz + +tar axf binutils-2.37.tar.xz +tar axf gcc-11.2.0.tar.xz +tar axf glibc-2.34.tar.xz +tar axf linux-5.10.70.tar.xz + +mv binutils-2.37 src-binu +mv gcc-11.2.0 src-gcc +mv glibc-2.34 src-glibc +mv linux-5.10.70 src-linux + +mkdir -p bld-hdr bld-binu bld-gcc bld-glibc +mkdir -p ${CROSS_SYSROOT}/usr/include + +# +# Install kernel and glibc headers +# + +cd src-linux +make headers_install ARCH=${LINUX_ARCH} INSTALL_HDR_PATH=${CROSS_SYSROOT}/usr +cd .. + +cd bld-hdr +../src-glibc/configure --prefix=/usr --host=${TARGET} +make install-headers DESTDIR=${CROSS_SYSROOT} +touch ${CROSS_SYSROOT}/usr/include/gnu/stubs.h +cd .. + +# +# Build binutils +# + +cd bld-binu +../src-binu/configure --disable-werror \ + --prefix=${TOOLCHAIN_INSTALL} --with-sysroot --target=${TARGET} +make -j${J} +make install +cd .. + +# +# Build gcc, without shared libraries, because we do not yet +# have a shared libc against which to link. +# + +cd bld-gcc +../src-gcc/configure --disable-werror --disable-shared \ + --prefix=${TOOLCHAIN_INSTALL} --with-sysroot --target=${TARGET} \ + --enable-languages=c --disable-libssp --disable-libsanitizer \ + --disable-libatomic --disable-libgomp --disable-libquadmath +make -j${J} +make install +cd .. + +# +# Build glibc +# There are a few random things that use c++ but we didn't build that +# cross-compiler. We can get away without them. Disable CXX so that +# glibc doesn't try to use the host c++ compiler. +# + +cd bld-glibc +CXX=false ../src-glibc/configure --prefix=/usr --host=${TARGET} +make -j${j} +make install DESTDIR=${CROSS_SYSROOT} +cd .. diff --git a/tests/docker/dockerfiles/debian-powerpc-test-cross.docker b/tests/docker/dockerfiles/debian-powerpc-test-cross.docker new file mode 100644 index 000000000..36b336f70 --- /dev/null +++ b/tests/docker/dockerfiles/debian-powerpc-test-cross.docker @@ -0,0 +1,17 @@ +# +# Docker powerpc/ppc64/ppc64le cross-compiler target +# +# This docker target builds on the debian Bullseye base image. +# +FROM qemu/debian11 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-powerpc-linux-gnu \ + libc6-dev-powerpc-cross \ + gcc-10-powerpc64-linux-gnu \ + libc6-dev-ppc64-cross \ + gcc-10-powerpc64le-linux-gnu \ + libc6-dev-ppc64el-cross + diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker new file mode 100644 index 000000000..1146a06be --- /dev/null +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -0,0 +1,28 @@ +# +# Docker ppc64el cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# +FROM qemu/debian10 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture ppc64el && \ + apt update && \ + apt install -yy crossbuild-essential-ppc64el + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a ppc64el --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=powerpc64le-linux-gnu- +ENV DEF_TARGET_LIST ppc64-softmmu,ppc64-linux-user,ppc64abi32-linux-user + +# Install extra libraries to increase code coverage +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:ppc64el \ + liblzo2-dev:ppc64el \ + librdmacm-dev:ppc64el \ + libsnappy-dev:ppc64el diff --git a/tests/docker/dockerfiles/debian-riscv64-cross.docker b/tests/docker/dockerfiles/debian-riscv64-cross.docker new file mode 100644 index 000000000..594d97982 --- /dev/null +++ b/tests/docker/dockerfiles/debian-riscv64-cross.docker @@ -0,0 +1,48 @@ +# +# Docker cross-compiler target for riscv64 +# +# Currently the only distro that gets close to cross compiling riscv64 +# images is Debian Sid (with unofficial ports). As this is a moving +# target we keep the library list minimal and are aiming to migrate +# from this hack as soon as we are able. +# +FROM docker.io/library/debian:sid-slim + +# Add ports +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata apt update -yy && \ + DEBIAN_FRONTEND=noninteractive eatmydata apt upgrade -yy + +# Install common build utilities +RUN DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy \ + bc \ + build-essential \ + ca-certificates \ + debian-ports-archive-keyring \ + dpkg-dev \ + gettext \ + git \ + ninja-build \ + pkg-config \ + python3 + +# Add ports and riscv64 architecture +RUN echo "deb http://ftp.ports.debian.org/debian-ports/ sid main" >> /etc/apt/sources.list +RUN dpkg --add-architecture riscv64 + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-riscv64-linux-gnu \ + libc6-dev-riscv64-cross \ + libffi-dev:riscv64 \ + libglib2.0-dev:riscv64 \ + libpixman-1-dev:riscv64 + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=riscv64-linux-gnu- +ENV DEF_TARGET_LIST riscv64-softmmu,riscv64-linux-user diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker new file mode 100644 index 000000000..9f2ab51eb --- /dev/null +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -0,0 +1,33 @@ +# +# Docker s390 cross-compiler target +# +# This docker target builds on the debian Stretch base image. +# +FROM qemu/debian10 + +# Add the s390x architecture +RUN dpkg --add-architecture s390x + +# Grab the updated list of packages +RUN apt update && apt dist-upgrade -yy +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-multilib-s390x-linux-gnu + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy -a s390x --arch-only qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=s390x-linux-gnu- +ENV DEF_TARGET_LIST s390x-softmmu,s390x-linux-user + +# Install extra libraries to increase code coverage +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + libbz2-dev:s390x \ + liblzo2-dev:s390x \ + librdmacm-dev:s390x \ + libsnappy-dev:s390x diff --git a/tests/docker/dockerfiles/debian-sh4-cross.docker b/tests/docker/dockerfiles/debian-sh4-cross.docker new file mode 100644 index 000000000..fd3af8957 --- /dev/null +++ b/tests/docker/dockerfiles/debian-sh4-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-sh4-linux-gnu \ + libc6-dev-sh4-cross diff --git a/tests/docker/dockerfiles/debian-sparc64-cross.docker b/tests/docker/dockerfiles/debian-sparc64-cross.docker new file mode 100644 index 000000000..f4bb9b561 --- /dev/null +++ b/tests/docker/dockerfiles/debian-sparc64-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian Buster base image. +# +FROM qemu/debian10 + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-sparc64-linux-gnu \ + libc6-dev-sparc64-cross diff --git a/tests/docker/dockerfiles/debian-toolchain.docker b/tests/docker/dockerfiles/debian-toolchain.docker new file mode 100644 index 000000000..738d808aa --- /dev/null +++ b/tests/docker/dockerfiles/debian-toolchain.docker @@ -0,0 +1,36 @@ +# +# Docker toolchain cross-compiler +# +# This dockerfile is used for building a cross-compiler toolchain. +# The script for building the toolchain is supplied via extra-files. +# +FROM qemu/debian10 + +# Install build utilities for building gcc and glibc. +# ??? The build-dep isn't working, missing a number of +# minimal build dependiencies, e.g. libmpc. + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bison \ + flex \ + gawk \ + libmpc-dev \ + libmpfr-dev \ + rsync \ + texinfo \ + wget && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy --arch-only gcc glibc + +ADD build-toolchain.sh /root/build-toolchain.sh + +RUN cd /root && ./build-toolchain.sh + +# Throw away the extra toolchain build deps, the downloaded source, +# and the build trees by restoring the original debian10 image, +# then copying the built toolchain from stage 0. +FROM qemu/debian10 +COPY --from=0 /usr/local /usr/local diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker new file mode 100644 index 000000000..d8df2c611 --- /dev/null +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -0,0 +1,47 @@ +# +# Docker TriCore cross-compiler target +# +# This docker target builds on the Debian Buster base image but +# doesn't inherit from the common one to avoid bringing in unneeded +# dependencies. +# +# Copyright (c) 2018 Philippe Mathieu-Daudé +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +FROM docker.io/library/debian:buster-slim + +MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy \ + bzip2 \ + ca-certificates \ + ccache \ + g++ \ + gcc \ + git \ + libglib2.0-dev \ + libpixman-1-dev \ + libtest-harness-perl \ + locales \ + make \ + ninja-build \ + perl-base \ + pkgconf \ + python3-pip \ + python3-setuptools \ + python3-wheel + +RUN git clone --single-branch \ + https://github.com/bkoppelmann/tricore-binutils.git \ + /usr/src/binutils && \ + cd /usr/src/binutils && chmod +x missing && \ + CFLAGS=-w ./configure --prefix=/usr/local --disable-nls --target=tricore && \ + make && make install && \ + rm -rf /usr/src/binutils + +# This image can only build a very minimal QEMU as well as the tests +ENV DEF_TARGET_LIST tricore-softmmu +ENV QEMU_CONFIGURE_OPTS --disable-user --disable-tools --disable-fdt diff --git a/tests/docker/dockerfiles/debian-xtensa-cross.docker b/tests/docker/dockerfiles/debian-xtensa-cross.docker new file mode 100644 index 000000000..2f11b3b7b --- /dev/null +++ b/tests/docker/dockerfiles/debian-xtensa-cross.docker @@ -0,0 +1,29 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian stretch base image, +# using a prebuilt toolchains for Xtensa cores from: +# https://github.com/foss-xtensa/toolchain/releases +# +FROM docker.io/library/debian:stretch-slim + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + gettext \ + git \ + python3-minimal + +ENV CPU_LIST dc232b dc233c de233_fpu dsp3400 +ENV TOOLCHAIN_RELEASE 2020.07 + +RUN for cpu in $CPU_LIST; do \ + curl -#SL http://github.com/foss-xtensa/toolchain/releases/download/$TOOLCHAIN_RELEASE/x86_64-$TOOLCHAIN_RELEASE-xtensa-$cpu-elf.tar.gz \ + | tar -xzC /opt; \ + done + +ENV PATH $PATH:/opt/$TOOLCHAIN_RELEASE/xtensa-dc232b-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-dc233c-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-de233_fpu-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-dsp3400-elf/bin diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker new file mode 100644 index 000000000..b414af1b9 --- /dev/null +++ b/tests/docker/dockerfiles/debian10.docker @@ -0,0 +1,37 @@ +# +# Docker multiarch cross-compiler target +# +# This docker target is builds on Debian cross compiler targets to build distro +# with a selection of cross compilers for building test binaries. +# +# On its own you can't build much but the docker-foo-cross targets +# build on top of the base debian image. +# +FROM docker.io/library/debian:buster-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bc \ + build-essential \ + ca-certificates \ + ccache \ + clang \ + dbus \ + gdb-multiarch \ + gettext \ + git \ + libffi-dev \ + libncurses5-dev \ + ninja-build \ + pkg-config \ + psmisc \ + python3 \ + python3-sphinx \ + python3-sphinx-rtd-theme \ + $(apt-get -s build-dep --arch-only qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) diff --git a/tests/docker/dockerfiles/debian11.docker b/tests/docker/dockerfiles/debian11.docker new file mode 100644 index 000000000..febf884f8 --- /dev/null +++ b/tests/docker/dockerfiles/debian11.docker @@ -0,0 +1,18 @@ +# +# Docker multiarch cross-compiler target +# +# This docker target uses the current development version of Debian as +# a base for cross compilers for building test binaries. We won't +# attempt to build QEMU on it yet given it is still in development. +# +# On its own you can't build much but the docker-foo-cross targets +# build on top of the base debian image. +# +FROM docker.io/library/debian:bullseye-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata diff --git a/tests/docker/dockerfiles/empty.docker b/tests/docker/dockerfiles/empty.docker new file mode 100644 index 000000000..9ba980f1a --- /dev/null +++ b/tests/docker/dockerfiles/empty.docker @@ -0,0 +1,8 @@ +# +# Empty Dockerfile +# + +FROM scratch + +# Add everything from the context into the container +ADD . / diff --git a/tests/docker/dockerfiles/fedora-cris-cross.docker b/tests/docker/dockerfiles/fedora-cris-cross.docker new file mode 100644 index 000000000..91c373fdd --- /dev/null +++ b/tests/docker/dockerfiles/fedora-cris-cross.docker @@ -0,0 +1,8 @@ +# +# Cross compiler for cris system tests +# + +FROM registry.fedoraproject.org/fedora:33 +ENV PACKAGES gcc-cris-linux-gnu +RUN dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/fedora-i386-cross.docker b/tests/docker/dockerfiles/fedora-i386-cross.docker new file mode 100644 index 000000000..13328e608 --- /dev/null +++ b/tests/docker/dockerfiles/fedora-i386-cross.docker @@ -0,0 +1,32 @@ +FROM registry.fedoraproject.org/fedora:34 + +ENV PACKAGES \ + bzip2 \ + ccache \ + diffutils \ + findutils \ + gcc \ + git \ + libffi-devel.i686 \ + libselinux-devel.i686 \ + libtasn1-devel.i686 \ + libzstd-devel.i686 \ + make \ + meson \ + ninja-build \ + glib2-devel.i686 \ + glibc-devel.i686 \ + glibc-static.i686 \ + gnutls-devel.i686 \ + nettle-devel.i686 \ + pcre-devel.i686 \ + perl-Test-Harness \ + pixman-devel.i686 \ + sysprof-capture-devel.i686 \ + zlib-devel.i686 + +ENV QEMU_CONFIGURE_OPTS --cpu=i386 --disable-vhost-user +ENV PKG_CONFIG_LIBDIR /usr/lib/pkgconfig + +RUN dnf update -y && dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker new file mode 100644 index 000000000..aad39dd97 --- /dev/null +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -0,0 +1,43 @@ +FROM registry.fedoraproject.org/fedora:33 + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + bc \ + bzip2 \ + ccache \ + diffutils \ + findutils \ + gcc \ + gettext \ + git \ + hostname \ + make \ + meson \ + mingw32-bzip2 \ + mingw32-curl \ + mingw32-glib2 \ + mingw32-gmp \ + mingw32-gnutls \ + mingw32-gtk3 \ + mingw32-libffi \ + mingw32-libjpeg-turbo \ + mingw32-libpng \ + mingw32-libtasn1 \ + mingw32-libusbx \ + mingw32-nettle \ + mingw32-nsis \ + mingw32-pixman \ + mingw32-pkg-config \ + mingw32-SDL2 \ + perl \ + perl-Test-Harness \ + python3 \ + python3-PyYAML \ + tar \ + which + +RUN dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=i686-w64-mingw32- diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker new file mode 100644 index 000000000..9a224a619 --- /dev/null +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -0,0 +1,40 @@ +FROM registry.fedoraproject.org/fedora:33 + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + bc \ + bzip2 \ + ccache \ + diffutils \ + findutils \ + gcc \ + gettext \ + git \ + hostname \ + make \ + meson \ + mingw32-nsis \ + mingw64-bzip2 \ + mingw64-curl \ + mingw64-glib2 \ + mingw64-gmp \ + mingw64-gtk3 \ + mingw64-libffi \ + mingw64-libjpeg-turbo \ + mingw64-libpng \ + mingw64-libtasn1 \ + mingw64-libusbx \ + mingw64-pixman \ + mingw64-pkg-config \ + perl \ + perl-Test-Harness \ + python3 \ + python3-PyYAML \ + tar \ + which + +RUN dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=x86_64-w64-mingw32- --disable-capstone diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker new file mode 100644 index 000000000..c6fd7e111 --- /dev/null +++ b/tests/docker/dockerfiles/fedora.docker @@ -0,0 +1,118 @@ +FROM registry.fedoraproject.org/fedora:33 + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + SDL2-devel \ + SDL2_image-devel \ + alsa-lib-devel \ + bc \ + brlapi-devel \ + bzip2 \ + bzip2-devel \ + ca-certificates \ + capstone-devel \ + ccache \ + clang \ + ctags \ + cyrus-sasl-devel \ + daxctl-devel \ + dbus-daemon \ + device-mapper-multipath-devel \ + diffutils \ + findutils \ + gcc \ + gcc-c++ \ + gcovr \ + genisoimage \ + gettext \ + git \ + glib2-devel \ + glibc-langpack-en \ + glibc-static \ + glusterfs-api-devel \ + gnutls-devel \ + gtk3-devel \ + hostname \ + jemalloc-devel \ + libaio-devel \ + libasan \ + libattr-devel \ + libbpf-devel \ + libcacard-devel \ + libcap-ng-devel \ + libcurl-devel \ + libdrm-devel \ + libepoxy-devel \ + libfdt-devel \ + libffi-devel \ + libgcrypt-devel \ + libiscsi-devel \ + libjpeg-devel \ + libnfs-devel \ + libpmem-devel \ + libpng-devel \ + librbd-devel \ + libseccomp-devel \ + libselinux-devel \ + libslirp-devel \ + libssh-devel \ + libtasn1-devel \ + libubsan \ + libudev-devel \ + liburing-devel \ + libusbx-devel \ + libxml2-devel \ + libzstd-devel \ + llvm \ + lttng-ust-devel \ + lzo-devel \ + make \ + mesa-libgbm-devel \ + meson \ + ncurses-devel \ + nettle-devel \ + ninja-build \ + nmap-ncat \ + numactl-devel \ + openssh-clients \ + pam-devel \ + perl-Test-Harness \ + perl-base \ + pixman-devel \ + pkgconfig \ + pulseaudio-libs-devel \ + python3 \ + python3-PyYAML \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-sphinx \ + python3-sphinx_rtd_theme \ + python3-virtualenv \ + rdma-core-devel \ + rpm \ + sed \ + snappy-devel \ + sparse \ + spice-protocol \ + spice-server-devel \ + systemd-devel \ + systemtap-sdt-devel \ + tar \ + tesseract \ + tesseract-langpack-eng \ + texinfo \ + usbredir-devel \ + util-linux \ + virglrenderer-devel \ + vte291-devel \ + which \ + xen-devel \ + xfsprogs-devel \ + zlib-devel +ENV QEMU_CONFIGURE_OPTS --python=/usr/bin/python3 + +RUN dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt +ENV PATH $PATH:/usr/libexec/python3-sphinx/ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker new file mode 100644 index 000000000..3bbdb67f4 --- /dev/null +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -0,0 +1,114 @@ +FROM registry.opensuse.org/opensuse/leap:15.2 + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + Mesa-devel \ + alsa-lib-devel \ + bc \ + brlapi-devel \ + bzip2 \ + ca-certificates \ + ccache \ + clang \ + ctags \ + cyrus-sasl-devel \ + dbus-1 \ + diffutils \ + findutils \ + gcc \ + gcc-c++ \ + gcovr \ + gettext-runtime \ + git \ + glib2-devel \ + glibc-locale \ + glibc-static \ + glusterfs-devel \ + gtk3-devel \ + hostname \ + jemalloc-devel \ + libSDL2-devel \ + libSDL2_image-devel \ + libaio-devel \ + libasan6 \ + libattr-devel \ + libbpf-devel \ + libbz2-devel \ + libcacard-devel \ + libcap-ng-devel \ + libcurl-devel \ + libdrm-devel \ + libepoxy-devel \ + libfdt-devel \ + libffi-devel \ + libgcrypt-devel \ + libgnutls-devel \ + libiscsi-devel \ + libjpeg8-devel \ + libndctl-devel \ + libnettle-devel \ + libnfs-devel \ + libnuma-devel \ + libpixman-1-0-devel \ + libpmem-devel \ + libpng16-devel \ + libpulse-devel \ + librbd-devel \ + libseccomp-devel \ + libselinux-devel \ + libspice-server-devel \ + libssh-devel \ + libtasn1-devel \ + libubsan1 \ + libudev-devel \ + libusb-1_0-devel \ + libxml2-devel \ + libzstd-devel \ + llvm \ + lttng-ust-devel \ + lzo-devel \ + make \ + mkisofs \ + ncat \ + ncurses-devel \ + ninja \ + openssh \ + pam-devel \ + perl-Test-Harness \ + perl-base \ + pkgconfig \ + python3-Pillow \ + python3-PyYAML \ + python3-Sphinx \ + python3-base \ + python3-numpy \ + python3-opencv \ + python3-pip \ + python3-setuptools \ + python3-sphinx_rtd_theme \ + python3-virtualenv \ + python3-wheel \ + rdma-core-devel \ + rpm \ + sed \ + snappy-devel \ + sparse \ + spice-protocol-devel \ + systemd-devel \ + systemtap-sdt-devel \ + tar \ + tesseract-ocr \ + tesseract-ocr-traineddata-english \ + texinfo \ + usbredir-devel \ + util-linux \ + virglrenderer-devel \ + vte-devel \ + which \ + xen-devel \ + xfsprogs-devel \ + zlib-devel +ENV QEMU_CONFIGURE_OPTS --python=/usr/bin/python3.6 + +RUN zypper update -y && zypper --non-interactive install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/python.docker b/tests/docker/dockerfiles/python.docker new file mode 100644 index 000000000..56d88417d --- /dev/null +++ b/tests/docker/dockerfiles/python.docker @@ -0,0 +1,18 @@ +# Python library testing environment + +FROM fedora:latest +MAINTAINER John Snow <jsnow@redhat.com> + +# Please keep this list sorted alphabetically +ENV PACKAGES \ + gcc \ + make \ + pipenv \ + python3 \ + python3-pip \ + python3-tox \ + python3-virtualenv \ + python3.10 + +RUN dnf install -y $PACKAGES +RUN rpm -q $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker new file mode 100644 index 000000000..f0e0180d2 --- /dev/null +++ b/tests/docker/dockerfiles/ubuntu.docker @@ -0,0 +1,71 @@ +# +# Latest Ubuntu Release +# +# Useful for testing against relatively bleeding edge libraries and +# compilers. We also have seperate recipe for the most recent LTS +# release. +# +# When updating use the full tag not :latest otherwise the build +# system won't pick up that it has changed. +# + +FROM docker.io/library/ubuntu:20.04 +ENV PACKAGES \ + ccache \ + clang \ + dbus \ + gcc \ + gettext \ + git \ + glusterfs-common \ + libaio-dev \ + libattr1-dev \ + libbrlapi-dev \ + libbz2-dev \ + libcacard-dev \ + libcap-ng-dev \ + libcurl4-gnutls-dev \ + libdrm-dev \ + libepoxy-dev \ + libfdt-dev \ + libffi-dev \ + libgbm-dev \ + libgnutls28-dev \ + libgtk-3-dev \ + libibverbs-dev \ + libiscsi-dev \ + libjemalloc-dev \ + libjpeg-turbo8-dev \ + liblzo2-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libnfs-dev \ + libnuma-dev \ + libpixman-1-dev \ + libpng-dev \ + librados-dev \ + librbd-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsdl2-dev \ + libseccomp-dev \ + libsnappy-dev \ + libspice-protocol-dev \ + libspice-server-dev \ + libssh-dev \ + libusb-1.0-0-dev \ + libusbredirhost-dev \ + libvdeplug-dev \ + libvte-2.91-dev \ + libxen-dev \ + libzstd-dev \ + make \ + ninja-build \ + python3-yaml \ + python3-sphinx \ + python3-sphinx-rtd-theme \ + sparse \ + xfslibs-dev +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES +RUN dpkg -l $PACKAGES | sort > /packages.txt diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker new file mode 100644 index 000000000..450fd06d0 --- /dev/null +++ b/tests/docker/dockerfiles/ubuntu1804.docker @@ -0,0 +1,117 @@ +FROM docker.io/library/ubuntu:18.04 +ENV PACKAGES \ + bc \ + bsdmainutils \ + bzip2 \ + ca-certificates \ + ccache \ + clang \ + dbus \ + debianutils \ + diffutils \ + exuberant-ctags \ + findutils \ + g++ \ + gcc \ + gcovr \ + genisoimage \ + gettext \ + git \ + glusterfs-common \ + hostname \ + libaio-dev \ + libasan5 \ + libasound2-dev \ + libattr1-dev \ + libbrlapi-dev \ + libbz2-dev \ + libc6-dev \ + libcacard-dev \ + libcap-ng-dev \ + libcapstone-dev \ + libcurl4-gnutls-dev \ + libdaxctl-dev \ + libdrm-dev \ + libepoxy-dev \ + libfdt-dev \ + libffi-dev \ + libgbm-dev \ + libgcrypt20-dev \ + libglib2.0-dev \ + libgnutls28-dev \ + libgtk-3-dev \ + libibverbs-dev \ + libiscsi-dev \ + libjemalloc-dev \ + libjpeg-turbo8-dev \ + liblttng-ust-dev \ + liblzo2-dev \ + libncursesw5-dev \ + libnfs-dev \ + libnuma-dev \ + libpam0g-dev \ + libpixman-1-dev \ + libpmem-dev \ + libpng-dev \ + libpulse-dev \ + librbd-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsdl2-dev \ + libsdl2-image-dev \ + libseccomp-dev \ + libselinux-dev \ + libsnappy-dev \ + libspice-protocol-dev \ + libspice-server-dev \ + libssh-dev \ + libsystemd-dev \ + libtasn1-6-dev \ + libtest-harness-perl \ + libubsan1 \ + libudev-dev \ + libusb-1.0-0-dev \ + libusbredirhost-dev \ + libvdeplug-dev \ + libvirglrenderer-dev \ + libvte-2.91-dev \ + libxen-dev \ + libxml2-dev \ + libzstd-dev \ + llvm \ + locales \ + make \ + multipath-tools \ + netcat-openbsd \ + nettle-dev \ + ninja-build \ + openssh-client \ + perl-base \ + pkgconf \ + python3 \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-setuptools \ + python3-sphinx \ + python3-sphinx-rtd-theme \ + python3-venv \ + python3-wheel \ + python3-yaml \ + rpm2cpio \ + sed \ + sparse \ + systemtap-sdt-dev \ + tar \ + tesseract-ocr \ + tesseract-ocr-eng \ + texinfo \ + xfslibs-dev \ + zlib1g-dev +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES +RUN dpkg -l $PACKAGES | sort > /packages.txt + +# https://bugs.launchpad.net/qemu/+bug/1838763 +ENV QEMU_CONFIGURE_OPTS --disable-libssh diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker new file mode 100644 index 000000000..15a026be0 --- /dev/null +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -0,0 +1,119 @@ +FROM docker.io/library/ubuntu:20.04 +ENV PACKAGES \ + bc \ + bsdmainutils \ + bzip2 \ + ca-certificates \ + ccache \ + clang \ + dbus \ + debianutils \ + diffutils \ + exuberant-ctags \ + findutils \ + g++ \ + gcc \ + gcovr \ + genisoimage \ + gettext \ + git \ + hostname \ + libaio-dev \ + libasan5 \ + libasound2-dev \ + libattr1-dev \ + libbrlapi-dev \ + libbz2-dev \ + libc6-dev \ + libcacard-dev \ + libcap-ng-dev \ + libcapstone-dev \ + libcurl4-gnutls-dev \ + libdaxctl-dev \ + libdrm-dev \ + libepoxy-dev \ + libfdt-dev \ + libffi-dev \ + libgbm-dev \ + libgcrypt20-dev \ + libglib2.0-dev \ + libglusterfs-dev \ + libgnutls28-dev \ + libgtk-3-dev \ + libibverbs-dev \ + libiscsi-dev \ + libjemalloc-dev \ + libjpeg-turbo8-dev \ + liblttng-ust-dev \ + liblzo2-dev \ + libncursesw5-dev \ + libnfs-dev \ + libnuma-dev \ + libpam0g-dev \ + libpixman-1-dev \ + libpmem-dev \ + libpng-dev \ + libpulse-dev \ + librbd-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsdl2-dev \ + libsdl2-image-dev \ + libseccomp-dev \ + libselinux-dev \ + libslirp-dev \ + libsnappy-dev \ + libspice-protocol-dev \ + libspice-server-dev \ + libssh-dev \ + libsystemd-dev \ + libtasn1-6-dev \ + libtest-harness-perl \ + libubsan1 \ + libudev-dev \ + libusb-1.0-0-dev \ + libusbredirhost-dev \ + libvdeplug-dev \ + libvirglrenderer-dev \ + libvte-2.91-dev \ + libxen-dev \ + libxml2-dev \ + libzstd-dev \ + llvm \ + locales \ + make \ + multipath-tools \ + ncat \ + nettle-dev \ + ninja-build \ + openssh-client \ + perl-base \ + pkgconf \ + python3 \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-setuptools \ + python3-sphinx \ + python3-sphinx-rtd-theme \ + python3-venv \ + python3-wheel \ + python3-yaml \ + rpm2cpio \ + sed \ + sparse \ + systemtap-sdt-dev \ + tar \ + tesseract-ocr \ + tesseract-ocr-eng \ + texinfo \ + xfslibs-dev \ + zlib1g-dev +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES +RUN dpkg -l $PACKAGES | sort > /packages.txt + +# Apply patch https://reviews.llvm.org/D75820 +# This is required for TSan in clang-10 to compile with QEMU. +RUN sed -i 's/^const/static const/g' /usr/lib/llvm-10/lib/clang/10.0.0/include/sanitizer/tsan_interface.h diff --git a/tests/docker/run b/tests/docker/run new file mode 100755 index 000000000..421393046 --- /dev/null +++ b/tests/docker/run @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Docker test runner +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +if test -n "$V"; then + set -x +fi + +BASE="$(dirname $(readlink -e $0))" + +# Prepare the environment +export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH + +if test -n "$J"; then + export MAKEFLAGS="$MAKEFLAGS -j$J" +fi + +# We are in the container so the whole file system belong to us +export TEST_DIR=/tmp/qemu-test +mkdir -p $TEST_DIR/{src,build,install} + +# Extract the source tarballs +tar -C $TEST_DIR/src -xf $BASE/qemu.tar || { echo "Failed to untar source"; exit 2; } + +if test -n "$SHOW_ENV"; then + if test -f /packages.txt; then + echo "Packages installed:" + cat /packages.txt + echo + fi + echo "Environment variables:" + env + echo +fi + +export QEMU_SRC="$TEST_DIR/src" +export BUILD_DIR="$TEST_DIR/build" +export INSTALL_DIR="$TEST_DIR/install" + +cd "$QEMU_SRC/tests/docker" + +CMD="./$@" + +if test -z "$DEBUG"; then + exec $CMD +fi + +# DEBUG workflow +echo "* Prepared to run command:" +echo " $CMD" +echo "* Hit Ctrl-D to continue, or type 'exit 1' to abort" +echo +env bash --noprofile --norc + +if "$CMD"; then + exit 0 +elif test -n "$DEBUG"; then + echo "* Command failed:" + echo " $CMD" + echo "* Hit Ctrl-D to exit" + echo + # Force error after shell exits + env bash --noprofile --norc && exit 1 +else + exit 1 +fi diff --git a/tests/docker/test-block b/tests/docker/test-block new file mode 100755 index 000000000..5624b8182 --- /dev/null +++ b/tests/docker/test-block @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Run block test cases +# +# Copyright 2017 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This code is licensed under the GPL version 2 or later. See +# the COPYING file in the top-level directory. + +. ./common.rc + +cd "$BUILD_DIR" + +build_qemu --target-list=x86_64-softmmu || test_fail "Build failed" +cd tests/qemu-iotests +for t in raw qcow2 nbd luks; do + ./check -g quick -$t || test_fail "Test failed: iotests $t" +done diff --git a/tests/docker/test-build b/tests/docker/test-build new file mode 100755 index 000000000..2b2a7832f --- /dev/null +++ b/tests/docker/test-build @@ -0,0 +1,20 @@ +#!/bin/bash -e +# +# Quick compile test without the make check step of test-quick. +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +build_qemu +install_qemu diff --git a/tests/docker/test-clang b/tests/docker/test-clang new file mode 100755 index 000000000..b57e0119d --- /dev/null +++ b/tests/docker/test-clang @@ -0,0 +1,25 @@ +#!/bin/bash -e +# +# Compile and check with clang. +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +requires_binary clang + +cd "$BUILD_DIR" + +OPTS="--cxx=clang++ --cc=clang --host-cc=clang" +OPTS="$OPTS --extra-cflags=-fsanitize=undefined \ + --extra-cflags=-fno-sanitize=float-divide-by-zero" +build_qemu $OPTS +check_qemu +install_qemu diff --git a/tests/docker/test-debug b/tests/docker/test-debug new file mode 100755 index 000000000..f52f16328 --- /dev/null +++ b/tests/docker/test-debug @@ -0,0 +1,27 @@ +#!/bin/bash -e +# +# Compile and check with clang & --enable-debug --enable-sanitizers. +# +# Copyright (c) 2016-2018 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# Marc-André Lureau <marcandre.lureau@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +requires_binary clang + +cd "$BUILD_DIR" + +OPTS="--cxx=clang++ --cc=clang --host-cc=clang" +OPTS="--enable-debug --enable-sanitizers $OPTS" + +export ASAN_OPTIONS=detect_leaks=0 +build_qemu $OPTS +check_qemu check V=1 +install_qemu diff --git a/tests/docker/test-full b/tests/docker/test-full new file mode 100755 index 000000000..aadc0f00a --- /dev/null +++ b/tests/docker/test-full @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Compile all the targets. +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +build_qemu && check_qemu && install_qemu diff --git a/tests/docker/test-mingw b/tests/docker/test-mingw new file mode 100755 index 000000000..0bc6d7887 --- /dev/null +++ b/tests/docker/test-mingw @@ -0,0 +1,36 @@ +#!/bin/bash -e +# +# Cross compile QEMU with mingw toolchain on Linux. +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +requires_binary x86_64-w64-mingw32-gcc +requires_binary i686-w64-mingw32-gcc + +cd "$BUILD_DIR" + +for prefix in x86_64-w64-mingw32- i686-w64-mingw32-; do + TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ + build_qemu --cross-prefix=$prefix \ + --enable-trace-backends=simple \ + --enable-gnutls \ + --enable-nettle \ + --enable-curl \ + --enable-vnc \ + --enable-bzip2 \ + --enable-guest-agent \ + --enable-docs + install_qemu + make installer + make clean + +done diff --git a/tests/docker/test-misc b/tests/docker/test-misc new file mode 100755 index 000000000..2a3c2c2e1 --- /dev/null +++ b/tests/docker/test-misc @@ -0,0 +1,24 @@ +#!/bin/bash -e +# +# Build the miscellaneous components +# +# Copyright (c) 2019 Linaro Ltd. +# +# Authors: +# Alex Bennée <alex.bennee@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +. common.rc + +requires_binary sphinx-build-3 sphinx-build + +cd "$BUILD_DIR" + +# build everything else but QEMU +configure_qemu --disable-user --disable-system --enable-docs --enable-tools +build_qemu +install_qemu diff --git a/tests/docker/test-quick b/tests/docker/test-quick new file mode 100755 index 000000000..feee44b24 --- /dev/null +++ b/tests/docker/test-quick @@ -0,0 +1,21 @@ +#!/bin/bash -e +# +# Quick compiling test that everyone already does. But why not automate it? +# +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +build_qemu +check_qemu +install_qemu diff --git a/tests/docker/test-static b/tests/docker/test-static new file mode 100755 index 000000000..372ef6fac --- /dev/null +++ b/tests/docker/test-static @@ -0,0 +1,24 @@ +#!/bin/bash -e +# +# Compile QEMU user mode emulators as static binaries on Linux. +# +# Copyright (c) 2020 Red Hat Inc. +# +# Authors: +# Paolo Bonzini <pbonzini@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +build_qemu \ + --disable-system \ + --disable-tools \ + --disable-guest-agent \ + --disable-docs \ + --static +install_qemu diff --git a/tests/docker/test-tcg b/tests/docker/test-tcg new file mode 100755 index 000000000..00993b73b --- /dev/null +++ b/tests/docker/test-tcg @@ -0,0 +1,22 @@ +#!/bin/bash -e +# +# Build and run the TCG tests +# +# Copyright (c) 2021 Linaro Ltd. +# +# Authors: +# Alex Bennée <alex.bennee@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +# although we are not building QEMU itself we still need a configured +# build for the unit tests to be built and run +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +build_qemu +check_qemu check-tcg diff --git a/tests/docker/test-tsan b/tests/docker/test-tsan new file mode 100755 index 000000000..53d90d2f7 --- /dev/null +++ b/tests/docker/test-tsan @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# This test will use TSan as part of a build and a make check. +# +# Copyright (c) 2020 Linaro +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Robert Foley <robert.foley@linaro.org> +# Originally based on test-quick from Fam Zheng <famz@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +setup_tsan() +{ + requires_binary clang + tsan_log_dir="/tmp/qemu-test/build/tsan" + mkdir -p $tsan_log_dir > /dev/null || true + EXTRA_CONFIGURE_OPTS="${EXTRA_CONFIGURE_OPTS} --enable-tsan \ + --cc=clang-10 --cxx=clang++-10 \ + --disable-werror --extra-cflags=-O0" + # detect deadlocks is false currently simply because + # TSan crashes immediately with deadlock detector enabled. + # We have maxed out the history size to get the best chance of finding + # warnings during testing. + # Note, to get TSan to fail on warning, use exitcode=66 below. + tsan_opts="suppressions=/tmp/qemu-test/src/tests/tsan/suppressions.tsan\ + detect_deadlocks=false history_size=7\ + halt_on_error=0 exitcode=0 verbose=5\ + log_path=$tsan_log_dir/tsan_warning" + export TSAN_OPTIONS="$tsan_opts" +} + +cd "$BUILD_DIR" + +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +setup_tsan +build_qemu +check_qemu +install_qemu diff --git a/tests/docker/test-unit b/tests/docker/test-unit new file mode 100755 index 000000000..8905d0115 --- /dev/null +++ b/tests/docker/test-unit @@ -0,0 +1,21 @@ +#!/bin/bash -e +# +# Build and run the unit tests +# +# Copyright (c) 2018 Linaro Ltd. +# +# Authors: +# Alex Bennée <alex.bennee@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +cd "$BUILD_DIR" + +# although we are not building QEMU itself we still need a configured +# build for the unit tests to be built and run +configure_qemu +check_qemu check-unit |