diff options
Diffstat (limited to 'meson/ci')
-rw-r--r-- | meson/ci/azure-steps.yml | 23 | ||||
-rw-r--r-- | meson/ci/ciimage/.gitignore | 3 | ||||
-rw-r--r-- | meson/ci/ciimage/arch/image.json | 7 | ||||
-rwxr-xr-x | meson/ci/ciimage/arch/install.sh | 54 | ||||
-rw-r--r-- | meson/ci/ciimage/bionic/image.json | 8 | ||||
-rwxr-xr-x | meson/ci/ciimage/bionic/install.sh | 60 | ||||
-rwxr-xr-x | meson/ci/ciimage/build.py | 241 | ||||
-rw-r--r-- | meson/ci/ciimage/common.sh | 47 | ||||
-rw-r--r-- | meson/ci/ciimage/cuda/image.json | 8 | ||||
-rwxr-xr-x | meson/ci/ciimage/cuda/install.sh | 21 | ||||
-rw-r--r-- | meson/ci/ciimage/fedora/image.json | 8 | ||||
-rwxr-xr-x | meson/ci/ciimage/fedora/install.sh | 29 | ||||
-rw-r--r-- | meson/ci/ciimage/opensuse/image.json | 9 | ||||
-rwxr-xr-x | meson/ci/ciimage/opensuse/install.sh | 48 | ||||
-rw-r--r-- | meson/ci/ciimage/ubuntu-rolling/image.json | 8 | ||||
-rwxr-xr-x | meson/ci/ciimage/ubuntu-rolling/install.sh | 54 | ||||
-rwxr-xr-x | meson/ci/ciimage/ubuntu-rolling/test.sh | 12 | ||||
-rw-r--r-- | meson/ci/run.ps1 | 106 | ||||
-rwxr-xr-x | meson/ci/upload_cov.sh | 13 | ||||
-rw-r--r-- | meson/ci/usercustomize.py | 19 |
20 files changed, 778 insertions, 0 deletions
diff --git a/meson/ci/azure-steps.yml b/meson/ci/azure-steps.yml new file mode 100644 index 000000000..233bbfa36 --- /dev/null +++ b/meson/ci/azure-steps.yml @@ -0,0 +1,23 @@ +steps: +- task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: .\ci\run.ps1 + +- task: PublishTestResults@2 + inputs: + testResultsFiles: meson-test-run.xml + testRunTitle: $(System.JobName) + publishRunAttachments: true + condition: not(canceled()) + +- task: CopyFiles@2 + inputs: + contents: 'meson-test-run.*' + targetFolder: $(Build.ArtifactStagingDirectory) + condition: not(canceled()) + +- task: PublishBuildArtifacts@1 + inputs: + artifactName: $(System.JobName) + condition: not(canceled()) diff --git a/meson/ci/ciimage/.gitignore b/meson/ci/ciimage/.gitignore new file mode 100644 index 000000000..cff1864bf --- /dev/null +++ b/meson/ci/ciimage/.gitignore @@ -0,0 +1,3 @@ +/build_* +/test_* +/user.sh diff --git a/meson/ci/ciimage/arch/image.json b/meson/ci/ciimage/arch/image.json new file mode 100644 index 000000000..4a399d69e --- /dev/null +++ b/meson/ci/ciimage/arch/image.json @@ -0,0 +1,7 @@ +{ + "base_image": "archlinux:latest", + "env": { + "CI": "1", + "MESON_CI_JOBNAME": "linux-arch-gcc" + } +} diff --git a/meson/ci/ciimage/arch/install.sh b/meson/ci/ciimage/arch/install.sh new file mode 100755 index 000000000..72816ab95 --- /dev/null +++ b/meson/ci/ciimage/arch/install.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +# Inspired by https://github.com/greyltc/docker-archlinux-aur/blob/master/add-aur.sh + +pkgs=( + python python-pip + ninja make git sudo fakeroot autoconf automake patch + libelf gcc gcc-fortran gcc-objc vala rust bison flex cython go dlang-dmd + mono boost qt5-base gtkmm3 gtest gmock protobuf wxgtk2 gobject-introspection + itstool gtk3 java-environment=8 gtk-doc llvm clang sdl2 graphviz + doxygen vulkan-validation-layers openssh mercurial gtk-sharp-2 qt5-tools + libwmf valgrind cmake netcdf-fortran openmpi nasm gnustep-base gettext + python-lxml hotdoc rust-bindgen qt6-base qt6-tools + # cuda +) + +aur_pkgs=(scalapack) +cleanup_pkgs=(go) + +AUR_USER=docker +PACMAN_OPTS='--needed --noprogressbar --noconfirm' + +# Patch config files +sed -i 's/#Color/Color/g' /etc/pacman.conf +sed -i 's,#MAKEFLAGS="-j2",MAKEFLAGS="-j$(nproc)",g' /etc/makepkg.conf +sed -i "s,PKGEXT='.pkg.tar.zst',PKGEXT='.pkg.tar',g" /etc/makepkg.conf + +# Install packages +pacman -Syu $PACMAN_OPTS "${pkgs[@]}" +install_python_packages + +# Setup the user +useradd -m $AUR_USER +echo "${AUR_USER}:" | chpasswd -e +echo "$AUR_USER ALL = NOPASSWD: ALL" >> /etc/sudoers + +# Install yay +su $AUR_USER -c 'cd; git clone https://aur.archlinux.org/yay.git' +su $AUR_USER -c 'cd; cd yay; makepkg' +pushd /home/$AUR_USER/yay/ +pacman -U *.pkg.tar --noprogressbar --noconfirm +popd +rm -rf /home/$AUR_USER/yay + +# Install yay deps +su $AUR_USER -c "yay -S $PACMAN_OPTS ${aur_pkgs[*]}" + +# cleanup +pacman -Rs --noconfirm "${cleanup_pkgs[@]}" +su $AUR_USER -c "yes | yay -Scc" diff --git a/meson/ci/ciimage/bionic/image.json b/meson/ci/ciimage/bionic/image.json new file mode 100644 index 000000000..5df709e7e --- /dev/null +++ b/meson/ci/ciimage/bionic/image.json @@ -0,0 +1,8 @@ +{ + "base_image": "ubuntu:bionic", + "env": { + "CI": "1", + "DC": "gdc", + "MESON_CI_JOBNAME": "linux-bionic-gcc" + } +} diff --git a/meson/ci/ciimage/bionic/install.sh b/meson/ci/ciimage/bionic/install.sh new file mode 100755 index 000000000..4dea73ba8 --- /dev/null +++ b/meson/ci/ciimage/bionic/install.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +export DEBIAN_FRONTEND=noninteractive +export LANG='C.UTF-8' +export DC=gdc + +pkgs=( + python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev + wget unzip cmake doxygen + clang + pkg-config-arm-linux-gnueabihf + qt4-linguist-tools qt5-default qtbase5-private-dev + python-dev + libomp-dev + llvm lcov + ldc + libclang-dev + libgcrypt20-dev + libgpgme-dev + libhdf5-dev openssh-server + libboost-python-dev libboost-regex-dev + libblocksruntime-dev + libperl-dev libscalapack-mpi-dev libncurses-dev +) + +boost_pkgs=(atomic chrono date-time filesystem log regex serialization system test thread) + +sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" +apt-get -y update +apt-get -y upgrade +apt-get -y install eatmydata + +# Base stuff +eatmydata apt-get -y build-dep meson + +# Add boost packages +for i in "${boost_pkgs[@]}"; do + for j in "1.62.0" "1.65.1"; do + pkgs+=("libboost-${i}${j}") + done +done + +# packages +eatmydata apt-get -y install "${pkgs[@]}" + +install_python_packages + +# Install the ninja 0.10 +wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip +unzip ninja-linux.zip -d /ci + +# cleanup +apt-get -y remove ninja-build +apt-get -y clean +apt-get -y autoclean +rm ninja-linux.zip diff --git a/meson/ci/ciimage/build.py b/meson/ci/ciimage/build.py new file mode 100755 index 000000000..1e1f23811 --- /dev/null +++ b/meson/ci/ciimage/build.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +import json +import argparse +import stat +import textwrap +import shutil +import subprocess +from tempfile import TemporaryDirectory +from pathlib import Path +import typing as T + +image_namespace = 'mesonbuild' + +image_def_file = 'image.json' +install_script = 'install.sh' + +class ImageDef: + def __init__(self, image_dir: Path) -> None: + path = image_dir / image_def_file + data = json.loads(path.read_text(encoding='utf-8')) + + assert isinstance(data, dict) + assert all([x in data for x in ['base_image', 'env']]) + assert isinstance(data['base_image'], str) + assert isinstance(data['env'], dict) + + self.base_image: str = data['base_image'] + self.args: T.List[str] = data.get('args', []) + self.env: T.Dict[str, str] = data['env'] + +class BuilderBase(): + def __init__(self, data_dir: Path, temp_dir: Path) -> None: + self.data_dir = data_dir + self.temp_dir = temp_dir + + self.common_sh = self.data_dir.parent / 'common.sh' + self.common_sh = self.common_sh.resolve(strict=True) + self.validate_data_dir() + + self.image_def = ImageDef(self.data_dir) + + self.docker = shutil.which('docker') + self.git = shutil.which('git') + if self.docker is None: + raise RuntimeError('Unable to find docker') + if self.git is None: + raise RuntimeError('Unable to find git') + + def validate_data_dir(self) -> None: + files = [ + self.data_dir / image_def_file, + self.data_dir / install_script, + ] + if not self.data_dir.exists(): + raise RuntimeError(f'{self.data_dir.as_posix()} does not exist') + for i in files: + if not i.exists(): + raise RuntimeError(f'{i.as_posix()} does not exist') + if not i.is_file(): + raise RuntimeError(f'{i.as_posix()} is not a regular file') + +class Builder(BuilderBase): + def gen_bashrc(self) -> None: + out_file = self.temp_dir / 'env_vars.sh' + out_data = '' + + # run_tests.py parameters + self.image_def.env['CI_ARGS'] = ' '.join(self.image_def.args) + + for key, val in self.image_def.env.items(): + out_data += f'export {key}="{val}"\n' + + # Also add /ci to PATH + out_data += 'export PATH="/ci:$PATH"\n' + + out_file.write_text(out_data, encoding='utf-8') + + # make it executable + mode = out_file.stat().st_mode + out_file.chmod(mode | stat.S_IEXEC) + + def gen_dockerfile(self) -> None: + out_file = self.temp_dir / 'Dockerfile' + out_data = textwrap.dedent(f'''\ + FROM {self.image_def.base_image} + + ADD install.sh /ci/install.sh + ADD common.sh /ci/common.sh + ADD env_vars.sh /ci/env_vars.sh + RUN /ci/install.sh + ''') + + out_file.write_text(out_data, encoding='utf-8') + + def do_build(self) -> None: + # copy files + for i in self.data_dir.iterdir(): + shutil.copy(str(i), str(self.temp_dir)) + shutil.copy(str(self.common_sh), str(self.temp_dir)) + + self.gen_bashrc() + self.gen_dockerfile() + + cmd_git = [self.git, 'rev-parse', '--short', 'HEAD'] + res = subprocess.run(cmd_git, cwd=self.data_dir, stdout=subprocess.PIPE) + if res.returncode != 0: + raise RuntimeError('Failed to get the current commit hash') + commit_hash = res.stdout.decode().strip() + + cmd = [ + self.docker, 'build', + '-t', f'{image_namespace}/{self.data_dir.name}:latest', + '-t', f'{image_namespace}/{self.data_dir.name}:{commit_hash}', + '--pull', + self.temp_dir.as_posix(), + ] + if subprocess.run(cmd).returncode != 0: + raise RuntimeError('Failed to build the docker image') + +class ImageTester(BuilderBase): + def __init__(self, data_dir: Path, temp_dir: Path, ci_root: Path) -> None: + super().__init__(data_dir, temp_dir) + self.meson_root = ci_root.parent.parent.resolve() + + def gen_dockerfile(self) -> None: + out_file = self.temp_dir / 'Dockerfile' + out_data = textwrap.dedent(f'''\ + FROM {image_namespace}/{self.data_dir.name} + + ADD meson /meson + ''') + + out_file.write_text(out_data, encoding='utf-8') + + def copy_meson(self) -> None: + shutil.copytree( + self.meson_root, + self.temp_dir / 'meson', + ignore=shutil.ignore_patterns( + '.git', + '*_cache', + '__pycache__', + # 'work area', + self.temp_dir.name, + ) + ) + + def do_test(self, tty: bool = False) -> None: + self.copy_meson() + self.gen_dockerfile() + + try: + build_cmd = [ + self.docker, 'build', + '-t', 'meson_test_image', + self.temp_dir.as_posix(), + ] + if subprocess.run(build_cmd).returncode != 0: + raise RuntimeError('Failed to build the test docker image') + + test_cmd = [] + if tty: + test_cmd = [ + self.docker, 'run', '--rm', '-t', '-i', 'meson_test_image', + '/bin/bash', '-c', '' + + 'cd meson;' + + 'source /ci/env_vars.sh;' + + f'echo -e "\\n\\nInteractive test shell in the {image_namespace}/{self.data_dir.name} container with the current meson tree";' + + 'echo -e "The file ci/ciimage/user.sh will be sourced if it exists to enable user specific configurations";' + + 'echo -e "Run the following command to run all CI tests: ./run_tests.py $CI_ARGS\\n\\n";' + + '[ -f ci/ciimage/user.sh ] && exec /bin/bash --init-file ci/ciimage/user.sh;' + + 'exec /bin/bash;' + ] + else: + test_cmd = [ + self.docker, 'run', '--rm', '-t', 'meson_test_image', + '/bin/bash', '-c', 'source /ci/env_vars.sh; cd meson; ./run_tests.py $CI_ARGS' + ] + + if subprocess.run(test_cmd).returncode != 0 and not tty: + raise RuntimeError('Running tests failed') + finally: + cleanup_cmd = [self.docker, 'rmi', '-f', 'meson_test_image'] + subprocess.run(cleanup_cmd).returncode + +class ImageTTY(BuilderBase): + def __init__(self, data_dir: Path, temp_dir: Path, ci_root: Path) -> None: + super().__init__(data_dir, temp_dir) + self.meson_root = ci_root.parent.parent.resolve() + + def do_run(self) -> None: + try: + tty_cmd = [ + self.docker, 'run', + '--name', 'meson_test_container', '-t', '-i', '-v', f'{self.meson_root.as_posix()}:/meson', + f'{image_namespace}/{self.data_dir.name}', + '/bin/bash', '-c', '' + + 'cd meson;' + + 'source /ci/env_vars.sh;' + + f'echo -e "\\n\\nInteractive test shell in the {image_namespace}/{self.data_dir.name} container with the current meson tree";' + + 'echo -e "The file ci/ciimage/user.sh will be sourced if it exists to enable user specific configurations";' + + 'echo -e "Run the following command to run all CI tests: ./run_tests.py $CI_ARGS\\n\\n";' + + '[ -f ci/ciimage/user.sh ] && exec /bin/bash --init-file ci/ciimage/user.sh;' + + 'exec /bin/bash;' + ] + subprocess.run(tty_cmd).returncode != 0 + finally: + cleanup_cmd = [self.docker, 'rm', '-f', 'meson_test_container'] + subprocess.run(cleanup_cmd).returncode + + +def main() -> None: + parser = argparse.ArgumentParser(description='Meson CI image builder') + parser.add_argument('what', type=str, help='Which image to build / test') + parser.add_argument('-t', '--type', choices=['build', 'test', 'testTTY', 'TTY'], help='What to do', required=True) + + args = parser.parse_args() + + ci_root = Path(__file__).parent + ci_data = ci_root / args.what + + with TemporaryDirectory(prefix=f'{args.type}_{args.what}_', dir=ci_root) as td: + ci_build = Path(td) + print(f'Build dir: {ci_build}') + + if args.type == 'build': + builder = Builder(ci_data, ci_build) + builder.do_build() + elif args.type == 'test': + tester = ImageTester(ci_data, ci_build, ci_root) + tester.do_test() + elif args.type == 'testTTY': + tester = ImageTester(ci_data, ci_build, ci_root) + tester.do_test(tty=True) + elif args.type == 'TTY': + tester = ImageTTY(ci_data, ci_build, ci_root) + tester.do_run() + +if __name__ == '__main__': + main() diff --git a/meson/ci/ciimage/common.sh b/meson/ci/ciimage/common.sh new file mode 100644 index 000000000..707b7515f --- /dev/null +++ b/meson/ci/ciimage/common.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +### +### Common functions for CI builder files. +### All functions can be accessed in install.sh via: +### +### $ source /ci/common.sh +### + +set -e +set -x + +base_python_pkgs=( + pytest + pytest-xdist + coverage + codecov + jsonschema +) + +python_pkgs=( + cython + gobject + PyGObject + lxml + gcovr +) + +dub_fetch() { + set +e + for (( i=1; i<=24; ++i )); do + dub fetch "$@" + (( $? == 0 )) && break + + echo "Dub Fetch failed. Retrying in $((i*5))s" + sleep $((i*5)) + done + set -e +} + +install_minimal_python_packages() { + python3 -m pip install "${base_python_pkgs[@]}" $* +} + +install_python_packages() { + python3 -m pip install "${base_python_pkgs[@]}" "${python_pkgs[@]}" $* +} diff --git a/meson/ci/ciimage/cuda/image.json b/meson/ci/ciimage/cuda/image.json new file mode 100644 index 000000000..f422723c1 --- /dev/null +++ b/meson/ci/ciimage/cuda/image.json @@ -0,0 +1,8 @@ +{ + "base_image": "archlinux:latest", + "args": ["--only", "cuda"], + "env": { + "CI": "1", + "MESON_CI_JOBNAME": "linux-cuda-gcc" + } +} diff --git a/meson/ci/ciimage/cuda/install.sh b/meson/ci/ciimage/cuda/install.sh new file mode 100755 index 000000000..0d412e00c --- /dev/null +++ b/meson/ci/ciimage/cuda/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +pkgs=( + python python-pip + ninja gcc gcc-objc git cmake + cuda zlib pkgconf +) + +PACMAN_OPTS='--needed --noprogressbar --noconfirm' + +pacman -Syu $PACMAN_OPTS "${pkgs[@]}" +install_minimal_python_packages + +# Manually remove cache to avoid GitHub space restrictions +rm -rf /var/cache/pacman + +echo "source /etc/profile.d/cuda.sh" >> /ci/env_vars.sh diff --git a/meson/ci/ciimage/fedora/image.json b/meson/ci/ciimage/fedora/image.json new file mode 100644 index 000000000..c6fdc9e28 --- /dev/null +++ b/meson/ci/ciimage/fedora/image.json @@ -0,0 +1,8 @@ +{ + "base_image": "fedora:latest", + "env": { + "CI": "1", + "SKIP_STATIC_BOOST": "1", + "MESON_CI_JOBNAME": "linux-fedora-gcc" + } +} diff --git a/meson/ci/ciimage/fedora/install.sh b/meson/ci/ciimage/fedora/install.sh new file mode 100755 index 000000000..df1d853cd --- /dev/null +++ b/meson/ci/ciimage/fedora/install.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +pkgs=( + python python-pip python3-devel + ninja-build make git autoconf automake patch + elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-objc++ vala rust bison flex ldc libasan libasan-static + mono-core boost-devel gtkmm30 gtest-devel gmock-devel protobuf-devel wxGTK3-devel gobject-introspection + boost-python3-devel + itstool gtk3-devel java-latest-openjdk-devel gtk-doc llvm-devel clang-devel SDL2-devel graphviz-devel zlib zlib-devel zlib-static + #hdf5-openmpi-devel hdf5-devel netcdf-openmpi-devel netcdf-devel netcdf-fortran-openmpi-devel netcdf-fortran-devel scalapack-openmpi-devel + doxygen vulkan-devel vulkan-validation-layers-devel openssh mercurial gtk-sharp2-devel libpcap-devel gpgme-devel + qt5-qtbase-devel qt5-qttools-devel qt5-linguist qt5-qtbase-private-devel + libwmf-devel valgrind cmake openmpi-devel nasm gnustep-base-devel gettext-devel ncurses-devel + libxml2-devel libxslt-devel libyaml-devel glib2-devel json-glib-devel libgcrypt-devel +) + +# Sys update +dnf -y upgrade + +# Install deps +dnf -y install "${pkgs[@]}" +install_python_packages hotdoc + +# Cleanup +dnf -y clean all diff --git a/meson/ci/ciimage/opensuse/image.json b/meson/ci/ciimage/opensuse/image.json new file mode 100644 index 000000000..6609aa08d --- /dev/null +++ b/meson/ci/ciimage/opensuse/image.json @@ -0,0 +1,9 @@ +{ + "base_image": "opensuse/tumbleweed:latest", + "env": { + "CI": "1", + "SKIP_STATIC_BOOST": "1", + "SINGLE_DUB_COMPILER": "1", + "MESON_CI_JOBNAME": "linux-opensuse-gcc" + } +} diff --git a/meson/ci/ciimage/opensuse/install.sh b/meson/ci/ciimage/opensuse/install.sh new file mode 100755 index 000000000..41cb96192 --- /dev/null +++ b/meson/ci/ciimage/opensuse/install.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +pkgs=( + python3-pip python3 python3-devel + ninja make git autoconf automake patch libjpeg-devel + elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-obj-c++ vala rust bison flex curl lcov + mono-core gtkmm3-devel gtest gmock protobuf-devel wxGTK3-3_2-devel gobject-introspection-devel + itstool gtk3-devel java-15-openjdk-devel gtk-doc llvm-devel clang-devel libSDL2-devel graphviz-devel zlib-devel zlib-devel-static + #hdf5-devel netcdf-devel libscalapack2-openmpi3-devel libscalapack2-gnu-openmpi3-hpc-devel openmpi3-devel + doxygen vulkan-devel vulkan-validationlayers openssh mercurial gtk-sharp3-complete gtk-sharp2-complete libpcap-devel libgpgme-devel + libqt5-qtbase-devel libqt5-qttools-devel libqt5-linguist libqt5-qtbase-private-headers-devel + libwmf-devel valgrind cmake nasm gnustep-base-devel gettext-tools gettext-runtime gettext-csharp ncurses-devel + libxml2-devel libxslt-devel libyaml-devel glib2-devel json-glib-devel + boost-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_system-devel + libboost_test-devel libboost_log-devel libboost_regex-devel + libboost_python3-devel libboost_regex-devel +) + +# Sys update +zypper --non-interactive patch --with-update --with-optional +zypper --non-interactive update + +# Install deps +zypper install -y "${pkgs[@]}" +install_python_packages hotdoc + +echo 'export PKG_CONFIG_PATH="/usr/lib64/mpi/gcc/openmpi3/lib64/pkgconfig:$PKG_CONFIG_PATH"' >> /ci/env_vars.sh + +# dmd is very special on OpenSUSE (as in the packages do not work) +# see https://bugzilla.opensuse.org/show_bug.cgi?id=1162408 +curl -fsS https://dlang.org/install.sh | bash -s dmd | tee dmd_out.txt +cat dmd_out.txt | grep source | sed 's/^[^`]*`//g' | sed 's/`.*//g' >> /ci/env_vars.sh +chmod +x /ci/env_vars.sh + +source /ci/env_vars.sh + +dub_fetch urld +dub build urld --compiler=dmd +dub_fetch dubtestproject +dub build dubtestproject:test1 --compiler=dmd +dub build dubtestproject:test2 --compiler=dmd + +# Cleanup +zypper --non-interactive clean --all diff --git a/meson/ci/ciimage/ubuntu-rolling/image.json b/meson/ci/ciimage/ubuntu-rolling/image.json new file mode 100644 index 000000000..f9f068d5a --- /dev/null +++ b/meson/ci/ciimage/ubuntu-rolling/image.json @@ -0,0 +1,8 @@ +{ + "base_image": "ubuntu:rolling", + "env": { + "CI": "1", + "DC": "gdc", + "MESON_CI_JOBNAME": "linux-ubuntu-rolling-gcc" + } +} diff --git a/meson/ci/ciimage/ubuntu-rolling/install.sh b/meson/ci/ciimage/ubuntu-rolling/install.sh new file mode 100755 index 000000000..770fd8531 --- /dev/null +++ b/meson/ci/ciimage/ubuntu-rolling/install.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +export DEBIAN_FRONTEND=noninteractive +export LANG='C.UTF-8' +export DC=gdc + +pkgs=( + python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev + wget unzip + qt5-qmake qtbase5-dev qtchooser qtbase5-dev-tools clang + pkg-config-arm-linux-gnueabihf + libomp-dev + llvm lcov + dub ldc + mingw-w64 mingw-w64-tools nim + libclang-dev + libgcrypt20-dev + libgpgme-dev + libhdf5-dev + libboost-python-dev libboost-regex-dev + libblocksruntime-dev + libperl-dev + liblapack-dev libscalapack-mpi-dev + bindgen +) + +sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" +apt-get -y update +apt-get -y upgrade +apt-get -y install eatmydata + +# Base stuff +eatmydata apt-get -y build-dep meson + +# packages +eatmydata apt-get -y install "${pkgs[@]}" +eatmydata apt-get -y install --no-install-recommends wine-stable # Wine is special + +install_python_packages hotdoc + +# dub stuff +dub_fetch urld +dub build urld --compiler=gdc +dub_fetch dubtestproject +dub build dubtestproject:test1 --compiler=ldc2 +dub build dubtestproject:test2 --compiler=ldc2 + +# cleanup +apt-get -y clean +apt-get -y autoclean diff --git a/meson/ci/ciimage/ubuntu-rolling/test.sh b/meson/ci/ciimage/ubuntu-rolling/test.sh new file mode 100755 index 000000000..f6956bb60 --- /dev/null +++ b/meson/ci/ciimage/ubuntu-rolling/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +testFN() { + set +e + false +} + +testFN +false +exit 0 diff --git a/meson/ci/run.ps1 b/meson/ci/run.ps1 new file mode 100644 index 000000000..5f256858d --- /dev/null +++ b/meson/ci/run.ps1 @@ -0,0 +1,106 @@ +python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin +if ($LastExitCode -ne 0) { + exit 0 +} + +# remove Chocolately, MinGW, Strawberry Perl from path, so we don't find gcc/gfortran and try to use it +# remove PostgreSQL from path so we don't pickup a broken zlib from it +$env:Path = ($env:Path.Split(';') | Where-Object { $_ -notmatch 'mingw|Strawberry|Chocolatey|PostgreSQL' }) -join ';' + +if ($env:arch -eq 'x64') { + # Rust puts its shared stdlib in a secret place, but it is needed to run tests. + $env:Path += ";$HOME/.rustup/toolchains/stable-x86_64-pc-windows-msvc/bin" +} elseif ($env:arch -eq 'x86') { + # Switch to the x86 Rust toolchain + rustup default stable-i686-pc-windows-msvc + # Rust puts its shared stdlib in a secret place, but it is needed to run tests. + $env:Path += ";$HOME/.rustup/toolchains/stable-i686-pc-windows-msvc/bin" + # Need 32-bit Python for tests that need the Python dependency + $env:Path = "C:\hostedtoolcache\windows\Python\3.6.8\x86;C:\hostedtoolcache\windows\Python\3.6.8\x86\Scripts;$env:Path" +} + +# Set the CI env var for the meson test framework +$env:CI = '1' + +# download and install prerequisites +function DownloadFile([String] $Source, [String] $Destination) { + $retries = 10 + echo "Downloading $Source" + for ($i = 1; $i -le $retries; $i++) { + try { + (New-Object net.webclient).DownloadFile($Source, $Destination) + break # succeeded + } catch [net.WebException] { + if ($i -eq $retries) { + throw # fail on last retry + } + $backoff = (10 * $i) # backoff 10s, 20s, 30s... + echo ('{0}: {1}' -f $Source, $_.Exception.Message) + echo ('Retrying in {0}s...' -f $backoff) + Start-Sleep -m ($backoff * 1000) + } + } +} + + +if (($env:backend -eq 'ninja') -and ($env:arch -ne 'arm64')) { $dmd = $true } else { $dmd = $false } + +DownloadFile -Source https://github.com/mesonbuild/cidata/releases/download/ci3/ci_data.zip -Destination $env:AGENT_WORKFOLDER\ci_data.zip +echo "Extracting ci_data.zip" +Expand-Archive $env:AGENT_WORKFOLDER\ci_data.zip -DestinationPath $env:AGENT_WORKFOLDER\ci_data +& "$env:AGENT_WORKFOLDER\ci_data\install.ps1" -Arch $env:arch -Compiler $env:compiler -Boost $true -DMD $dmd + + +echo "=== PATH BEGIN ===" +echo ($env:Path).Replace(';',"`n") +echo "=== PATH END ===" +echo "" + +$progs = @("python","ninja","pkg-config","cl","rc","link") +foreach ($prog in $progs) { + echo "" + echo "Locating ${prog}:" + where.exe $prog +} + +echo "" +echo "Ninja / MSBuld version:" +if ($env:backend -eq 'ninja') { + ninja --version +} else { + MSBuild /version +} + +echo "" +echo "Python version:" +python --version + +# Needed for running unit tests in parallel. +echo "" +python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist jsonschema coverage + +echo "" +echo "=== Start running tests ===" +# Starting from VS2019 Powershell(?) will fail the test run +# if it prints anything to stderr. Python's test runner +# does that by default so we need to forward it. +cmd /c "python 2>&1 ./tools/run_with_cov.py run_tests.py --backend $env:backend $env:extraargs" + +$result = $LastExitCode + +echo "" +echo "" +echo "=== Gathering coverage report ===" +echo "" + +python3 -m coverage combine +python3 -m coverage xml +python3 -m coverage report + +# Currently codecov.py does not handle Azure, use this fork of a fork to get it +# working without requireing a token +git clone https://github.com/mensinda/codecov-python +python3 -m pip install --ignore-installed ./codecov-python +python3 -m codecov -f .coverage/coverage.xml -n "VS$env:compiler $env:arch $env:backend" -c $env:SOURCE_VERSION + +exit $result diff --git a/meson/ci/upload_cov.sh b/meson/ci/upload_cov.sh new file mode 100755 index 000000000..089641b47 --- /dev/null +++ b/meson/ci/upload_cov.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "Combining coverage reports..." +coverage combine + +echo "Generating XML report..." +coverage xml + +echo "Printing report" +coverage report + +echo "Uploading to codecov..." +codecov -f .coverage/coverage.xml -n "$1" diff --git a/meson/ci/usercustomize.py b/meson/ci/usercustomize.py new file mode 100644 index 000000000..72421ba0d --- /dev/null +++ b/meson/ci/usercustomize.py @@ -0,0 +1,19 @@ +# Copyright 2021 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used by coverage (see tools/run_with_cov.py) to enable coverage +# reports in python subprocesses + +import coverage +coverage.process_startup() |