From 2ccb65481c04e9c03f99abb8b93a135016e96dad Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 08:37:28 +0100 Subject: lava-master-base: 2019.03 needs python3-junit.xml LAVA 2019.03 needs python3-junit.xml from stretch-backports. --- lava-master-base/99-stretch-backports | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lava-master-base/99-stretch-backports b/lava-master-base/99-stretch-backports index cfd4832..02491b1 100644 --- a/lava-master-base/99-stretch-backports +++ b/lava-master-base/99-stretch-backports @@ -37,3 +37,7 @@ Pin-Priority: 500 Package: lavacli Pin: release n=stretch-backports Pin-Priority: 500 + +Package: python3-junit.xml +Pin: release n=stretch-backports +Pin-Priority: 500 -- cgit 1.2.3-korg From 2ef40504a0d5dac643b643271f21a6a07adb3e8e Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 09:52:31 +0100 Subject: lava-xxx-base: permit to install 2019.03 LAVA has changed its repository location. Furthermore, it is now accessible via https which need additionnal package to use it. --- lava-master-base/Dockerfile | 10 ++++++---- lava-slave-base/Dockerfile | 12 +++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lava-master-base/Dockerfile b/lava-master-base/Dockerfile index 03b634f..fc7471a 100644 --- a/lava-master-base/Dockerfile +++ b/lava-master-base/Dockerfile @@ -20,16 +20,18 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ sudo \ wget \ e2fsprogs \ + letsencrypt \ + apt-transport-https \ gnupg \ vim -RUN wget http://images.validation.linaro.org/production-repo/production-repo.key.asc \ - && apt-key add production-repo.key.asc \ - && echo 'deb http://images.validation.linaro.org/production-repo/ stretch-backports main' > /etc/apt/sources.list.d/lava.list \ +RUN wget https://apt.lavasoftware.org/lavasoftware.key.asc \ + && apt-key add lavasoftware.key.asc \ + && echo 'deb https://apt.lavasoftware.org/release stretch-backports main' > /etc/apt/sources.list.d/lava.list \ && apt-get clean && apt-get update && apt-get -y upgrade RUN service postgresql start \ - && DEBIAN_FRONTEND=noninteractive apt-get -y install lava \ + && DEBIAN_FRONTEND=noninteractive apt-get -y install lava lava-server \ && sudo -u postgres psql lavaserver -c "DELETE FROM lava_scheduler_app_worker WHERE lava_scheduler_app_worker.hostname = 'example.com';" \ && service postgresql stop diff --git a/lava-slave-base/Dockerfile b/lava-slave-base/Dockerfile index ec884cf..5bca9b6 100644 --- a/lava-slave-base/Dockerfile +++ b/lava-slave-base/Dockerfile @@ -10,6 +10,8 @@ RUN \ DEBIAN_FRONTEND=noninteractive apt-get -y install \ locales \ vim \ + letsencrypt \ + apt-transport-https \ sudo \ python-setproctitle \ tftpd-hpa \ @@ -24,11 +26,11 @@ RUN \ RUN if [ "$(uname -m)" = "x86_64" -o "$(uname -m)" = "x86" ] ;then apt-get -y install qemu-kvm ; fi RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget gnupg -RUN wget http://images.validation.linaro.org/production-repo/production-repo.key.asc -RUN apt-key add production-repo.key.asc -RUN echo 'deb http://images.validation.linaro.org/production-repo/ stretch-backports main' > /etc/apt/sources.list.d/lava.list -RUN echo "deb http://deb.debian.org/debian/ stretch-backports main" >> /etc/apt/sources.list -RUN apt-get clean && apt-get update +RUN wget https://apt.lavasoftware.org/lavasoftware.key.asc \ + && apt-key add lavasoftware.key.asc \ + && echo 'deb https://apt.lavasoftware.org/release stretch-backports main' > /etc/apt/sources.list.d/lava.list \ + && echo "deb http://deb.debian.org/debian/ stretch-backports main" >> /etc/apt/sources.list \ + && apt-get clean && apt-get update COPY 99-stretch-backports /etc/apt/preferences.d/ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install lava-dispatcher -- cgit 1.2.3-korg From a2ca551b15f506fa418f63cf9002524ab4937f40 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 09:55:08 +0100 Subject: lava-xxx-base: do not install recommended packages For limiting size of docker images, do not install recommended packages --- lava-master-base/Dockerfile | 2 ++ lava-slave-base/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lava-master-base/Dockerfile b/lava-master-base/Dockerfile index fc7471a..ee7d47f 100644 --- a/lava-master-base/Dockerfile +++ b/lava-master-base/Dockerfile @@ -12,6 +12,8 @@ RUN echo 'lava-server lava-server/instance-name string lava-docker-instance' | && echo 'locales locales/locales_to_be_generated multiselect C.UTF-8 UTF-8, en_US.UTF-8 UTF-8 ' | debconf-set-selections \ && echo 'locales locales/default_environment_locale select en_US.UTF-8' | debconf-set-selections +RUN echo "APT::Install-Recommends false;" > /etc/apt/apt.conf.d/01norecommands + # e2fsprogs is for libguestfs RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ locales \ diff --git a/lava-slave-base/Dockerfile b/lava-slave-base/Dockerfile index 5bca9b6..d7852a7 100644 --- a/lava-slave-base/Dockerfile +++ b/lava-slave-base/Dockerfile @@ -1,5 +1,7 @@ FROM bitnami/minideb:stretch +RUN echo "APT::Install-Recommends false;" > /etc/apt/apt.conf.d/01norecommands + RUN apt-get update # e2fsprogs is for libguestfs -- cgit 1.2.3-korg From 8d0512681e342e2781282169928b617ab58ed01e Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 09:57:29 +0100 Subject: lava-master: handle order of LAVA patchs The current way to add patchs is non-determinist and could lead to failure to build. This patch made the order determinist by using sort --- lava-master/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile index 5d2a119..54a922d 100644 --- a/lava-master/Dockerfile +++ b/lava-master/Dockerfile @@ -24,7 +24,7 @@ COPY device-types-patch/ /root/device-types-patch/ RUN cd /etc/lava-server/dispatcher-config/device-types/ && for patch in $(ls /root/device-types-patch/*patch) ; do sed -i 's,lava_scheduler_app/tests/device-types/,,' $patch && patch < $patch || exit $?; done COPY lava-patch/ /root/lava-patch -RUN cd /usr/lib/python3/dist-packages && for patch in $(ls /root/lava-patch/*patch) ; do patch -p1 < $patch || exit $?;done +RUN cd /usr/lib/python3/dist-packages && for patch in $(ls /root/lava-patch/*patch| sort) ; do echo $patch && patch -p1 < $patch || exit $?;done COPY zmq_auth/ /etc/lava-dispatcher/certificates.d/ -- cgit 1.2.3-korg From 3d09473aa69318828525acb9e5eb94bac8834a7c Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 17:15:51 +0100 Subject: lava-slave: Applying patch need the patch package --- lava-slave/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index 32a76b0..93179c2 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -44,6 +44,7 @@ COPY default/* /etc/default/ COPY phyhostname /root/ COPY scripts/setup.sh . +RUN apt-get -y install patch COPY lava-patch/ /root/lava-patch RUN cd /usr/lib/python3/dist-packages && for patch in $(ls /root/lava-patch/*patch) ; do patch -p1 < $patch || exit $?;done -- cgit 1.2.3-korg From 39fce0d2a724c6577c56b763a48d2d0f0b0c6f0f Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 20 Mar 2019 16:46:59 +0100 Subject: lava-slave: telnet is needed for ser2net --- lava-slave/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index 93179c2..bd89f7a 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -2,7 +2,7 @@ FROM baylibre/lava-slave-base:2018.11-1_bpo9_1 RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get -y install cu conmux +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install cu conmux telnet COPY configs/lava-slave /etc/lava-dispatcher/lava-slave -- cgit 1.2.3-korg From 86b4e7b155fd3e18d52ba022b34106a21469108a Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 10:00:02 +0100 Subject: lava-slave: dont hide lavacli device-types list errors --- lava-slave/scripts/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lava-slave/scripts/setup.sh b/lava-slave/scripts/setup.sh index e0e8322..0b8a0b2 100755 --- a/lava-slave/scripts/setup.sh +++ b/lava-slave/scripts/setup.sh @@ -18,7 +18,7 @@ TIMEOUT=300 while [ $TIMEOUT -ge 1 ]; do STEP=2 - lavacli $LAVACLIOPTS device-types list 2>/dev/null >/dev/null + lavacli $LAVACLIOPTS device-types list >/dev/null if [ $? -eq 0 ];then TIMEOUT=0 else -- cgit 1.2.3-korg From f18959ec2357e7b1c54bca47d3a2d0f0a7c53579 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 10:00:29 +0100 Subject: lava-slave: fix build failure due to pxe on non-amd64 The package grub-efi-amd64-bin is only availlable on amd64. Installing it on arm64 lead to a build failure. This patch permits to install it even on arm64 --- lava-slave/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index bd89f7a..b3ae057 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -35,7 +35,9 @@ RUN apt-get update RUN apt-get -y install lavacli # PXE stuff -RUN apt-get -y install grub-efi-amd64-bin +RUN if [ $(uname -m) != amd64 ]; then dpkg --add-architecture amd64 && apt-get update; fi +RUN apt-get -y install grub-efi-amd64-bin:amd64 +RUN if [ $(uname -m) != amd64 ]; then dpkg --remove architecture amd64 && apt-get update; fi RUN cd /var/lib/lava/dispatcher/tmp && grub-mknetdir --net-directory=. COPY grub.cfg /var/lib/lava/dispatcher/tmp/boot/grub/ -- cgit 1.2.3-korg From ff4536c2f51e8b376d7b2d7b78d391cbccc726a5 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 10:02:29 +0100 Subject: lavalab-gen.py: add ALLOWED_HOSTS Since 2019.03 , ALLOWED_HOSTS is mandatory in settings.conf. Without it all requests are denied with code 500. --- README.md | 3 +++ lavalab-gen.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 177667a..7002c57 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,9 @@ masters: slave_keys: optional path to a directory with slaves public key. Usefull when you want to create a master without slaves nodes in boards.yaml. persistent_db: True/False (default False) Is the postgres DB is persistent over reboot http_fqdn: The FQDN used to access the LAVA web interface. This is necessary if you use https otherwise you will issue CSRF errors. + allowed_hosts: A list of FQDN used to access the LAVA master + - "fqdn1" + - "fqdn2" loglevel: lava-logs: DEBUG/INFO/WARN/ERROR (optional) select the loglevel of lava-logs (default to DEBUG) lava-slave: DEBUG/INFO/WARN/ERROR (optional) select the loglevel of lava-slave (default to DEBUG) diff --git a/lavalab-gen.py b/lavalab-gen.py index b0b58b4..d1a78e2 100755 --- a/lavalab-gen.py +++ b/lavalab-gen.py @@ -14,6 +14,7 @@ tokens_yaml = "tokens.yaml" baud_default = 115200 ser2net_port_start = 63001 ser2net_ports = {} +allowed_hosts_list = [ '"127.0.0.1"' ] template_conmux = string.Template("""# # auto-generated by lavalab-gen.py for ${board} @@ -60,6 +61,7 @@ template_settings_conf = string.Template(""" "HTTPS_XML_RPC": false, "LOGIN_URL": "/accounts/login/", "LOGIN_REDIRECT_URL": "/", + "ALLOWED_HOSTS": [ $allowed_hosts ], "CSRF_TRUSTED_ORIGINS": ["$lava_http_fqdn"], "CSRF_COOKIE_SECURE": $cookie_secure, "SESSION_COOKIE_SECURE": $session_cookie_secure @@ -94,7 +96,7 @@ def main(): else: masters = workers["masters"] for master in masters: - keywords_master = [ "name", "type", "host", "users", "groups", "tokens", "webadmin_https", "persistent_db", "zmq_auth", "zmq_auth_key", "zmq_auth_key_secret", "http_fqdn", "slave_keys", "slaveenv", "loglevel" ] + keywords_master = [ "name", "type", "host", "users", "groups", "tokens", "webadmin_https", "persistent_db", "zmq_auth", "zmq_auth_key", "zmq_auth_key_secret", "http_fqdn", "slave_keys", "slaveenv", "loglevel", "allowed_hosts" ] for keyword in master: if not keyword in keywords_master: print("WARNING: unknown keyword %s" % keyword) @@ -149,13 +151,19 @@ def main(): session_cookie_secure = "false" if "http_fqdn" in worker: lava_http_fqdn = worker["http_fqdn"] + allowed_hosts_list.append('"%s"' % lava_http_fqdn) else: - lava_http_fqdn = "example.com" + lava_http_fqdn = "127.0.0.1" + allowed_hosts_list.append('"%s"' % name) + if "allowed_hosts" in worker: + for allow_host in worker["allowed_hosts"]: + allowed_hosts_list.append('"%s"' % allow_host) + allowed_hosts = ','.join(allowed_hosts_list) f_fqdn = open("%s/lava_http_fqdn" % workerdir, 'w') f_fqdn.write(lava_http_fqdn) f_fqdn.close() fsettings = open("%s/settings.conf" % workerdir, 'w') - fsettings.write(template_settings_conf.substitute(cookie_secure=cookie_secure, session_cookie_secure=session_cookie_secure, lava_http_fqdn=lava_http_fqdn)) + fsettings.write(template_settings_conf.substitute(cookie_secure=cookie_secure, session_cookie_secure=session_cookie_secure, lava_http_fqdn=lava_http_fqdn, allowed_hosts=allowed_hosts)) fsettings.close() master_use_zmq_auth = False if "zmq_auth" in worker: -- cgit 1.2.3-korg From aa5bbacca152343b5d8b692462d547b525b04ccf Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 19 Mar 2019 10:06:27 +0100 Subject: Use 2019.03 Since 2019.03 is ready, use it --- lava-master/Dockerfile | 2 +- lava-slave/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile index 54a922d..ba9496e 100644 --- a/lava-master/Dockerfile +++ b/lava-master/Dockerfile @@ -1,4 +1,4 @@ -FROM baylibre/lava-master-base:2018.11-1_bpo9_1 +FROM baylibre/lava-master-base:2019.03_stretch COPY backup /root/backup/ diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index b3ae057..3ff2888 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -1,4 +1,4 @@ -FROM baylibre/lava-slave-base:2018.11-1_bpo9_1 +FROM baylibre/lava-slave-base:2019.03_stretch RUN apt-get update -- cgit 1.2.3-korg From 12d276e4a5d62b571c196115aea3659e8545f265 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 20 Mar 2019 11:56:39 +0100 Subject: lava-master: warning when overwriting existing device-types --- lava-master/scripts/setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh index c22a06f..489dd51 100755 --- a/lava-master/scripts/setup.sh +++ b/lava-master/scripts/setup.sh @@ -151,6 +151,10 @@ mkdir -p /root/.lavadocker/ if [ -e /root/device-types ];then for i in $(ls /root/device-types/*jinja2) do + if [ -e /etc/lava-server/dispatcher-config/device-types/$(basename $i) ];then + echo "WARNING: overwriting device-type $i" + diff -u "/etc/lava-server/dispatcher-config/device-types/$(basename $i)" $i + fi cp $i /etc/lava-server/dispatcher-config/device-types/ devicetype=$(basename $i |sed 's,.jinja2,,') lava-server manage device-types list | grep -q "[[:space:]]$devicetype[[:space:]]" -- cgit 1.2.3-korg From a8bbe0d9f99824c2faff228a50a1a4830c227b9c Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Mon, 1 Apr 2019 11:03:40 +0200 Subject: lava-slave: PXE: remove a File Not Found error biosdisk is not provided in default install and is not used, remove it. --- lava-slave/grub.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/lava-slave/grub.cfg b/lava-slave/grub.cfg index e133fed..d7074b3 100644 --- a/lava-slave/grub.cfg +++ b/lava-slave/grub.cfg @@ -5,7 +5,6 @@ insmod loopback insmod iso9660 insmod all_video insmod regexp -insmod biosdisk set pager=1 # This fake menu is necessary for letting LAVA see that grub is started -- cgit 1.2.3-korg From 14e3c762e575d8742b39799018913e7d944111b6 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 3 Apr 2019 11:42:28 +0200 Subject: lava-master: fix the applying of patchs against LAVA Lava patch which contains device-type changes are ignored since final device-types are already in /etc/lava-server/device-types/ So for fixing this, we apply lava-patch before device-type patch and synchronize them. --- lava-master/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile index ba9496e..f8ed805 100644 --- a/lava-master/Dockerfile +++ b/lava-master/Dockerfile @@ -20,11 +20,12 @@ COPY scripts/setup.sh / COPY settings.conf /etc/lava-server/ -COPY device-types-patch/ /root/device-types-patch/ -RUN cd /etc/lava-server/dispatcher-config/device-types/ && for patch in $(ls /root/device-types-patch/*patch) ; do sed -i 's,lava_scheduler_app/tests/device-types/,,' $patch && patch < $patch || exit $?; done - COPY lava-patch/ /root/lava-patch RUN cd /usr/lib/python3/dist-packages && for patch in $(ls /root/lava-patch/*patch| sort) ; do echo $patch && patch -p1 < $patch || exit $?;done +RUN rsync -avr /usr/lib/python3/dist-packages/lava_scheduler_app/tests/device-types/ /etc/lava-server/dispatcher-config/device-types/ + +COPY device-types-patch/ /root/device-types-patch/ +RUN cd /etc/lava-server/dispatcher-config/device-types/ && for patch in $(ls /root/device-types-patch/*patch) ; do sed -i 's,lava_scheduler_app/tests/device-types/,,' $patch && echo $patch && patch < $patch || exit $?; done COPY zmq_auth/ /etc/lava-dispatcher/certificates.d/ -- cgit 1.2.3-korg