From d42030d39800b930634dba1efafcf43959c40205 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 4 Jul 2018 14:45:58 +0200 Subject: Handle ZMQ auth This patch add support for using ZMQ auth. Basicly adding "zmq_auth: True" to a master is sufficient to enable it. Since "ZMQ certificates" are using a custom format (vs X509 classic), we need to use the custom generator. For helping with that a temporary docker is generated which handle generating thoses files. --- README.md | 5 ++++ lava-master/Dockerfile | 2 ++ lava-master/scripts/setup.sh | 9 ++++++ lava-master/zmq_auth/.empty | 0 lava-slave/Dockerfile | 2 ++ lava-slave/scripts/setup.sh | 8 ++++++ lava-slave/zmq_auth/.empty | 0 lavalab-gen.py | 33 +++++++++++++++++++-- zmqauth/docker-compose.yml | 6 ++++ zmqauth/zmq_auth_fill.sh | 7 +++++ zmqauth/zmq_auth_gen/Dockerfile | 17 +++++++++++ zmqauth/zmq_auth_gen/create_certificate.py | 46 ++++++++++++++++++++++++++++++ zmqauth/zmq_auth_gen/zmq_gen.sh | 23 +++++++++++++++ 13 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 lava-master/zmq_auth/.empty create mode 100644 lava-slave/zmq_auth/.empty create mode 100644 zmqauth/docker-compose.yml create mode 100755 zmqauth/zmq_auth_fill.sh create mode 100644 zmqauth/zmq_auth_gen/Dockerfile create mode 100755 zmqauth/zmq_auth_gen/create_certificate.py create mode 100644 zmqauth/zmq_auth_gen/zmq_gen.sh diff --git a/README.md b/README.md index 4dd43d9..66bec8f 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,9 @@ masters: - name: lava-master name of the master host: name name of the host running lava-master (default to "local") webadmin_https: Does the LAVA webadmin is accessed via https + zmq_auth: True/False Does the master requires ZMQ authentication. + zmq_auth_key: optional path to a public ZMQ key + zmq_auth_key_secret: optional path to a private ZMQ key persistent_db: True/False (default False) Is the postgres DB is persistent over reboot users: - name: LAVA username @@ -229,6 +232,8 @@ masters: slaves: - name: lab-slave-XX The name of the slave (where XX is a number) host: name name of the host running lava-slave-XX (default to "local") + zmq_auth_key: optional path to a public ZMQ key + zmq_auth_key_secret: optional path to a private ZMQ key dispatcher_ip: the IP where the slave could be contacted. In lava-docker it is the host IP since docker proxify TFTP from host to the slave. remote_master: the name of the master to connect to remote_address: the FQDN or IP address of the master (if different from remote_master) diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile index a074570..3299922 100644 --- a/lava-master/Dockerfile +++ b/lava-master/Dockerfile @@ -93,6 +93,8 @@ RUN cd /etc/lava-server/dispatcher-config/device-types/ && for patch in $(ls /ro COPY lava-patch/ /root/lava-patch RUN cd /usr/lib/python3/dist-packages && for patch in $(ls /root/lava-patch/*patch) ; do patch -p1 < $patch || exit $?;done +COPY zmq_auth/ /etc/lava-dispatcher/certificates.d/ + EXPOSE 69/udp 80 3079 5555 5556 CMD /start.sh && while [ true ];do sleep 365d; done diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh index 6ab0663..c7807dc 100755 --- a/lava-master/scripts/setup.sh +++ b/lava-master/scripts/setup.sh @@ -118,3 +118,12 @@ do lava-server manage devices add --device-type $devicetype --worker $worker $devicename || exit $? done done + +if [ -e /etc/lava-dispatcher/certificates.d/$(hostname).key ];then + echo "INFO: Enabling encryption" + sed -i 's,.*ENCRYPT=.*,ENCRYPT="--encrypt",' /etc/lava-server/lava-master || exit $? + sed -i 's,.*MASTER_CERT=.*,MASTER_CERT="--master-cert /etc/lava-dispatcher/certificates.d/$(hostname).key_secret",' /etc/lava-server/lava-master || exit $? + sed -i 's,.*ENCRYPT=.*,ENCRYPT="--encrypt",' /etc/lava-server/lava-logs || exit $? + sed -i 's,.*MASTER_CERT=.*,MASTER_CERT="--master-cert /etc/lava-dispatcher/certificates.d/$(hostname).key_secret",' /etc/lava-server/lava-logs || exit $? +fi +exit 0 diff --git a/lava-master/zmq_auth/.empty b/lava-master/zmq_auth/.empty new file mode 100644 index 0000000..e69de29 diff --git a/lava-slave/Dockerfile b/lava-slave/Dockerfile index a5b0148..4bd14b4 100644 --- a/lava-slave/Dockerfile +++ b/lava-slave/Dockerfile @@ -86,6 +86,8 @@ RUN ssh-keygen -q -f /root/.ssh/id_rsa RUN cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys COPY lava-screen.conf /root/ +COPY zmq_auth/ /etc/lava-dispatcher/certificates.d/ + EXPOSE 69/udp 80 CMD /start.sh diff --git a/lava-slave/scripts/setup.sh b/lava-slave/scripts/setup.sh index bf91c7a..e696e57 100755 --- a/lava-slave/scripts/setup.sh +++ b/lava-slave/scripts/setup.sh @@ -91,3 +91,11 @@ do fi done done + +if [ -e /etc/lava-dispatcher/certificates.d/$(hostname).key ];then + echo "INFO: Enabling encryption" + sed -i 's,.*ENCRYPT=.*,ENCRYPT="--encrypt",' /etc/lava-dispatcher/lava-slave + sed -i "s,.*SLAVE_CERT=.*,SLAVE_CERT=\"--slave-cert /etc/lava-dispatcher/certificates.d/$(hostname).key_secret\"," /etc/lava-dispatcher/lava-slave + sed -i "s,.*MASTER_CERT=.*,MASTER_CERT=\"--master-cert /etc/lava-dispatcher/certificates.d/$LAVA_MASTER.key\"," /etc/lava-dispatcher/lava-slave +fi +exit 0 diff --git a/lava-slave/zmq_auth/.empty b/lava-slave/zmq_auth/.empty new file mode 100644 index 0000000..e69de29 diff --git a/lavalab-gen.py b/lavalab-gen.py index e0c1c16..a642400 100755 --- a/lavalab-gen.py +++ b/lavalab-gen.py @@ -12,7 +12,7 @@ import shutil boards_yaml = "boards.yaml" tokens_yaml = "tokens.yaml" baud_default = 115200 - + template_conmux = string.Template("""# # auto-generated by lavalab-gen.py for ${board} # @@ -74,18 +74,20 @@ template_settings_conf = string.Template(""" """) def main(): + need_zmq_auth_gen = False fp = open(boards_yaml, "r") workers = yaml.load(fp) fp.close() os.mkdir("output") + zmq_auth_genlist = open("zmqauth/zmq_auth_gen/zmq_genlist", 'w') if "masters" not in workers: print("Missing masters entry in boards.yaml") sys.exit(1) masters = workers["masters"] for master in masters: - keywords_master = [ "name", "type", "host", "users", "tokens", "webadmin_https", "persistent_db" ] + keywords_master = [ "name", "type", "host", "users", "tokens", "webadmin_https", "persistent_db", "zmq_auth", "zmq_auth_key", "zmq_auth_key_secret" ] for keyword in master: if not keyword in keywords_master: print("WARNING: unknown keyword %s" % keyword) @@ -139,6 +141,16 @@ def main(): fsettings = open("%s/settings.conf" % workerdir, 'w') fsettings.write(template_settings_conf.substitute(cookie_secure=cookie_secure, session_cookie_secure=session_cookie_secure)) fsettings.close() + master_use_zmq_auth = False + if "zmq_auth" in worker: + master_use_zmq_auth = True + if master_use_zmq_auth: + if "zmq_auth_key" in worker: + shutil.copy(worker["zmq_auth_key"], "%s/zmq_auth/" % workerdir) + shutil.copy(worker["zmq_auth_key_secret"], "%s/zmq_auth/" % workerdir) + else: + zmq_auth_genlist.write("%s/%s\n" % (host, name)) + need_zmq_auth_gen = True if "users" in worker: for user in worker["users"]: keywords_users = [ "name", "staff", "superuser", "password", "token" ] @@ -195,7 +207,7 @@ def main(): sys.exit(1) slaves = workers["slaves"] for slave in slaves: - keywords_slaves = [ "name", "host", "dispatcher_ip", "remote_user", "remote_master", "remote_address", "remote_rpc_port", "remote_proto", "extra_actions" ] + keywords_slaves = [ "name", "host", "dispatcher_ip", "remote_user", "remote_master", "remote_address", "remote_rpc_port", "remote_proto", "extra_actions", "zmq_auth_key", "zmq_auth_key_secret" ] for keyword in slave: if not keyword in keywords_slaves: print("WARNING: unknown keyword %s" % keyword) @@ -264,6 +276,17 @@ def main(): for fuser in fm["users"]: if fuser["name"] == remote_user: remote_token = fuser["token"] + if "zmq_auth" in fm: + if "zmq_auth_key" in fm: + shutil.copy(fm["zmq_auth_key"], "%s/zmq_auth/" % workerdir) + if "zmq_auth_key" in worker: + shutil.copy(worker["zmq_auth_key"], "%s/zmq_auth/" % workerdir) + shutil.copy(worker["zmq_auth_key_secret"], "%s/zmq_auth/" % workerdir) + if "zmq_auth_key" in fm: + shutil.copy(worker["zmq_auth_key"], "output/%s/%s/zmq_auth/" % (fm["host"], fm["name"])) + else: + zmq_auth_genlist.write("%s/%s %s/%s\n" % (host, name, fm["host"], fm["name"])) + need_zmq_auth_gen = True if remote_token is "BAD": print("Cannot find %s on %s" % (remote_user, remote_master)) sys.exit(1) @@ -416,6 +439,10 @@ def main(): fp.close() with open(dockcomposeymlpath, 'w') as f: yaml.dump(dockcomp, f) + zmq_auth_genlist.close() + if need_zmq_auth_gen: + print("Gen ZMQ auth files") + subprocess.check_call(["./zmqauth/zmq_auth_fill.sh"], stdin=None) if __name__ == "__main__": diff --git a/zmqauth/docker-compose.yml b/zmqauth/docker-compose.yml new file mode 100644 index 0000000..a7147e1 --- /dev/null +++ b/zmqauth/docker-compose.yml @@ -0,0 +1,6 @@ +services: + master1: + build: {context: zmq_auth_gen } + hostname: zmqauth_builder + volumes: ['../output:/root/output'] +version: '2.0' diff --git a/zmqauth/zmq_auth_fill.sh b/zmqauth/zmq_auth_fill.sh new file mode 100755 index 0000000..31d406f --- /dev/null +++ b/zmqauth/zmq_auth_fill.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd $(dirname $0) +id -u > zmq_auth_gen/id +docker-compose build || exit $? +docker-compose up || exit $? +docker-compose down --rmi all diff --git a/zmqauth/zmq_auth_gen/Dockerfile b/zmqauth/zmq_auth_gen/Dockerfile new file mode 100644 index 0000000..46ae47a --- /dev/null +++ b/zmqauth/zmq_auth_gen/Dockerfile @@ -0,0 +1,17 @@ +FROM bitnami/minideb:stretch + +RUN apt-get update + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3-zmq + +COPY create_certificate.py /root/ +RUN chmod 750 /root/create_certificate.py +RUN mkdir /root/output + +COPY id /root/ + +COPY zmq_gen.sh /root/ +RUN chmod 755 /root/zmq_gen.sh +COPY zmq_genlist /root/ + +CMD /root/zmq_gen.sh diff --git a/zmqauth/zmq_auth_gen/create_certificate.py b/zmqauth/zmq_auth_gen/create_certificate.py new file mode 100755 index 0000000..2c4445d --- /dev/null +++ b/zmqauth/zmq_auth_gen/create_certificate.py @@ -0,0 +1,46 @@ +#! /usr/bin/python3 +# -*- coding: utf-8 -*- +# +# Copyright 2016 Rémi Duraffort +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# + +import argparse +import zmq.auth + + +def main(): + """ + Parse options and create the certificate + """ + parser = argparse.ArgumentParser(description="") + parser.add_argument("--directory", type=str, + default="/etc/lava-dispatcher/certificates.d", + help="Directory where to store the certificates") + parser.add_argument(type=str, dest="name", + help="Name of the certificate") + args = parser.parse_args() + + # Create the certificate + print("Creating the certificate in %s" % args.directory) + zmq.auth.create_certificates(args.directory, args.name) + print(" - %s.key" % args.name) + print(" - %s.key_secret" % args.name) + + +if __name__ == '__main__': + main() diff --git a/zmqauth/zmq_auth_gen/zmq_gen.sh b/zmqauth/zmq_auth_gen/zmq_gen.sh new file mode 100644 index 0000000..8b67280 --- /dev/null +++ b/zmqauth/zmq_auth_gen/zmq_gen.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +#rm /root/output/* +while read line +do + NAME=$(echo $line | cut -d' ' -f1 | sed 's,.*/,,') + DIR=$(echo $line | cut -d' ' -f1) + MASTERDIR=$(echo $line | cut -d' ' -f2) + echo "DEBUG: $LINE NAME=$NAME DIR=$DIR" + if [ ! -e /root/output/$DIR/zmq_auth/${NAME}.key ];then + /root/create_certificate.py $NAME --directory /root/output/$DIR/zmq_auth/ || exit $? + else + echo "DEBUG: ZMQ files for $NAME already exists" + fi + if [ ! -z "$MASTERDIR" -a "$MASTERDIR" != "$DIR" ];then + MASTERNAME=$(echo $MASTERDIR | sed 's,.*/,,') + cp /root/output/$MASTERDIR/zmq_auth/$MASTERNAME.key /root/output/$DIR/zmq_auth/master.key || exit $? + cp /root/output/$DIR/zmq_auth/$NAME.key /root/output/$MASTERDIR/zmq_auth/ || exit $? + chown $(cat /root/id) /root/output/$MASTERDIR/zmq_auth/* || exit $? + fi + # All files are generated by root, chown them to the user using the docker + chown $(cat /root/id) /root/output/$DIR/zmq_auth/* || exit $? +done < /root/zmq_genlist -- cgit 1.2.3-korg