Merge pull request #122 from montjoie/postgres-fix

Postgres fix

4 files changed, 75 insertions, 25 deletions

diff --git a/README.md b/README.md
index 5895a51..bb4b02f 100644
--- a/README.md
+++ b/README.md
@@ -221,6 +221,7 @@ masters:
     slave_keys:			optional path to a directory with slaves public key. Usefull when you want to create a master without slaves nodes in boards.yaml.
     lava-coordinator:		Does the master should ran a lava-coordinator and export its port
     persistent_db: True/False	(default False) Is the postgres DB is persistent over reboot
+    pg_lava_password:		The Postgres lavaserver password to set
     http_fqdn:			The FQDN used to access the LAVA web interface. This is necessary if you use https otherwise you will issue CSRF errors.
     healthcheck_url:		Hack healthchecks hosting URL. See hosting healthchecks below
     build_args:
diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile
index 5bf989a..6396253 100644
--- a/lava-master/Dockerfile
+++ b/lava-master/Dockerfile
@@ -39,6 +39,8 @@ COPY apache2/ /etc/apache2/
 # Fixes 'postgresql ERROR:  invalid locale name: "en_US.UTF-8"' when restoring a backup
 RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen en_US.UTF-8
 
+COPY pg_lava_password /root
+
 # TODO: send this fix to upstream
 RUN sed -i 's,find /root/entrypoint.d/ -type f,find /root/entrypoint.d/ -type f | sort,' /root/entrypoint.sh
 # TODO: send this fix to upstream
diff --git a/lava-master/entrypoint.d/01_setup.sh b/lava-master/entrypoint.d/01_setup.sh
index 53db47e..e3619f7 100755
--- a/lava-master/entrypoint.d/01_setup.sh
+++ b/lava-master/entrypoint.d/01_setup.sh
@@ -1,8 +1,11 @@
 #!/bin/bash
 
 # always reset the lavaserver user, since its password could have been reseted in a "docker build --nocache"
-if [ ! -e /root/pg_lava_password ];then
-       < /dev/urandom tr -dc A-Za-z0-9 | head -c16 > /root/pg_lava_password
+if [ ! -s /root/pg_lava_password ];then
+	echo "DEBUG: Generating a random LAVA password"
+	< /dev/urandom tr -dc A-Za-z0-9 | head -c16 > /root/pg_lava_password
+else
+	echo "DEBUG: use the given LAVA password"
 fi
 sudo -u postgres psql -c "ALTER USER lavaserver WITH PASSWORD '$(cat /root/pg_lava_password)';" || exit $?
 if [ -e /etc/lava-server/instance.conf ];then
@@ -13,32 +16,37 @@ else
 	sed -i "s,PASSWORD:.*,PASSWORD: '$(cat /root/pg_lava_password)'," /etc/lava-server/settings.d/00-database.yaml || exit $?
 fi
 
-if [ -e /root/backup/db_lavaserver.gz ];then
-	gunzip /root/backup/db_lavaserver.gz || exit $?
-fi
+# verify that the backup was not already applied in case of persistent_db
+if [ ! -e "/var/lib/postgresql/lava-docker.backup_done" ];then
+	if [ -e /root/backup/db_lavaserver.gz ];then
+		gunzip /root/backup/db_lavaserver.gz || exit $?
+	fi
 
-if [ -e /root/backup/db_lavaserver ];then
-	echo "Restore database from backup"
-	sudo -u postgres psql < /root/backup/db_lavaserver || exit $?
-	yes yes | lava-server manage migrate || exit $?
-	echo "Restore jobs output from backup"
-	rm -r /var/lib/lava-server/default/media/job-output/*
+	if [ -e /root/backup/db_lavaserver ];then
+		echo "Restore database from backup"
+		sudo -u postgres psql < /root/backup/db_lavaserver || exit $?
+		yes yes | lava-server manage migrate || exit $?
+		echo "Restore jobs output from backup"
+		rm -r /var/lib/lava-server/default/media/job-output/*
 
-        # allow using different folder for tar operations (/tmp by default)
-        TMPDIR=${TMPDIR:-/tmp}
+	        # allow using different folder for tar operations (/tmp by default)
+		TMPDIR=${TMPDIR:-/tmp}
 
-	tar xzf /root/backup/joboutput.tar.gz || exit $?
-fi
-	lava-server manage makemigrations
-	yes yes | lava-server manage migrate || exit $?
-
-if [ -e /root/backup/devices.tar.gz ];then
-	echo "INFO: Restoring devices files"
-	tar xzf /root/backup/devices.tar.gz
-	chown -R lavaserver:lavaserver /etc/lava-server/dispatcher-config/devices
+		tar xzf /root/backup/joboutput.tar.gz || exit $?
+		chown -R lavaserver:lavaserver /var/lib/lava-server/default/media/job-output/
+		touch /var/lib/postgresql/lava-docker.backup_done
+	fi
+	if [ -e /root/backup/devices.tar.gz ];then
+		echo "INFO: Restoring devices files"
+		tar xzf /root/backup/devices.tar.gz
+		chown -R lavaserver:lavaserver /etc/lava-server/dispatcher-config/devices
+	fi
+else
+	echo "DEBUG: backup already applied"
 fi
 
-chown -R lavaserver:lavaserver /var/lib/lava-server/default/media/job-output/
+lava-server manage makemigrations
+yes yes | lava-server manage migrate || exit $?
 
 # default site is set as example.com
 if [ -e /root/lava_http_fqdn ];then
diff --git a/lavalab-gen.py b/lavalab-gen.py
index bcc727f..f7cd66d 100755
--- a/lavalab-gen.py
+++ b/lavalab-gen.py
@@ -114,7 +114,21 @@ def main():
     else:
         masters = workers["masters"]
     for master in masters:
-        keywords_master = [ "name", "type", "host", "users", "groups", "tokens", "webadmin_https", "persistent_db", "zmq_auth", "zmq_auth_key", "zmq_auth_key_secret", "http_fqdn", "slave_keys", "slaveenv", "loglevel", "allowed_hosts", "lava-coordinator", "healthcheck_url", "smtp", "version", "build_args" ]
+        keywords_master = [
+            "allowed_hosts",
+            "build_args",
+            "groups",
+            "healthcheck_url", "host", "http_fqdn",
+            "loglevel", "lava-coordinator",
+            "name",
+            "persistent_db", "pg_lava_password",
+            "slave_keys", "slaveenv", "smtp",
+            "tokens", "type",
+            "users",
+            "version",
+            "webadmin_https",
+            "zmq_auth", "zmq_auth_key", "zmq_auth_key_secret",
+            ]
         for keyword in master:
             if not keyword in keywords_master:
                 print("WARNING: unknown keyword %s" % keyword)
@@ -145,8 +159,11 @@ def main():
         if persistent_db:
             pg_volume_name = "pgdata_" + name
             dockcomp["services"][name]["volumes"].append(pg_volume_name + ":/var/lib/postgresql")
+            etc_volume_name = "lava_etc_" + name
+            dockcomp["services"][name]["volumes"].append(etc_volume_name + ":/etc/lava-server/")
             dockcomp["services"][name]["volumes"].append("lava_job_output:/var/lib/lava-server/default/media/job-output/")
             dockcomp["volumes"] = {}
+            dockcomp["volumes"][etc_volume_name] = {}
             dockcomp["volumes"][pg_volume_name] = {}
             dockcomp["volumes"]["lava_job_output"] = {}
 
@@ -158,6 +175,13 @@ def main():
         groupdir = "%s/groups" % workerdir
         os.mkdir(groupdir)
         worker = master
+        if "pg_lava_password" in master:
+            f_pg = open("%s/pg_lava_password" % workerdir, 'w')
+            f_pg.write(master["pg_lava_password"])
+            f_pg.close()
+        else:
+            f_pg = open("%s/pg_lava_password" % workerdir, 'w')
+            f_pg.close()
         if "version" in worker:
             dockerfile = open("%s/Dockerfile" % workerdir, "r+")
             dockerfilec = re.sub('(^FROM.*:).*', '\g<1>%s' % worker["version"], dockerfile.read())
@@ -373,7 +397,22 @@ def main():
     else:
         slaves = workers["slaves"]
     for slave in slaves:
-        keywords_slaves = [ "name", "host", "dispatcher_ip", "remote_user", "remote_master", "remote_address", "remote_rpc_port", "remote_proto", "extra_actions", "zmq_auth_key", "zmq_auth_key_secret", "default_slave", "export_ser2net", "expose_ser2net", "remote_user_token", "zmq_auth_master_key", "expose_ports", "env", "bind_dev", "loglevel", "use_nfs", "arch", "devices", "lava-coordinator", "use_tap", "host_healthcheck", "use_tftp", "use_nbd", "use_overlay_server", "tags", "use_docker", "version", "custom_volumes","build_args" ]
+        keywords_slaves = [
+            "arch",
+            "bind_dev", "build_args",
+            "custom_volumes",
+            "devices", "dispatcher_ip", "default_slave",
+            "extra_actions", "export_ser2net", "expose_ser2net", "expose_ports", "env",
+            "host", "host_healthcheck",
+            "loglevel", "lava-coordinator",
+            "name",
+            "remote_user", "remote_master", "remote_address", "remote_rpc_port", "remote_proto", "remote_user_token",
+            "tags",
+            "use_docker", "use_nfs", "use_nbd", "use_overlay_server", "use_tftp", "use_tap",
+            "version",
+            "zmq_auth_key", "zmq_auth_key_secret",
+            "zmq_auth_master_key",
+        ]
         for keyword in slave:
             if not keyword in keywords_slaves:
                 print("WARNING: unknown keyword %s" % keyword)