summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkhilman <khilman@users.noreply.github.com>2018-07-20 11:38:40 -0500
committerGitHub <noreply@github.com>2018-07-20 11:38:40 -0500
commit45f76344b4278f1ecf9956f3a81bfe11cd525c43 (patch)
tree08cfb059940190cac3eb62bcb8b5a6d01ca67396
parented83f7aa1af39e75c10d5e8c4f1385c302beec12 (diff)
parent0f09e5c9b89cee21a6ee39db9daf8e17525dd493 (diff)
Merge pull request #32 from montjoie/persistantdb_pr
Persistantdb pr
-rw-r--r--README.md12
-rwxr-xr-xbackup.sh22
-rw-r--r--lava-master/Dockerfile2
-rw-r--r--lava-master/backup/.empty0
-rwxr-xr-xlava-master/scripts/setup.sh30
-rwxr-xr-xlavalab-gen.py12
6 files changed, 75 insertions, 3 deletions
diff --git a/README.md b/README.md
index 279fe25..4dd43d9 100644
--- a/README.md
+++ b/README.md
@@ -215,6 +215,7 @@ masters:
- name: lava-master name of the master
host: name name of the host running lava-master (default to "local")
webadmin_https: Does the LAVA webadmin is accessed via https
+ persistent_db: True/False (default False) Is the postgres DB is persistent over reboot
users:
- name: LAVA username
token: The token of this user
@@ -321,6 +322,17 @@ For the moment, it is unsupported and unbuilded.
## Backporting LAVA patches
All upstream LAVA patches could be backported by placing them in lava-master/lava-patch/
+## Backups / restore
+For backupping a running docker, the "backup.sh" script could be used.
+It will store boards.yaml + postgresql database backup + joboutputs.
+
+For restoring a backup, postgresql database backup + joboutputs must be copied in master backup directory before build.
+
+Example:
+./backup.sh
+This produce a backup-20180704_1206 directory
+For restoring this backup, simply cp backup-20180704_1206/* output/local/master/backup/
+
## Security
Note that this container provides defaults which are unsecure. If you plan on deploying this in a production enviroment please consider the following items:
diff --git a/backup.sh b/backup.sh
new file mode 100755
index 0000000..f803486
--- /dev/null
+++ b/backup.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+BACKUP_DIR="backup-$(date +%Y%m%d_%H%M)"
+
+mkdir $BACKUP_DIR
+cp boards.yaml $BACKUP_DIR
+
+DOCKERID=$(docker ps |grep master | cut -d' ' -f1)
+if [ -z "$DOCKERID" ];then
+ exit 1
+fi
+# for an unknown reason pg_dump > file doesnt work
+docker exec -ti $DOCKERID sudo -u postgres pg_dump --create --clean lavaserver --file /tmp/db_lavaserver || exit $?
+docker exec -ti $DOCKERID gzip /tmp/db_lavaserver || exit $?
+docker cp $DOCKERID:/tmp/db_lavaserver.gz $BACKUP_DIR/ || exit $?
+docker exec -ti $DOCKERID rm /tmp/db_lavaserver.gz || exit $?
+
+docker exec -ti $DOCKERID tar czf /root/joboutput.tar.gz /var/lib/lava-server/default/media/job-output/ || exit $?
+docker cp $DOCKERID:/root/joboutput.tar.gz $BACKUP_DIR/ || exit $?
+docker exec -ti $DOCKERID rm /root/joboutput.tar.gz || exit $?
+
+echo "Backup done in $BACKUP_DIR"
diff --git a/lava-master/Dockerfile b/lava-master/Dockerfile
index 7d61103..a074570 100644
--- a/lava-master/Dockerfile
+++ b/lava-master/Dockerfile
@@ -49,6 +49,8 @@ RUN /etc/init.d/postgresql start && \
bash /build-lava lava-server 2018.4 2018.4-1 && \
/etc/init.d/postgresql stop
+COPY backup /
+
RUN a2enmod proxy \
&& a2enmod proxy_http \
&& a2dissite 000-default \
diff --git a/lava-master/backup/.empty b/lava-master/backup/.empty
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lava-master/backup/.empty
diff --git a/lava-master/scripts/setup.sh b/lava-master/scripts/setup.sh
index 585499e..6ab0663 100755
--- a/lava-master/scripts/setup.sh
+++ b/lava-master/scripts/setup.sh
@@ -1,5 +1,26 @@
#!/bin/bash
+# always reset the lavaserver user, since its password could have been reseted in a "docker build --nocache"
+if [ ! -e /root/pg_lava_password ];then
+ < /dev/urandom tr -dc A-Za-z0-9 | head -c16 > /root/pg_lava_password
+fi
+sudo -u postgres psql -c "ALTER USER lavaserver WITH PASSWORD '$(cat /root/pg_lava_password)';" || exit $?
+sed -i "s,^LAVA_DB_PASSWORD=.*,LAVA_DB_PASSWORD='$(cat /root/pg_lava_password)'," /etc/lava-server/instance.conf || exit $?
+
+if [ -e /db_lavaserver.gz ];then
+ gunzip /db_lavaserver.gz || exit $?
+fi
+
+if [ -e /db_lavaserver ];then
+ echo "Restore database from backup"
+ sudo -u postgres psql < /db_lavaserver || exit $?
+ lava-server manage migrate || exit $?
+ echo "Restore jobs output from backup"
+ rm -r /var/lib/lava-server/default/media/job-output/*
+ tar xzf /joboutput.tar.gz || exit $?
+fi
+chown -R lavaserver:lavaserver /var/lib/lava-server/default/media/job-output/
+
if [ -e /root/lava-users ];then
for ut in $(ls /root/lava-users)
do
@@ -49,8 +70,13 @@ if [ -e /root/lava-callback-tokens ];then
echo "Missing DESCRIPTION for $USER"
exit 1
fi
- echo "Adding $USER ($DESCRIPTION) DEBUG($TOKEN)"
- lava-server manage tokens add --user $USER --secret $TOKEN --description "$DESCRIPTION" || exit 1
+ lava-server manage tokens list --user $USER |grep -q $TOKEN
+ if [ $? -eq 0 ];then
+ echo "SKIP already present token for $USER"
+ else
+ echo "Adding $USER ($DESCRIPTION) DEBUG($TOKEN)"
+ lava-server manage tokens add --user $USER --secret $TOKEN --description "$DESCRIPTION" || exit 1
+ fi
done
fi
diff --git a/lavalab-gen.py b/lavalab-gen.py
index f6663a2..6637bc5 100755
--- a/lavalab-gen.py
+++ b/lavalab-gen.py
@@ -85,7 +85,7 @@ def main():
sys.exit(1)
masters = workers["masters"]
for master in masters:
- keywords_master = [ "name", "type", "host", "users", "tokens", "webadmin_https" ]
+ keywords_master = [ "name", "type", "host", "users", "tokens", "webadmin_https", "persistent_db" ]
for keyword in master:
if not keyword in keywords_master:
print("WARNING: unknown keyword %s" % keyword)
@@ -108,6 +108,16 @@ def main():
dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
dockcomp["services"][name]["build"] = {}
dockcomp["services"][name]["build"]["context"] = name
+ persistent_db = False
+ if "persistent_db" in master:
+ persistent_db = master["persistent_db"]
+ if persistent_db:
+ pg_volume_name = "pgdata_" + name
+ dockcomp["services"][name]["volumes"].append(pg_volume_name + ":/var/lib/postgresql")
+ dockcomp["services"][name]["volumes"].append("lava_job_output:/var/lib/lava-server/default/media/job-output/")
+ dockcomp["volumes"] = {}
+ dockcomp["volumes"][pg_volume_name] = {}
+ dockcomp["volumes"]["lava_job_output"] = {}
with open(dockcomposeymlpath, 'w') as f:
yaml.dump(dockcomp, f)