From 87eff59f943ed44cc9d568602ed9e2756088398c Mon Sep 17 00:00:00 2001 From: Philippe Mazet Date: Thu, 21 Nov 2019 15:10:49 +0100 Subject: backup.sh: allow using different tmp dir When backup size grows, we had cases where tar was filling up /tmp This patch allows specifying a different tmp dir, used by tar (through TMPDIR variable). Example usage: TMPDIR=/home/user/mytempdir ./backup.sh --- backup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backup.sh b/backup.sh index f4cf1de..33323b7 100755 --- a/backup.sh +++ b/backup.sh @@ -1,6 +1,11 @@ #!/bin/sh BACKUP_DIR="backup-$(date +%Y%m%d_%H%M)" +# use /tmp by default on host (this is used by tar) +TMPDIR=${TMPDIR:-/tmp} +export TMPDIR + +mkdir -p $TMPDIR mkdir $BACKUP_DIR cp boards.yaml $BACKUP_DIR -- cgit 1.2.3-korg From 9783987dcc1b2b4935ea9c5d4ff7db443f0f7abe Mon Sep 17 00:00:00 2001 From: Philippe Mazet Date: Thu, 21 Nov 2019 15:44:09 +0100 Subject: backup.sh: avoid "file changed as we read it" error In some case (long jobs), backup failed because some files were changed during tar operation. --- backup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index 33323b7..35f7d9c 100755 --- a/backup.sh +++ b/backup.sh @@ -24,7 +24,8 @@ docker exec -ti $DOCKERID gzip /tmp/db_lavaserver || exit $? docker cp $DOCKERID:/tmp/db_lavaserver.gz $BACKUP_DIR/ || exit $? docker exec -ti $DOCKERID rm /tmp/db_lavaserver.gz || exit $? -docker exec -ti $DOCKERID tar czf /root/joboutput.tar.gz /var/lib/lava-server/default/media/job-output/ || exit $? +# tar outputs warnings when file changes on disk while creating tar file. So do not "exit on error" +docker exec -ti $DOCKERID tar czf /root/joboutput.tar.gz /var/lib/lava-server/default/media/job-output/ || echo "WARNING: tar operation returned $?" docker cp $DOCKERID:/root/joboutput.tar.gz $BACKUP_DIR/ || exit $? docker exec -ti $DOCKERID rm /root/joboutput.tar.gz || exit $? -- cgit 1.2.3-korg From 231f89625b0528732348dfc2cd10e3efad1ea14b Mon Sep 17 00:00:00 2001 From: Philippe Mazet Date: Fri, 6 Dec 2019 17:41:23 +0100 Subject: Allow using different folder than /tmp for tar operations When backup size grows to several GB, tar operation can fail with "no space left on device" error. This is because of the limited size of /tmp on certain machines This changes allows using a different folder. This can be done by simply adding such line in Dockerfile: ENV TMPDIR /wherever/tmp --- lava-master/entrypoint.d/01_setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lava-master/entrypoint.d/01_setup.sh b/lava-master/entrypoint.d/01_setup.sh index 4755bc4..ba81f77 100755 --- a/lava-master/entrypoint.d/01_setup.sh +++ b/lava-master/entrypoint.d/01_setup.sh @@ -17,6 +17,10 @@ if [ -e /root/backup/db_lavaserver ];then yes yes | lava-server manage migrate || exit $? echo "Restore jobs output from backup" rm -r /var/lib/lava-server/default/media/job-output/* + + # allow using different folder for tar operations (/tmp by default) + TMPDIR=${TMPDIR:-/tmp} + tar xzf /root/backup/joboutput.tar.gz || exit $? fi -- cgit 1.2.3-korg From cccd52d88687294131116f2d1104e001cb58210d Mon Sep 17 00:00:00 2001 From: Philippe Mazet Date: Thu, 21 Nov 2019 16:01:05 +0100 Subject: backup.sh: remove docker interactive switch (does not work through crontab) --- backup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backup.sh b/backup.sh index 35f7d9c..3edd133 100755 --- a/backup.sh +++ b/backup.sh @@ -15,19 +15,19 @@ if [ -z "$DOCKERID" ];then exit 1 fi -docker exec -ti $DOCKERID tar czf /root/devices.tar.gz /etc/lava-server/dispatcher-config/devices/ || exit $? +docker exec -t $DOCKERID tar czf /root/devices.tar.gz /etc/lava-server/dispatcher-config/devices/ || exit $? docker cp $DOCKERID:/root/devices.tar.gz $BACKUP_DIR/ || exit $? # for an unknown reason pg_dump > file doesnt work -docker exec -ti $DOCKERID sudo -u postgres pg_dump --create --clean lavaserver --file /tmp/db_lavaserver || exit $? -docker exec -ti $DOCKERID gzip /tmp/db_lavaserver || exit $? +docker exec -t $DOCKERID sudo -u postgres pg_dump --create --clean lavaserver --file /tmp/db_lavaserver || exit $? +docker exec -t $DOCKERID gzip /tmp/db_lavaserver || exit $? docker cp $DOCKERID:/tmp/db_lavaserver.gz $BACKUP_DIR/ || exit $? -docker exec -ti $DOCKERID rm /tmp/db_lavaserver.gz || exit $? +docker exec -t $DOCKERID rm /tmp/db_lavaserver.gz || exit $? # tar outputs warnings when file changes on disk while creating tar file. So do not "exit on error" -docker exec -ti $DOCKERID tar czf /root/joboutput.tar.gz /var/lib/lava-server/default/media/job-output/ || echo "WARNING: tar operation returned $?" +docker exec -t $DOCKERID tar czf /root/joboutput.tar.gz /var/lib/lava-server/default/media/job-output/ || echo "WARNING: tar operation returned $?" docker cp $DOCKERID:/root/joboutput.tar.gz $BACKUP_DIR/ || exit $? -docker exec -ti $DOCKERID rm /root/joboutput.tar.gz || exit $? +docker exec -t $DOCKERID rm /root/joboutput.tar.gz || exit $? echo "Backup done in $BACKUP_DIR" rm -f backup-latest -- cgit 1.2.3-korg