From 36f5fc1c9ee8c69dfe97b199e669d4998b971d0c Mon Sep 17 00:00:00 2001
From: Corentin LABBE <clabbe@baylibre.com>
Date: Wed, 2 Dec 2020 10:10:34 +0000
Subject: SPEC-3705: handle lavacli wait errors

Sometime lavacli wait exit with an error.
Until the LAVA/lavacli bug is fixed, let's try to handle the situation.
Since the error seems to happen when the job end, let's just check if
the job ended, in the other case retry to wait.

Change-Id: I7b7b1aee0dcc2ecc2ef971cc40c29374307f6b0f
Bug-AGL: SPEC-3705
Signed-off-by: Corentin LABBE <clabbe@baylibre.com>
---
 jjb/common/include-agl-lava-jobs-submit.sh | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

(limited to 'jjb/common/include-agl-lava-jobs-submit.sh')

diff --git a/jjb/common/include-agl-lava-jobs-submit.sh b/jjb/common/include-agl-lava-jobs-submit.sh
index 08b2f4e8..b2769c97 100644
--- a/jjb/common/include-agl-lava-jobs-submit.sh
+++ b/jjb/common/include-agl-lava-jobs-submit.sh
@@ -57,9 +57,29 @@ sed -i -e "s/ /\\\ /g" .msg
 #eval ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} -n NONE --message='"$(cat .msg)"'  || true
 #set +x
 
+lavacli --version
+
+set +e
 
 # use lavacli wait job instead of the polling.
-lavacli -i $lab jobs wait $job_id --timeout 7200 || true
+lavacli -i $lab jobs wait $job_id --timeout 7200
+if [ $? -ne 0 ];then
+        retry=1
+        # retry a few times with a small delay
+        while [ $retry -le 5 ] ; do
+            sleep 5
+	    # Work aroung SPEC-3705 "Unable to call 'jobs.wait': [Errno 2] No such file or directory"
+	    lavacli -i $lab jobs show $job_id > status
+	    grep -q 'state.*Finished' status
+	    if [ $? -ne 0 ];then
+		# retry
+                retry=$(( $retry + 1 ))
+		lavacli -i $lab jobs wait $job_id --timeout 7200 || true
+            else
+                break
+	    fi
+         done
+fi
 lavacli -i $lab jobs show $job_id --yaml > $JOB_STATUS
 state=$(grep ^state: $JOB_STATUS| cut -d' ' -f2)
 if [ "$state" == "Finished" ]; then
@@ -67,6 +87,8 @@ if [ "$state" == "Finished" ]; then
         echo "LAVA job $job_id completed with status: $status"
 fi
 
+set -e
+
 echo "####"
 echo "#### Start: Output from LAVA job $job_id ####"
 echo "####"
-- 
cgit 1.2.3-korg