From d212724c5b1fbf323b71ad09ca2d069b15b27995 Mon Sep 17 00:00:00 2001
From: Corentin LABBE <clabbe@baylibre.com>
Date: Wed, 22 Apr 2020 10:52:12 +0200
Subject: SPEC-3004: resubmit failling jobs for rpi4

The RPI4 fail often in uboot phase due to network timeouts.
This patchs adds a way to restart failled jobs for rpi4.
For the moment we restart all failed jobs, a subsequent patch will add a
check on error case.

Bug-AGL: SPEC-3004

Change-Id: I556ec30d1af4adfa57ce3553f79f4152d335e158
Signed-off-by: Corentin LABBE <clabbe@baylibre.com>
---
 jjb/common/include-agl-lava-jobs-submit.sh | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

(limited to 'jjb')

diff --git a/jjb/common/include-agl-lava-jobs-submit.sh b/jjb/common/include-agl-lava-jobs-submit.sh
index 912487dc..f21c6ac6 100644
--- a/jjb/common/include-agl-lava-jobs-submit.sh
+++ b/jjb/common/include-agl-lava-jobs-submit.sh
@@ -36,6 +36,12 @@ if [ $? -ne 0 ]; then
 	echo "ERROR: job submission error"
 	exit 123
 fi
+
+# check if the job ends
+# resend the jobs in some error case
+# arg1 is the retry count
+wait_jobs() {
+retrycount=$1
 # Printing the job URL in the log
 echo "THe job id is: ${job_id}"
 JOB_URL="${url}scheduler/job/${job_id}"
@@ -54,7 +60,27 @@ state=$(grep ^state: $JOB_STATUS| cut -d' ' -f2)
 if [ "$state" == "Finished" ]; then
         status=$(grep ^health: $JOB_STATUS| cut -d' ' -f2)
         echo "LAVA job $job_id completed with status: $status"
+	# Check for resend RPI4
+	grep -q 'bcm2711-rpi-4-b' $JOB_STATUS
+	if [ $? -eq 0 ];then
+		if [ "$status" == 'Incomplete' ];then
+			if [ $retrycount -ge 1 ];then
+				# resubmit the jobs, overwritting the job_id
+				job_id=$(lavacli -i $lab jobs submit $JOB_FILE_NEW)
+				if [ $? -ne 0 ]; then
+					echo "ERROR: job submission error"
+					return 1
+				fi
+				wait_jobs $(($retrycount-1))
+			else
+				echo "INFO: too many retries"
+			fi
+		fi
+	fi
 fi
+}
+
+wait_jobs 2
 
 echo "####"
 echo "#### Start: Output from LAVA job $job_id ####"
-- 
cgit 1.2.3-korg