ci-platform-jjb: try LAVA job in multiple labs

Currently, LAVA jobs are only submitted to a single lab.  This patch
adds the ability look for a matching device-type in a list of
available labs, and submit the job to the first lab with an available
device type.

To accomplish this, the job submission part is removed from
run-test-short.sh, which now only generates the LAVA YAML job.  A new
lava-jobs-submit.sh script is added which looks for an available
device-type in a list of LAVA labs, and submits the job there.

NOTES:
- run-test-short.sh should probably be renamed, but was left
  for now to limit the diff

Change-Id: I95086d89a95f39e2b59fec8a8afada70806c52e5
Signed-off-by: Kevin Hilman <khilman@baylibre.com>

5 files changed, 105 insertions, 28 deletions

diff --git a/jjb/ci-AGL-verify/ci-platform-jjb.yaml b/jjb/ci-AGL-verify/ci-platform-jjb.yaml
index 50b440b0..5d7e02a4 100644
--- a/jjb/ci-AGL-verify/ci-platform-jjb.yaml
+++ b/jjb/ci-AGL-verify/ci-platform-jjb.yaml
@@ -497,6 +497,8 @@
           - ../common/include-agl-run-test-prepare.sh
           - ../common/include-agl-lava-labs-prepare.sh
           - ../common/include-agl-run-test-short.sh
+          - ../common/include-agl-lava-jobs-submit.sh
+
     publishers:
       - naginator:
           max-failed-builds: 2
diff --git a/jjb/ci-apps-verify/ci-apps-jjb.yaml b/jjb/ci-apps-verify/ci-apps-jjb.yaml
index da6e539b..2fbebf3a 100644
--- a/jjb/ci-apps-verify/ci-apps-jjb.yaml
+++ b/jjb/ci-apps-verify/ci-apps-jjb.yaml
@@ -394,7 +394,9 @@
           - ../common/include-agl-repo.sh
           - ../common/include-agl-select.sh
           - ../common/include-agl-run-test-prepare.sh
+          - ../common/include-agl-lava-labs-prepare.sh
           - ../common/include-agl-run-test-short.sh
+          - ../common/include-agl-lava-jobs-submit.sh
 
     publishers:
       - naginator:
diff --git a/jjb/common/include-agl-lava-jobs-submit.sh b/jjb/common/include-agl-lava-jobs-submit.sh
new file mode 100644
index 00000000..8de2c78a
--- /dev/null
+++ b/jjb/common/include-agl-lava-jobs-submit.sh
@@ -0,0 +1,99 @@
+# (c) 2017 Kevin Hilman <khilman@baylibre.com>
+# License GPLv2
+#
+# Submit LAVA YAML job file (default testjob.yaml) to first available LAVA lab
+# with matching device-type
+#
+JOB_FILE=${1:-testjob.yaml}
+
+if [ ! -e $JOB_FILE ]; then
+    echo "ERROR: LAVA job file $JOB_FILE not present."
+    exit 1
+fi
+JOB_BASE=$(basename $JOB_FILE .yaml)
+
+# find device_type from job file
+line=$(grep ^device_type: $JOB_FILE | tr -d '[:space:]')
+device_type=${line/device_type:/}
+echo "Found device_type $device_type in LAVA job $JOB_FILE"
+
+# iterate over available labs
+for lab in "${!labs[@]}"; do
+    val=${labs[$lab]}
+    OFS=${IFS}
+    IFS=';'
+    arr=(${labs[$lab]})
+    IFS=${OFS}
+
+    url=${arr[0]}
+    user=${arr[1]}
+
+    # LAVA URL with username
+    full_url=${url/:\/\//:\/\/${user}\@}
+
+    echo -n "Checking for $device_type at $full_url... "
+    line=$(lava-tool devices-list $full_url |grep $device_type | tr -d '[:space:]')
+    if [ -z "$line" ]; then
+	echo "not found."
+	continue
+    fi
+    IFS='|'
+    arr=($line)
+    device_status=${arr[2]}
+    IFS=${OFS}
+
+    # device is only available if "idle" or "running"
+    device_available=0
+    if [ x"$device_status" = x"idle" ]; then
+	device_available=1
+    elif [ x"$device_status" = x"running" ]; then
+	device_available=1;
+    fi
+
+    if [ $device_available = 0 ]; then
+	echo " Not Available.  Status: $device_status"
+	continue
+    else
+	echo " Found and available.  Status: $device_status"
+    fi
+
+    #
+    # LAVA job submit, get job ID and status from lava-tool output
+    #
+    JOB_STATUS="${JOB_BASE}_${lab}.status"
+    lava-tool submit-job --block $full_url $JOB_FILE |tee $JOB_STATUS
+
+    IFS=':'
+    line=$(grep "job id" $JOB_STATUS | tr -d '[:space:]')
+    arr=($line)
+    job_id=${arr[1]}
+    line=$(grep "Job Status:" $JOB_STATUS | tr -d '[:space:]')
+    arr=($line)
+    status=${arr[1]}
+    IFS=${OFS}
+
+    echo "LAVA job $job_id completed with status: $status"
+
+    echo "####"
+    echo "#### Start: Output from LAVA job $job_id ####"
+    echo "####"
+
+    JOB_OUTPUT="${JOB_BASE}_output.yaml"
+    JOB_LOG="${JOB_BASE}_output.log"
+    curl -s -o $JOB_OUTPUT $full_url/scheduler/job/$job_id/log_file/plain
+    cat $JOB_OUTPUT | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' | tee $JOB_LOG
+
+    echo "####"
+    echo "#### End: Output from LAVA job $job_id ####"
+    echo "####"
+
+    # after one successful submit, we're done
+    if [ x"$status" = x"Complete" ]; then
+	exit 0
+    else
+	continue
+    fi
+done
+
+# if we get here, none of the labs had a successful completion
+exit 1
diff --git a/jjb/common/include-agl-run-test-short.sh b/jjb/common/include-agl-run-test-short.sh
index 26795d46..ee7b6230 100644
--- a/jjb/common/include-agl-run-test-short.sh
+++ b/jjb/common/include-agl-run-test-short.sh
@@ -120,31 +120,3 @@ sed -i -e "s#@REPLACE_KERNEL_CMDLINE@#${DEVICE_KERNEL_CMDLINE}#g" testjob.yaml
 
 cat testjob.yaml
 
-lava-tool submit-job --block https://agl-jenkins-user@lava.automotivelinux.org testjob.yaml | tee .myjob
-
-MYJOB=`cat .myjob | grep "submitted as job" | sed -e "s#submitted as job id: ##g"`
-
-echo "#### JOBID: $MYJOB #####"
-
-( lava-tool job-status https://agl-jenkins-user@lava.automotivelinux.org $MYJOB | tee .status ) || true
-STATUS=`grep "Job Status:" .status | sed -e "s#Job Status: ##g"`
-
-if [ x"Complete" = x"$STATUS"  ] ; then
-    echo "YAY! $STATUS"
-    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
-    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
-
-    # cleanup
-    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
-
-    exit 0
-else
-    echo "Nooooooooo! $STATUS"
-    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
-    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
-
-    # cleanup
-    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
-
-    exit 1
-fi
diff --git a/jjb/test-lava/test-lava.yaml b/jjb/test-lava/test-lava.yaml
index 4f0a20c0..31935843 100644
--- a/jjb/test-lava/test-lava.yaml
+++ b/jjb/test-lava/test-lava.yaml
@@ -78,4 +78,6 @@
           - ../common/include-agl-bitbake-image.sh
           - ../common/include-agl-run-test-prepare.sh
           - ../common/include-agl-run-test-rsync-changeid.sh
+          - ../common/include-agl-lava-labs-prepare.sh
           - ../common/include-agl-run-test-short.sh
+          - ../common/include-agl-lava-jobs-submit.sh