diff options
author | 2017-05-26 22:11:18 +0000 | |
---|---|---|
committer | 2017-07-23 00:48:20 +0000 | |
commit | 2895b7c5dd5e411f2e843518ff1d15064810a915 (patch) | |
tree | 446572063e99ea5175fe7d2b9c9d886699d27066 /jjb | |
parent | 6c80975df08873ddcadb885ac0fd8b1d7655b12e (diff) |
ci-platform-jjb: try LAVA job in multiple labs
Currently, LAVA jobs are only submitted to a single lab. This patch
adds the ability look for a matching device-type in a list of
available labs, and submit the job to the first lab with an available
device type.
To accomplish this, the job submission part is removed from
run-test-short.sh, which now only generates the LAVA YAML job. A new
lava-jobs-submit.sh script is added which looks for an available
device-type in a list of LAVA labs, and submits the job there.
NOTES:
- run-test-short.sh should probably be renamed, but was left
for now to limit the diff
Change-Id: I95086d89a95f39e2b59fec8a8afada70806c52e5
Signed-off-by: Kevin Hilman <khilman@baylibre.com>
Diffstat (limited to 'jjb')
-rw-r--r-- | jjb/ci-AGL-verify/ci-platform-jjb.yaml | 2 | ||||
-rw-r--r-- | jjb/ci-apps-verify/ci-apps-jjb.yaml | 2 | ||||
-rw-r--r-- | jjb/common/include-agl-lava-jobs-submit.sh | 99 | ||||
-rw-r--r-- | jjb/common/include-agl-run-test-short.sh | 28 | ||||
-rw-r--r-- | jjb/test-lava/test-lava.yaml | 2 |
5 files changed, 105 insertions, 28 deletions
diff --git a/jjb/ci-AGL-verify/ci-platform-jjb.yaml b/jjb/ci-AGL-verify/ci-platform-jjb.yaml index 50b440b0..5d7e02a4 100644 --- a/jjb/ci-AGL-verify/ci-platform-jjb.yaml +++ b/jjb/ci-AGL-verify/ci-platform-jjb.yaml @@ -497,6 +497,8 @@ - ../common/include-agl-run-test-prepare.sh - ../common/include-agl-lava-labs-prepare.sh - ../common/include-agl-run-test-short.sh + - ../common/include-agl-lava-jobs-submit.sh + publishers: - naginator: max-failed-builds: 2 diff --git a/jjb/ci-apps-verify/ci-apps-jjb.yaml b/jjb/ci-apps-verify/ci-apps-jjb.yaml index da6e539b..2fbebf3a 100644 --- a/jjb/ci-apps-verify/ci-apps-jjb.yaml +++ b/jjb/ci-apps-verify/ci-apps-jjb.yaml @@ -394,7 +394,9 @@ - ../common/include-agl-repo.sh - ../common/include-agl-select.sh - ../common/include-agl-run-test-prepare.sh + - ../common/include-agl-lava-labs-prepare.sh - ../common/include-agl-run-test-short.sh + - ../common/include-agl-lava-jobs-submit.sh publishers: - naginator: diff --git a/jjb/common/include-agl-lava-jobs-submit.sh b/jjb/common/include-agl-lava-jobs-submit.sh new file mode 100644 index 00000000..8de2c78a --- /dev/null +++ b/jjb/common/include-agl-lava-jobs-submit.sh @@ -0,0 +1,99 @@ +# (c) 2017 Kevin Hilman <khilman@baylibre.com> +# License GPLv2 +# +# Submit LAVA YAML job file (default testjob.yaml) to first available LAVA lab +# with matching device-type +# +JOB_FILE=${1:-testjob.yaml} + +if [ ! -e $JOB_FILE ]; then + echo "ERROR: LAVA job file $JOB_FILE not present." + exit 1 +fi +JOB_BASE=$(basename $JOB_FILE .yaml) + +# find device_type from job file +line=$(grep ^device_type: $JOB_FILE | tr -d '[:space:]') +device_type=${line/device_type:/} +echo "Found device_type $device_type in LAVA job $JOB_FILE" + +# iterate over available labs +for lab in "${!labs[@]}"; do + val=${labs[$lab]} + OFS=${IFS} + IFS=';' + arr=(${labs[$lab]}) + IFS=${OFS} + + url=${arr[0]} + user=${arr[1]} + + # LAVA URL with username + full_url=${url/:\/\//:\/\/${user}\@} + + echo -n "Checking for $device_type at $full_url... " + line=$(lava-tool devices-list $full_url |grep $device_type | tr -d '[:space:]') + if [ -z "$line" ]; then + echo "not found." + continue + fi + IFS='|' + arr=($line) + device_status=${arr[2]} + IFS=${OFS} + + # device is only available if "idle" or "running" + device_available=0 + if [ x"$device_status" = x"idle" ]; then + device_available=1 + elif [ x"$device_status" = x"running" ]; then + device_available=1; + fi + + if [ $device_available = 0 ]; then + echo " Not Available. Status: $device_status" + continue + else + echo " Found and available. Status: $device_status" + fi + + # + # LAVA job submit, get job ID and status from lava-tool output + # + JOB_STATUS="${JOB_BASE}_${lab}.status" + lava-tool submit-job --block $full_url $JOB_FILE |tee $JOB_STATUS + + IFS=':' + line=$(grep "job id" $JOB_STATUS | tr -d '[:space:]') + arr=($line) + job_id=${arr[1]} + line=$(grep "Job Status:" $JOB_STATUS | tr -d '[:space:]') + arr=($line) + status=${arr[1]} + IFS=${OFS} + + echo "LAVA job $job_id completed with status: $status" + + echo "####" + echo "#### Start: Output from LAVA job $job_id ####" + echo "####" + + JOB_OUTPUT="${JOB_BASE}_output.yaml" + JOB_LOG="${JOB_BASE}_output.log" + curl -s -o $JOB_OUTPUT $full_url/scheduler/job/$job_id/log_file/plain + cat $JOB_OUTPUT | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' | tee $JOB_LOG + + echo "####" + echo "#### End: Output from LAVA job $job_id ####" + echo "####" + + # after one successful submit, we're done + if [ x"$status" = x"Complete" ]; then + exit 0 + else + continue + fi +done + +# if we get here, none of the labs had a successful completion +exit 1 diff --git a/jjb/common/include-agl-run-test-short.sh b/jjb/common/include-agl-run-test-short.sh index 26795d46..ee7b6230 100644 --- a/jjb/common/include-agl-run-test-short.sh +++ b/jjb/common/include-agl-run-test-short.sh @@ -120,31 +120,3 @@ sed -i -e "s#@REPLACE_KERNEL_CMDLINE@#${DEVICE_KERNEL_CMDLINE}#g" testjob.yaml cat testjob.yaml -lava-tool submit-job --block https://agl-jenkins-user@lava.automotivelinux.org testjob.yaml | tee .myjob - -MYJOB=`cat .myjob | grep "submitted as job" | sed -e "s#submitted as job id: ##g"` - -echo "#### JOBID: $MYJOB #####" - -( lava-tool job-status https://agl-jenkins-user@lava.automotivelinux.org $MYJOB | tee .status ) || true -STATUS=`grep "Job Status:" .status | sed -e "s#Job Status: ##g"` - -if [ x"Complete" = x"$STATUS" ] ; then - echo "YAY! $STATUS" - curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain - cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' - - # cleanup - #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/ - - exit 0 -else - echo "Nooooooooo! $STATUS" - curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain - cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' - - # cleanup - #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/ - - exit 1 -fi diff --git a/jjb/test-lava/test-lava.yaml b/jjb/test-lava/test-lava.yaml index 4f0a20c0..31935843 100644 --- a/jjb/test-lava/test-lava.yaml +++ b/jjb/test-lava/test-lava.yaml @@ -78,4 +78,6 @@ - ../common/include-agl-bitbake-image.sh - ../common/include-agl-run-test-prepare.sh - ../common/include-agl-run-test-rsync-changeid.sh + - ../common/include-agl-lava-labs-prepare.sh - ../common/include-agl-run-test-short.sh + - ../common/include-agl-lava-jobs-submit.sh |