From cc12309dd23b7f76d310c69d4412ab7afb33ef1f Mon Sep 17 00:00:00 2001
From: Loys Ollivier <lollivier@baylibre.com>
Date: Fri, 1 Dec 2017 10:59:03 +0100
Subject: jjb/common: remove hwtest.enable files v2

releng-scripts now supports LAVA job generation for all boards.

Remove the check in run-test-prepare.sh on whether or not to create LAVA
test jobs. Replace this test by several exports using releng-scripts
job-prereq tool. These exports will be used to rsync the files to the CI
upload server.
If the calls to job-prereq returns an error, make the script fail
gracefully.

Transition all platforms in run-test-short.sh to use releng-scripts.
Transition all platforms in run-test-short-release.sh to use releng scripts.

JIRA: SPEC-1144

Change-Id: I1e0e6d39b986770adcdfcf92c5d589a7fbe99a33
Signed-off-by: Loys Ollivier <lollivier@baylibre.com>
---
 jjb/common/include-agl-run-test-prepare.sh       |  69 +++-------
 jjb/common/include-agl-run-test-short-release.sh | 130 ++++---------------
 jjb/common/include-agl-run-test-short.sh         | 152 ++++-------------------
 3 files changed, 60 insertions(+), 291 deletions(-)

(limited to 'jjb/common')

diff --git a/jjb/common/include-agl-run-test-prepare.sh b/jjb/common/include-agl-run-test-prepare.sh
index 1613bdcf..2c6611d9 100644
--- a/jjb/common/include-agl-run-test-prepare.sh
+++ b/jjb/common/include-agl-run-test-prepare.sh
@@ -24,63 +24,28 @@ if [ -e releng-scripts/utils ]; then
     RELENG=$PWD/releng-scripts
 fi
 
-ls -alhR meta-agl/templates/machine/${MACHINE}
+CREATE_ARGS=""
+CREATE_ARGS+="--machine ${MACHINE} "
 
-#### Find out if machine can be tested.
-#### We keep a stamp file in meta-agl/templates/machine/$MACHINE/test
-if [ ! -d meta-agl/templates/machine/${MACHINE}/test ]; then
-  echo "No HW test templates for ${MACHINE} configured."
-  echo "Add meta-agl/templates/machine/${MACHINE}/test/ and its contents to enable"
-  exit 0
+# If it's a release build
+if [[ $RELEASE_BRANCH ]] && [[ $RELEASE_VERSION ]]; then
+    CREATE_ARGS+="--build-type release $RELEASE_BRANCH $RELEASE_VERSION "
 fi
-
-if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.enable ] ; then 
-  echo "No HW test for ${MACHINE} configured."
-  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable to enable"
-  exit 0
+# If it's a CI build
+if [[ $GERRIT_CHANGE_NUMBER ]] && [[ $GERRIT_PATCHSET_NUMBER ]]; then
+    CREATE_ARGS+="--build-type ci $GERRIT_CHANGE_NUMBER $GERRIT_PATCHSET_NUMBER "
 fi
 
-if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.short.enable ] ; then
-  echo "No short HW test for ${MACHINE} configured."
-  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable.short to enable"
-else
-  eval export ENVFILE=meta-agl/templates/machine/${MACHINE}/test/hwtest.short.environment
+$RELENG/utils/job-prereq.py ${CREATE_ARGS}
+if [ $? -ne 0 ]; then
+    echo "Board not supported by releng-scripts: job-prereq.py"
+    exit 0
 fi
-
-if [ ! -f $ENVFILE ] ; then
-  echo "No short HW test environment file available for ${MACHINE}."
-  echo "Add ${ENVFILE} to enable."
-  exit 1
-fi
-
-# some defaults
-#export DEVICE_ARCH=arm
-#export DEVICE_TYPE=raspberrypi3-uboot
-#export DEVICE_NAME=raspberrypi3
-#export DEVICE_DTB=uImage-bcm2710-rpi-3-b.dtb
-#export DEVICE_KERNEL=uImage
-#export DEVICE_INITRAMFS=initramfs-netboot-image-raspberrypi3.ext4
-#export DEVICE_INITRAMFS_COMPRESSION=false
-#export DEVICE_NBDROOT=agl-demo-platform-raspberrypi3.ext4
-#export DEVICE_NBDROOT_COMPRESSION=false
-#export DEVICE_BOOT_METHOD=u-boot
-#export DEVICE_BOOT_TYPE=bootm
-export DEVICE_URL_PREFIX='https://download.automotivelinux.org/AGL/upload/ci/'
-
-
-# import device defaults. Format 'a=b'
-for i in DEVICE_ARCH DEVICE_TYPE DEVICE_NAME DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_INITRAMFS_COMPRESSION DEVICE_NBDROOT DEVICE_NBDROOT_COMPRESSION DEVICE_BOOT_METHOD DEVICE_BOOT_TYPE DEVICE_URL_PREFIX DEVICE_QEMU_ARGS DEVICE_KERNEL_CMDLINE; do
-    if grep -q ^$i $ENVFILE ; then
-        X=$(grep $i $ENVFILE | sed -e "s#${i}=##g" -e "s#;.*##g")
-        eval export ${i}=${X}
-    else
-        echo "Variable ${i} missing in environment file $ENVFILE"
-    fi
-done
-
-echo "Resulting values:"
-set | grep DEVICE_
-
+# First call to job-prereq suceeded, other should suceed as they use the same args.
+export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb`
+export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel`
+export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd`
+export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot`
 
 # echo NEXT is rsync
 #exit 0
diff --git a/jjb/common/include-agl-run-test-short-release.sh b/jjb/common/include-agl-run-test-short-release.sh
index f6388080..2af1cc4d 100644
--- a/jjb/common/include-agl-run-test-short-release.sh
+++ b/jjb/common/include-agl-run-test-short-release.sh
@@ -5,118 +5,32 @@
 ## Run SHORT CI test
 ################################################################################
 
-
-#set -x
+set -x
 
 echo "## ${MACHINE} ##"
 cd $REPODIR
 
-cat <<EOF > testjob.yaml
-# Your first LAVA JOB definition for a $MACHINE board
-device_type: @REPLACE_DEVICE_TYPE@
-job_name: AGL-${RELEASE_VERSION}-build-${BUILD_NUMBER}
-
-protocols:
-  lava-xnbd:
-    port: auto
-
-timeouts:
-  job:
-    minutes: 30
-  action:
-    minutes: 15
-  connection:
-    minutes: 5
-priority: medium
-visibility: public
-
-# ACTION_BLOCK
-actions:
-- deploy:
-    timeout:
-      minutes: 15
-    to: nbd
-    dtb:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_DTB@'
-    kernel:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
-    initrd:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
-      allow_modify: false
-    nbdroot:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_NBDROOT@'
-      compression: @REPLACE_NBDROOT_COMPRESSION@
-    os: debian
-    failure_retry: 2
-
-# BOOT_BLOCK
-- boot:
-    method: @REPLACE_BOOT_METHOD@
-    commands: nbd
-    type: @REPLACE_BOOT_TYPE@
-    prompts: ["root@@REPLACE_MACHINE@:~"]
-    auto_login:
-      login_prompt: "login:"
-      username: root
-    transfer_overlay:
-      download_command: wget
-      unpack_command: tar -C / -xvpf
-- test:
-    definitions:
-    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
-      from: git
-      path: test-suites/short-smoke/busybox.yaml
-      name: busybox
-    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
-      from: git
-      path: test-suites/short-smoke/smoke-tests-basic.yaml
-      name: smoke-tests-basic
-    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
-      from: git
-      path: test-suites/short-smoke/service-check.yaml
-      name: service-check
-EOF
-
-CHID=${RELEASE_BRANCH}/${RELEASE_VERSION}/${MACHINE}
-# REPLACE_DEVICE_TYPE
-sed -i -e "s#@REPLACE_DEVICE_TYPE@#${DEVICE_TYPE}#g" testjob.yaml
-sed -i -e "s#@REPLACE_DTB@#${CHID}/${DEVICE_DTB}#g" testjob.yaml
-sed -i -e "s#@REPLACE_KERNEL@#${CHID}/${DEVICE_KERNEL}#g" testjob.yaml
-sed -i -e "s#@REPLACE_INITRAMFS@#${CHID}/${DEVICE_INITRAMFS}#g" testjob.yaml
-sed -i -e "s#@REPLACE_NBDROOT@#${CHID}/${DEVICE_NBDROOT}#g" testjob.yaml
-sed -i -e "s#@REPLACE_NBDROOT_COMPRESSION@#${DEVICE_NBDROOT_COMPRESSION}#g" testjob.yaml
-sed -i -e "s#@REPLACE_BOOT_METHOD@#${DEVICE_BOOT_METHOD}#g" testjob.yaml
-sed -i -e "s#@REPLACE_BOOT_TYPE@#${DEVICE_BOOT_TYPE}#g" testjob.yaml
-sed -i -e "s#@REPLACE_MACHINE@#${DEVICE_NAME}#g" testjob.yaml
-sed -i -e "s#@REPLACE_URL_PREFIX@#${DEVICE_URL_PREFIX}#g" testjob.yaml
-
-cat testjob.yaml
-
-lava-tool submit-job --block https://agl-jenkins-user@lava.automotivelinux.org testjob.yaml | tee .myjob
-
-MYJOB=`cat .myjob | grep "submitted as job" | sed -e "s#submitted as job id: ##g"`
-
-echo "#### JOBID: $MYJOB #####"
-
-( lava-tool job-status https://agl-jenkins-user@lava.automotivelinux.org $MYJOB | tee .status ) || true
-STATUS=`grep "Job Status:" .status | sed -e "s#Job Status: ##g"`
-
-if [ x"Complete" = x"$STATUS"  ] ; then
-    echo "YAY! $STATUS"
-    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
-    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
-
-    # cleanup
-    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
-
+__MACHINE=${MACHINE%-nogfx} # releng doesn't care
+echo "# LAVA lab target: ${LAVA_LAB} #"
+# releng-scripts depends on jinja2 >= 2.9
+echo "## Check version of python-jinja2: "
+pip show jinja2
+CREATE_ARGS=""
+CREATE_ARGS+="--machine ${releng_device} "
+CREATE_ARGS+="--url ci "
+[[ ! -z $RELEASE_BRANCH ]] && CREATE_ARGS+="--branch $RELEASE_BRANCH "
+[[ ! -z $RELEASE_VERSION ]] && CREATE_ARGS+="--version $RELEASE_VERSION "
+BUILD_VERSION="AGL-gerrit-$RELEASE_BRANCH-$RELEASE_VERSION"
+[[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
+CREATE_ARGS+="--name AGL-gerrit "
+CREATE_ARGS+="--build-version $BUILD_VERSION "
+CREATE_ARGS+="--callback-from ${LAVA_LAB} "
+CREATE_ARGS+="--test all"
+$RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
+
+if [ $? != 0 ]; then
+    echo "ERROR: Machine ${$__MACHINE}: LAVA job creation failed."
     exit 0
 else
-    echo "Nooooooooo! $STATUS"
-    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
-    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
-
-    # cleanup
-    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
-    # exit 1
-    exit 0
+    cat testjob.yaml
 fi
diff --git a/jjb/common/include-agl-run-test-short.sh b/jjb/common/include-agl-run-test-short.sh
index deeaf3c5..58273b99 100644
--- a/jjb/common/include-agl-run-test-short.sh
+++ b/jjb/common/include-agl-run-test-short.sh
@@ -10,137 +10,27 @@ set -x
 echo "## ${MACHINE} ##"
 cd $REPODIR
 
-# WIP: use $RELENG for subset of boards
 __MACHINE=${MACHINE%-nogfx} # releng doesn't care
-if [ ! -z $RELENG ]; then
-    echo "# LAVA lab target: ${LAVA_LAB} #"
-    # releng-scripts depends on jinja2 >= 2.9
-    echo "## Check version of python-jinja2: "
-    pip show jinja2
-    CREATE_ARGS=""
-    CREATE_ARGS+="--machine ${releng_device} "
-    CREATE_ARGS+="--url ci "
-    [[ ! -z $GERRIT_CHANGE_NUMBER ]] && CREATE_ARGS+="--changeid $GERRIT_CHANGE_NUMBER "
-    [[ ! -z $GERRIT_PATCHSET_NUMBER ]] && CREATE_ARGS+="--patchset $GERRIT_PATCHSET_NUMBER "
-    BUILD_VERSION="AGL-gerrit-$GERRIT_CHANGE_NUMBER-$GERRIT_PATCHSET_NUMBER"
-    [[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
-    CREATE_ARGS+="--name AGL-gerrit "
-    CREATE_ARGS+="--build-version $BUILD_VERSION "
-    CREATE_ARGS+="--callback-from ${LAVA_LAB} "
-    CREATE_ARGS+="--test all"
-    $RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
-    cat testjob.yaml
+echo "# LAVA lab target: ${LAVA_LAB} #"
+# releng-scripts depends on jinja2 >= 2.9
+echo "## Check version of python-jinja2: "
+pip show jinja2
+CREATE_ARGS=""
+CREATE_ARGS+="--machine ${releng_device} "
+CREATE_ARGS+="--url ci "
+[[ ! -z $GERRIT_CHANGE_NUMBER ]] && CREATE_ARGS+="--changeid $GERRIT_CHANGE_NUMBER "
+[[ ! -z $GERRIT_PATCHSET_NUMBER ]] && CREATE_ARGS+="--patchset $GERRIT_PATCHSET_NUMBER "
+BUILD_VERSION="AGL-gerrit-$GERRIT_CHANGE_NUMBER-$GERRIT_PATCHSET_NUMBER"
+[[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
+CREATE_ARGS+="--name AGL-gerrit "
+CREATE_ARGS+="--build-version $BUILD_VERSION "
+CREATE_ARGS+="--callback-from ${LAVA_LAB} "
+CREATE_ARGS+="--test all"
+$RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
+
+if [ $? != 0 ]; then
+    echo "ERROR: Machine ${$__MACHINE}: LAVA job creation failed."
+    exit 0
 else
-
-cat <<EOF > testjob.yaml
-# Your first LAVA JOB definition for a $MACHINE board
-device_type: @REPLACE_DEVICE_TYPE@
-job_name: AGL-short-smoke
-
-timeouts:
-  job:
-    minutes: 30
-  action:
-    minutes: 15
-  connection:
-    minutes: 5
-priority: medium
-visibility: public
-EOF
-
-if [ ${DEVICE_BOOT_METHOD} = "u-boot" ]; then
-cat <<EOF >> testjob.yaml
-
-protocols:
-  lava-xnbd:
-    port: auto
-
-# ACTION_BLOCK
-actions:
-- deploy:
-    timeout:
-      minutes: 15
-    to: nbd
-    dtb:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_DTB@'
-    kernel:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
-    initrd:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
-      allow_modify: false
-    nbdroot:
-      url: '@REPLACE_URL_PREFIX@/@REPLACE_NBDROOT@'
-      compression: @REPLACE_NBDROOT_COMPRESSION@
-    os: debian
-    failure_retry: 2
-
-# BOOT_BLOCK
-- boot:
-    timeout:
-      minutes: 10
-    method: @REPLACE_BOOT_METHOD@
-    commands: nbd
-    type: @REPLACE_BOOT_TYPE@
-    prompts: ["root@@REPLACE_MACHINE@:~"]
-    auto_login:
-      login_prompt: "login:"
-      username: root
-
-EOF
-fi
-
-if [ ${DEVICE_BOOT_METHOD} = "qemu" ]; then
-cat <<EOF >>testjob.yaml
-context:
-  no_kvm: false
-  arch: @REPLACE_DEVICE_ARCH@
-  extra_options: [@REPLACE_QEMU_ARGS@]
- 
-actions:
-- deploy:
-    timeout:
-      minutes: 15
-    to: tmpfs
-    os: oe
-    images:
-        kernel:
-          image_arg: '-kernel {kernel} -append @REPLACE_KERNEL_CMDLINE@'
-          url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
-        ramdisk:
-          image_arg: '-drive format=raw,file={ramdisk}'
-          url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
-          compression: @REPLACE_INITRAMFS_COMPRESSION@
-
-- boot:
-    timeout:
-      minutes: 10
-    method: @REPLACE_BOOT_METHOD@
-    media: tmpfs
-    prompts: ["root@@REPLACE_MACHINE@:~"]
-    auto_login:
-      login_prompt: "login:"
-      username: root
-
-EOF
-fi
-
-CHID=${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}/${MACHINE}
-# REPLACE_DEVICE_TYPE
-sed -i -e "s#@REPLACE_DEVICE_ARCH@#${DEVICE_ARCH}#g" testjob.yaml
-sed -i -e "s#@REPLACE_DEVICE_TYPE@#${DEVICE_TYPE}#g" testjob.yaml
-sed -i -e "s#@REPLACE_DTB@#${CHID}/${DEVICE_DTB}#g" testjob.yaml
-sed -i -e "s#@REPLACE_KERNEL@#${CHID}/${DEVICE_KERNEL}#g" testjob.yaml
-sed -i -e "s#@REPLACE_INITRAMFS@#${CHID}/${DEVICE_INITRAMFS}#g" testjob.yaml
-sed -i -e "s#@REPLACE_INITRAMFS_COMPRESSION@#${DEVICE_INITRAMFS_COMPRESSION}#g" testjob.yaml
-sed -i -e "s#@REPLACE_NBDROOT@#${CHID}/${DEVICE_NBDROOT}#g" testjob.yaml
-sed -i -e "s#@REPLACE_NBDROOT_COMPRESSION@#${DEVICE_NBDROOT_COMPRESSION}#g" testjob.yaml
-sed -i -e "s#@REPLACE_BOOT_METHOD@#${DEVICE_BOOT_METHOD}#g" testjob.yaml
-sed -i -e "s#@REPLACE_BOOT_TYPE@#${DEVICE_BOOT_TYPE}#g" testjob.yaml
-sed -i -e "s#@REPLACE_MACHINE@#${DEVICE_NAME}#g" testjob.yaml
-sed -i -e "s#@REPLACE_URL_PREFIX@#${DEVICE_URL_PREFIX}#g" testjob.yaml
-sed -i -e "s#@REPLACE_QEMU_ARGS@#${DEVICE_QEMU_ARGS}#g" testjob.yaml
-sed -i -e "s#@REPLACE_KERNEL_CMDLINE@#${DEVICE_KERNEL_CMDLINE}#g" testjob.yaml
-
-cat testjob.yaml
-
+    cat testjob.yaml
 fi
-- 
cgit 1.2.3-korg