Revert "jjb/common: remove hwtest.enable files"

Issues with the jobs. 

This reverts commit f28a1005057b096a95ba73bc3bf3e029b835deaa.

Change-Id: Icf75e5b462b0d4aff3c8d2c5806b8ca9270ad22b

3 files changed, 292 insertions, 55 deletions

diff --git a/jjb/common/include-agl-run-test-prepare.sh b/jjb/common/include-agl-run-test-prepare.sh
index 304b74e9..1613bdcf 100644
--- a/jjb/common/include-agl-run-test-prepare.sh
+++ b/jjb/common/include-agl-run-test-prepare.sh
@@ -24,22 +24,63 @@ if [ -e releng-scripts/utils ]; then
     RELENG=$PWD/releng-scripts
 fi
 
-CREATE_ARGS=""
-CREATE_ARGS+="--machine ${MACHINE} "
+ls -alhR meta-agl/templates/machine/${MACHINE}
 
-# If ii's a release build
-if [[ ! -z $RELEASE_BRANCH ]] && [[ ! -z $RELEASE_VERSION ]]; then
-    CREATE_ARGS+="--build-type {release,$RELEASE_BRANCH,$RELEASE_VERSION} "
+#### Find out if machine can be tested.
+#### We keep a stamp file in meta-agl/templates/machine/$MACHINE/test
+if [ ! -d meta-agl/templates/machine/${MACHINE}/test ]; then
+  echo "No HW test templates for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/ and its contents to enable"
+  exit 0
 fi
-# If it's a CI build
-if [[ ! -z $GERRIT_CHANGE_NUMBER ]] && [[ ! -z $GERRIT_PATCHSET_NUMBER ]]; then
-    CREATE_ARGS+="--build-type {ci,$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER} "
+
+if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.enable ] ; then 
+  echo "No HW test for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable to enable"
+  exit 0
+fi
+
+if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.short.enable ] ; then
+  echo "No short HW test for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable.short to enable"
+else
+  eval export ENVFILE=meta-agl/templates/machine/${MACHINE}/test/hwtest.short.environment
+fi
+
+if [ ! -f $ENVFILE ] ; then
+  echo "No short HW test environment file available for ${MACHINE}."
+  echo "Add ${ENVFILE} to enable."
+  exit 1
 fi
 
-export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb`
-export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel`
-export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd`
-export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot`
+# some defaults
+#export DEVICE_ARCH=arm
+#export DEVICE_TYPE=raspberrypi3-uboot
+#export DEVICE_NAME=raspberrypi3
+#export DEVICE_DTB=uImage-bcm2710-rpi-3-b.dtb
+#export DEVICE_KERNEL=uImage
+#export DEVICE_INITRAMFS=initramfs-netboot-image-raspberrypi3.ext4
+#export DEVICE_INITRAMFS_COMPRESSION=false
+#export DEVICE_NBDROOT=agl-demo-platform-raspberrypi3.ext4
+#export DEVICE_NBDROOT_COMPRESSION=false
+#export DEVICE_BOOT_METHOD=u-boot
+#export DEVICE_BOOT_TYPE=bootm
+export DEVICE_URL_PREFIX='https://download.automotivelinux.org/AGL/upload/ci/'
+
+
+# import device defaults. Format 'a=b'
+for i in DEVICE_ARCH DEVICE_TYPE DEVICE_NAME DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_INITRAMFS_COMPRESSION DEVICE_NBDROOT DEVICE_NBDROOT_COMPRESSION DEVICE_BOOT_METHOD DEVICE_BOOT_TYPE DEVICE_URL_PREFIX DEVICE_QEMU_ARGS DEVICE_KERNEL_CMDLINE; do
+    if grep -q ^$i $ENVFILE ; then
+        X=$(grep $i $ENVFILE | sed -e "s#${i}=##g" -e "s#;.*##g")
+        eval export ${i}=${X}
+    else
+        echo "Variable ${i} missing in environment file $ENVFILE"
+    fi
+done
+
+echo "Resulting values:"
+set | grep DEVICE_
+
 
 # echo NEXT is rsync
 #exit 0
diff --git a/jjb/common/include-agl-run-test-short-release.sh b/jjb/common/include-agl-run-test-short-release.sh
index 2af1cc4d..f6388080 100644
--- a/jjb/common/include-agl-run-test-short-release.sh
+++ b/jjb/common/include-agl-run-test-short-release.sh
@@ -5,32 +5,118 @@
 ## Run SHORT CI test
 ################################################################################
 
-set -x
+
+#set -x
 
 echo "## ${MACHINE} ##"
 cd $REPODIR
 
-__MACHINE=${MACHINE%-nogfx} # releng doesn't care
-echo "# LAVA lab target: ${LAVA_LAB} #"
-# releng-scripts depends on jinja2 >= 2.9
-echo "## Check version of python-jinja2: "
-pip show jinja2
-CREATE_ARGS=""
-CREATE_ARGS+="--machine ${releng_device} "
-CREATE_ARGS+="--url ci "
-[[ ! -z $RELEASE_BRANCH ]] && CREATE_ARGS+="--branch $RELEASE_BRANCH "
-[[ ! -z $RELEASE_VERSION ]] && CREATE_ARGS+="--version $RELEASE_VERSION "
-BUILD_VERSION="AGL-gerrit-$RELEASE_BRANCH-$RELEASE_VERSION"
-[[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
-CREATE_ARGS+="--name AGL-gerrit "
-CREATE_ARGS+="--build-version $BUILD_VERSION "
-CREATE_ARGS+="--callback-from ${LAVA_LAB} "
-CREATE_ARGS+="--test all"
-$RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
-
-if [ $? != 0 ]; then
-    echo "ERROR: Machine ${$__MACHINE}: LAVA job creation failed."
+cat <<EOF > testjob.yaml
+# Your first LAVA JOB definition for a $MACHINE board
+device_type: @REPLACE_DEVICE_TYPE@
+job_name: AGL-${RELEASE_VERSION}-build-${BUILD_NUMBER}
+
+protocols:
+  lava-xnbd:
+    port: auto
+
+timeouts:
+  job:
+    minutes: 30
+  action:
+    minutes: 15
+  connection:
+    minutes: 5
+priority: medium
+visibility: public
+
+# ACTION_BLOCK
+actions:
+- deploy:
+    timeout:
+      minutes: 15
+    to: nbd
+    dtb:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_DTB@'
+    kernel:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
+    initrd:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
+      allow_modify: false
+    nbdroot:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_NBDROOT@'
+      compression: @REPLACE_NBDROOT_COMPRESSION@
+    os: debian
+    failure_retry: 2
+
+# BOOT_BLOCK
+- boot:
+    method: @REPLACE_BOOT_METHOD@
+    commands: nbd
+    type: @REPLACE_BOOT_TYPE@
+    prompts: ["root@@REPLACE_MACHINE@:~"]
+    auto_login:
+      login_prompt: "login:"
+      username: root
+    transfer_overlay:
+      download_command: wget
+      unpack_command: tar -C / -xvpf
+- test:
+    definitions:
+    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+      from: git
+      path: test-suites/short-smoke/busybox.yaml
+      name: busybox
+    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+      from: git
+      path: test-suites/short-smoke/smoke-tests-basic.yaml
+      name: smoke-tests-basic
+    - repository: https://git.automotivelinux.org/src/qa-testdefinitions
+      from: git
+      path: test-suites/short-smoke/service-check.yaml
+      name: service-check
+EOF
+
+CHID=${RELEASE_BRANCH}/${RELEASE_VERSION}/${MACHINE}
+# REPLACE_DEVICE_TYPE
+sed -i -e "s#@REPLACE_DEVICE_TYPE@#${DEVICE_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_DTB@#${CHID}/${DEVICE_DTB}#g" testjob.yaml
+sed -i -e "s#@REPLACE_KERNEL@#${CHID}/${DEVICE_KERNEL}#g" testjob.yaml
+sed -i -e "s#@REPLACE_INITRAMFS@#${CHID}/${DEVICE_INITRAMFS}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT@#${CHID}/${DEVICE_NBDROOT}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT_COMPRESSION@#${DEVICE_NBDROOT_COMPRESSION}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_METHOD@#${DEVICE_BOOT_METHOD}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_TYPE@#${DEVICE_BOOT_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_MACHINE@#${DEVICE_NAME}#g" testjob.yaml
+sed -i -e "s#@REPLACE_URL_PREFIX@#${DEVICE_URL_PREFIX}#g" testjob.yaml
+
+cat testjob.yaml
+
+lava-tool submit-job --block https://agl-jenkins-user@lava.automotivelinux.org testjob.yaml | tee .myjob
+
+MYJOB=`cat .myjob | grep "submitted as job" | sed -e "s#submitted as job id: ##g"`
+
+echo "#### JOBID: $MYJOB #####"
+
+( lava-tool job-status https://agl-jenkins-user@lava.automotivelinux.org $MYJOB | tee .status ) || true
+STATUS=`grep "Job Status:" .status | sed -e "s#Job Status: ##g"`
+
+if [ x"Complete" = x"$STATUS"  ] ; then
+    echo "YAY! $STATUS"
+    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
+    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
+
+    # cleanup
+    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
+
     exit 0
 else
-    cat testjob.yaml
+    echo "Nooooooooo! $STATUS"
+    curl -o plain_output.yaml https://lava.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
+    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
+
+    # cleanup
+    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
+    # exit 1
+    exit 0
 fi
diff --git a/jjb/common/include-agl-run-test-short.sh b/jjb/common/include-agl-run-test-short.sh
index 58273b99..deeaf3c5 100644
--- a/jjb/common/include-agl-run-test-short.sh
+++ b/jjb/common/include-agl-run-test-short.sh
@@ -10,27 +10,137 @@ set -x
 echo "## ${MACHINE} ##"
 cd $REPODIR
 
+# WIP: use $RELENG for subset of boards
 __MACHINE=${MACHINE%-nogfx} # releng doesn't care
-echo "# LAVA lab target: ${LAVA_LAB} #"
-# releng-scripts depends on jinja2 >= 2.9
-echo "## Check version of python-jinja2: "
-pip show jinja2
-CREATE_ARGS=""
-CREATE_ARGS+="--machine ${releng_device} "
-CREATE_ARGS+="--url ci "
-[[ ! -z $GERRIT_CHANGE_NUMBER ]] && CREATE_ARGS+="--changeid $GERRIT_CHANGE_NUMBER "
-[[ ! -z $GERRIT_PATCHSET_NUMBER ]] && CREATE_ARGS+="--patchset $GERRIT_PATCHSET_NUMBER "
-BUILD_VERSION="AGL-gerrit-$GERRIT_CHANGE_NUMBER-$GERRIT_PATCHSET_NUMBER"
-[[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
-CREATE_ARGS+="--name AGL-gerrit "
-CREATE_ARGS+="--build-version $BUILD_VERSION "
-CREATE_ARGS+="--callback-from ${LAVA_LAB} "
-CREATE_ARGS+="--test all"
-$RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
-
-if [ $? != 0 ]; then
-    echo "ERROR: Machine ${$__MACHINE}: LAVA job creation failed."
-    exit 0
-else
+if [ ! -z $RELENG ]; then
+    echo "# LAVA lab target: ${LAVA_LAB} #"
+    # releng-scripts depends on jinja2 >= 2.9
+    echo "## Check version of python-jinja2: "
+    pip show jinja2
+    CREATE_ARGS=""
+    CREATE_ARGS+="--machine ${releng_device} "
+    CREATE_ARGS+="--url ci "
+    [[ ! -z $GERRIT_CHANGE_NUMBER ]] && CREATE_ARGS+="--changeid $GERRIT_CHANGE_NUMBER "
+    [[ ! -z $GERRIT_PATCHSET_NUMBER ]] && CREATE_ARGS+="--patchset $GERRIT_PATCHSET_NUMBER "
+    BUILD_VERSION="AGL-gerrit-$GERRIT_CHANGE_NUMBER-$GERRIT_PATCHSET_NUMBER"
+    [[ -e output/repo-manifest-r.txt.sha1 ]] && BUILD_VERSION+="-$(cat output/repo-manifest-r.txt.sha1)"
+    CREATE_ARGS+="--name AGL-gerrit "
+    CREATE_ARGS+="--build-version $BUILD_VERSION "
+    CREATE_ARGS+="--callback-from ${LAVA_LAB} "
+    CREATE_ARGS+="--test all"
+    $RELENG/utils/create-jobs.py ${CREATE_ARGS} > testjob.yaml
     cat testjob.yaml
+else
+
+cat <<EOF > testjob.yaml
+# Your first LAVA JOB definition for a $MACHINE board
+device_type: @REPLACE_DEVICE_TYPE@
+job_name: AGL-short-smoke
+
+timeouts:
+  job:
+    minutes: 30
+  action:
+    minutes: 15
+  connection:
+    minutes: 5
+priority: medium
+visibility: public
+EOF
+
+if [ ${DEVICE_BOOT_METHOD} = "u-boot" ]; then
+cat <<EOF >> testjob.yaml
+
+protocols:
+  lava-xnbd:
+    port: auto
+
+# ACTION_BLOCK
+actions:
+- deploy:
+    timeout:
+      minutes: 15
+    to: nbd
+    dtb:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_DTB@'
+    kernel:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
+    initrd:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
+      allow_modify: false
+    nbdroot:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_NBDROOT@'
+      compression: @REPLACE_NBDROOT_COMPRESSION@
+    os: debian
+    failure_retry: 2
+
+# BOOT_BLOCK
+- boot:
+    timeout:
+      minutes: 10
+    method: @REPLACE_BOOT_METHOD@
+    commands: nbd
+    type: @REPLACE_BOOT_TYPE@
+    prompts: ["root@@REPLACE_MACHINE@:~"]
+    auto_login:
+      login_prompt: "login:"
+      username: root
+
+EOF
+fi
+
+if [ ${DEVICE_BOOT_METHOD} = "qemu" ]; then
+cat <<EOF >>testjob.yaml
+context:
+  no_kvm: false
+  arch: @REPLACE_DEVICE_ARCH@
+  extra_options: [@REPLACE_QEMU_ARGS@]
+ 
+actions:
+- deploy:
+    timeout:
+      minutes: 15
+    to: tmpfs
+    os: oe
+    images:
+        kernel:
+          image_arg: '-kernel {kernel} -append @REPLACE_KERNEL_CMDLINE@'
+          url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
+        ramdisk:
+          image_arg: '-drive format=raw,file={ramdisk}'
+          url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
+          compression: @REPLACE_INITRAMFS_COMPRESSION@
+
+- boot:
+    timeout:
+      minutes: 10
+    method: @REPLACE_BOOT_METHOD@
+    media: tmpfs
+    prompts: ["root@@REPLACE_MACHINE@:~"]
+    auto_login:
+      login_prompt: "login:"
+      username: root
+
+EOF
+fi
+
+CHID=${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}/${MACHINE}
+# REPLACE_DEVICE_TYPE
+sed -i -e "s#@REPLACE_DEVICE_ARCH@#${DEVICE_ARCH}#g" testjob.yaml
+sed -i -e "s#@REPLACE_DEVICE_TYPE@#${DEVICE_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_DTB@#${CHID}/${DEVICE_DTB}#g" testjob.yaml
+sed -i -e "s#@REPLACE_KERNEL@#${CHID}/${DEVICE_KERNEL}#g" testjob.yaml
+sed -i -e "s#@REPLACE_INITRAMFS@#${CHID}/${DEVICE_INITRAMFS}#g" testjob.yaml
+sed -i -e "s#@REPLACE_INITRAMFS_COMPRESSION@#${DEVICE_INITRAMFS_COMPRESSION}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT@#${CHID}/${DEVICE_NBDROOT}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT_COMPRESSION@#${DEVICE_NBDROOT_COMPRESSION}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_METHOD@#${DEVICE_BOOT_METHOD}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_TYPE@#${DEVICE_BOOT_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_MACHINE@#${DEVICE_NAME}#g" testjob.yaml
+sed -i -e "s#@REPLACE_URL_PREFIX@#${DEVICE_URL_PREFIX}#g" testjob.yaml
+sed -i -e "s#@REPLACE_QEMU_ARGS@#${DEVICE_QEMU_ARGS}#g" testjob.yaml
+sed -i -e "s#@REPLACE_KERNEL_CMDLINE@#${DEVICE_KERNEL_CMDLINE}#g" testjob.yaml
+
+cat testjob.yaml
+
 fi