Add new ci jobs using multijob and control slave.

Change-Id: I4cbfe33ec05b4de1558834a902fe7b55bea4c31e
Signed-off-by: Jan-Simon Möller <jsmoeller@linuxfoundation.org>

10 files changed, 316 insertions, 107 deletions

diff --git a/jjb/common/include-agl-build.sh b/jjb/common/.include-agl-build.sh
index 25873455..25873455 100644
--- a/jjb/common/include-agl-build.sh
+++ b/jjb/common/.include-agl-build.sh
diff --git a/jjb/common/include-agl-auto-conf-cvscheck.sh b/jjb/common/include-agl-auto-conf-cvscheck.sh
new file mode 100644
index 00000000..a5dc135a
--- /dev/null
+++ b/jjb/common/include-agl-auto-conf-cvscheck.sh
@@ -0,0 +1,15 @@
+# (c) 2017 Jan-Simon Moeller dl9pf(at)gmx.de
+# License GPLv2
+
+################################################################################
+## Add archiver
+################################################################################
+
+# only morty and newer support the cve-check, so check for master branch
+
+if [ x"master" = x"$GERRIT_BRANCH" ] ; then
+
+# archive sources within  tmp/deploy/
+echo '' >> conf/auto.conf
+echo 'INHERIT += "cve-check"' >> conf/auto.conf
+fi
\ No newline at end of file
diff --git a/jjb/common/include-agl-bitbake-image-fake.sh b/jjb/common/include-agl-bitbake-image-fake.sh
new file mode 100644
index 00000000..47eca230
--- /dev/null
+++ b/jjb/common/include-agl-bitbake-image-fake.sh
@@ -0,0 +1,37 @@
+# (c) 2016 Jan-Simon Moeller dl9pf(at)gmx.de
+# License GPLv2
+
+################################################################################
+## bitbake the image
+################################################################################
+
+# finally, build the agl-demo-platform (we retry to catch the tar-native bug)
+#( ( bitbake $TARGETIMAGE ) || ( echo '## Failed once, retry ..  ##' ; sync ; sleep 2 ;  bitbake $TARGETIMAGE ) ) || ( echo '## Failed again, bail out ...' ; exit 1 ) 
+
+if [ x"${MACHINE}" = x"raspberrypi3" ] ; then
+export DEVICE_TYPE=raspberrypi3-uboot
+export DEVICE_NAME=raspberrypi3
+export DEVICE_DTB=uImage-bcm2710-rpi-3-b.dtb
+export DEVICE_KERNEL=uImage
+export DEVICE_INITRAMFS=initramfs-netboot-image-raspberrypi3.ext4.gz.u-boot
+export DEVICE_NBDROOT=agl-demo-platform-raspberrypi3.ext4
+else
+    echo "This fake build script is for raspberrypi3 only."
+    exit 1
+fi
+
+
+mkdir -p tmp/deploy/images/${MACHINE}/
+
+pushd tmp/deploy/images/${MACHINE}/
+
+for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT ; do
+    eval curl -o "$(echo "$"${i})" "http://download-new.automotivelinux.org/AGL/snapshots/master/2017-01-08-b241/${MACHINE}/deploy/images/${MACHINE}/$(echo "$"${i})"
+    ls -alh 
+done
+
+ls -alh
+
+popd
+
+du -hs tmp/deploy/*
diff --git a/jjb/common/include-agl-cd-repoclone-output.sh b/jjb/common/include-agl-cd-repoclone-output.sh
new file mode 100644
index 00000000..56e8b53f
--- /dev/null
+++ b/jjb/common/include-agl-cd-repoclone-output.sh
@@ -0,0 +1,3 @@
+cd repoclone
+export REPODIR=`pwd`
+cd output
diff --git a/jjb/common/include-agl-local-conf-cvscheck.sh b/jjb/common/include-agl-local-conf-cvscheck.sh
new file mode 100644
index 00000000..d5741717
--- /dev/null
+++ b/jjb/common/include-agl-local-conf-cvscheck.sh
@@ -0,0 +1,16 @@
+# (c) 2017 Jan-Simon Moeller dl9pf(at)gmx.de
+# License GPLv2
+
+################################################################################
+## Add archiver
+################################################################################
+
+# only morty and newer support the cve-check, so check for master branch
+
+if [ x"master" = x"$GERRIT_BRANCH" ] ; then
+
+# archive sources within  tmp/deploy/
+echo '' >> conf/local.conf
+echo 'INHERIT += "cve-check"' >> conf/local.conf
+
+fi
\ No newline at end of file
diff --git a/jjb/common/include-agl-repo.sh b/jjb/common/include-agl-repo.sh
index 6b21b80c..fc85e6bc 100644
--- a/jjb/common/include-agl-repo.sh
+++ b/jjb/common/include-agl-repo.sh
@@ -16,6 +16,8 @@ mv repoclone repoclone$XTMP || true
 mkdir -p repoclone
 cd repoclone
 
+export REPODIR=`pwd`
+
 repo init --reference=/opt/AGL/preclone -q -b $TARGETBRANCH -u https://gerrit.automotivelinux.org/gerrit/AGL/AGL-repo
 
 # next: repo sync and dump manifest
diff --git a/jjb/common/include-agl-run-test-prepare.sh b/jjb/common/include-agl-run-test-prepare.sh
new file mode 100644
index 00000000..1c0594e8
--- /dev/null
+++ b/jjb/common/include-agl-run-test-prepare.sh
@@ -0,0 +1,76 @@
+# (c) 2016 Jan-Simon Moeller dl9pf(at)gmx.de
+# License GPLv2
+
+################################################################################
+## Run SHORT CI test
+################################################################################
+set -x
+
+ls -alh
+pwd
+set
+
+cd $REPODIR
+
+#### Find out if machine can be tested.
+#### We keep a stamp file in meta-agl/templates/machine/$MACHINE/test
+if [ ! -d meta-agl/templates/machine/${MACHINE}/test ]; then
+  echo "No HW test templates for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/ and its contents to enable"
+  exit 0
+fi
+
+if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.enable ] ; then 
+  echo "No HW test for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable to enable"
+  exit 0
+fi
+
+if [ ! -f meta-agl/templates/machine/${MACHINE}/test/hwtest.short.enable ] ; then
+  echo "No short HW test for ${MACHINE} configured."
+  echo "Add meta-agl/templates/machine/${MACHINE}/test/hwtest.enable.short to enable"
+else
+  eval export ENVFILE=meta-agl/templates/machine/${MACHINE}/test/hwtest.short.environment
+  eval export TESTJOBFILE=meta-agl/templates/machine/${MACHINE}/test/testjob_short.yaml
+fi
+
+if [ ! -f $ENVFILE ] ; then
+  echo "No short HW test environment file available for ${MACHINE}."
+  echo "Add ${ENVFILE} to enable."
+  exit 1
+fi
+
+if [ ! -e $TESTJOBFILE ] ; then
+  echo "No short HW test environment file available for ${MACHINE}."
+  echo "Add ${TESTJOBFILE} to enable."
+  exit 1
+fi
+
+# some defaults
+export DEVICE_TYPE=raspberrypi3-uboot
+export DEVICE_NAME=raspberrypi3
+export DEVICE_DTB=uImage-bcm2710-rpi-3-b.dtb
+export DEVICE_KERNEL=uImage
+export DEVICE_INITRAMFS=initramfs-netboot-image-raspberrypi3.ext4
+export DEVICE_NBDROOT=agl-demo-platform-raspberrypi3.ext4
+export DEVICE_NBDROOT_COMPRESSION=none
+export DEVICE_BOOT_METHOD=u-boot
+export DEVICE_BOOT_TYPE=bootm
+export DEVICE_URL_PREFIX='https://download.automotivelinux.org/AGL/upload/ci/'
+
+
+# import device defaults. Format 'a=b'
+for i in DEVICE_TYPE DEVICE_NAME DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT DEVICE_NBDROOT_COMPRESSION DEVICE_BOOT_METHOD DEVICE_BOOT_TYPE; do
+    if grep -q $i $ENVFILE ; then
+        X=$(grep $i $ENVFILE | sed -e "s#${i}=##g" -e "s#;.*##g")
+        eval export ${i}=${X}
+    fi
+done
+
+echo "Resulting values:"
+set | grep DEVICE_
+
+
+# echo NEXT is rsync
+#exit 0
+
diff --git a/jjb/common/include-agl-run-test-rsync-changeid.sh b/jjb/common/include-agl-run-test-rsync-changeid.sh
new file mode 100644
index 00000000..8c679546
--- /dev/null
+++ b/jjb/common/include-agl-run-test-rsync-changeid.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#set -x
+set -e
+
+echo "\n\n\n"
+echo "#########################################"
+echo "\n\n\n"
+
+
+export RSYNCDST="/srv/download/AGL/upload/ci/${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}"
+export RSYNCSRC=$(pwd)/UPLOAD/
+
+# construct upload folder
+mv UPLOAD UPLOAD2 || true
+rm -rf UPLOAD2 || true
+mkdir -p UPLOAD/${MACHINE}
+export DEST=$(pwd)/UPLOAD/${MACHINE}
+
+
+cd $REPODIR
+
+cd output
+
+pwd
+ls -alhR tmp/deploy/images
+
+
+# copy files to $DEST
+for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do
+    eval cp -avL tmp/deploy/images/${MACHINE}/$(echo "$"${i}) ${DEST}/
+done
+
+tree $DEST
+ls -alhR $DEST
+
+ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 mkdir -p ${RSYNCDST}
+
+rsync -avr -L -e "ssh -o StrictHostKeyChecking=no" $RSYNCSRC jenkins-slave@10.30.72.8:$RSYNCDST
diff --git a/jjb/common/include-agl-run-test-short.sh b/jjb/common/include-agl-run-test-short.sh
new file mode 100644
index 00000000..0e382727
--- /dev/null
+++ b/jjb/common/include-agl-run-test-short.sh
@@ -0,0 +1,128 @@
+# (c) 2016 Jan-Simon Moeller dl9pf(at)gmx.de
+# License GPLv2
+
+################################################################################
+## Run SHORT CI test
+################################################################################
+
+
+set -x
+
+echo "## ${MACHINE} ##"
+cd $REPODIR
+
+echo "default keyring config"
+mkdir -p ~/.local/share/python_keyring/
+cat <<EOF >  ~/.local/share/python_keyring/keyringrc.cfg
+[backend]
+default-keyring=keyring.backends.file.PlaintextKeyring
+EOF
+
+cat <<EOF > ~/.local/token
+$AGLLAVATOKEN
+EOF
+
+lava-tool auth-add --token-file ~/.local/token https://agl-jenkins-user@porter.automotivelinux.org
+rm ~/.local/token
+
+echo "## $TESTJOBFILE ##"
+
+if [ -n $TESTJOBFILE -a -e $TESTJOBFILE ] ; then
+   cp $TESTJOBFILE testjob.yaml
+else
+echo "!! NO TESTJOBFILE - trying to continue with a default !!"
+fi
+
+cat <<EOF > testjob.yaml
+# Your first LAVA JOB definition for a $MACHINE board
+device_type: @REPLACE_DEVICE_TYPE@
+job_name: AGL-short-smoke
+
+protocols:
+  lava-xnbd:
+    port: auto
+
+timeouts:
+  job:
+    minutes: 30
+  action:
+    minutes: 15
+  connection:
+    minutes: 5
+priority: medium
+visibility: public
+
+# ACTION_BLOCK
+actions:
+- deploy:
+    timeout:
+      minutes: 10
+    to: nbd
+    dtb:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_DTB@'
+    kernel:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_KERNEL@'
+    initramfs:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_INITRAMFS@'
+      allow_modify: false
+    nbdroot:
+      url: '@REPLACE_URL_PREFIX@/@REPLACE_NBDROOT@'
+      compression: @REPLACE_NBDROOT_COMPRESSION@
+    os: debian
+    failure_retry: 2
+
+# BOOT_BLOCK
+- boot:
+    method: @REPLACE_BOOT_METHOD@
+    commands: nbd
+    type: @REPLACE_BOOT_TYPE@
+    prompts: ["root@@REPLACE_MACHINE@:~"]
+    auto_login:
+      login_prompt: "login:"
+      username: root
+
+EOF
+
+CHID=${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}/${MACHINE}
+# REPLACE_DEVICE_TYPE
+sed -i -e "s#@REPLACE_DEVICE_TYPE@#${DEVICE_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_DTB@#${CHID}/${DEVICE_DTB}#g" testjob.yaml
+sed -i -e "s#@REPLACE_KERNEL@#${CHID}/${DEVICE_KERNEL}#g" testjob.yaml
+sed -i -e "s#@REPLACE_INITRAMFS@#${CHID}/${DEVICE_INITRAMFS}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT@#${CHID}/${DEVICE_NBDROOT}#g" testjob.yaml
+sed -i -e "s#@REPLACE_NBDROOT_COMPRESSION@#${DEVICE_NBDROOT_COMPRESSION}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_METHOD@#${DEVICE_BOOT_METHOD}#g" testjob.yaml
+sed -i -e "s#@REPLACE_BOOT_TYPE@#${DEVICE_BOOT_TYPE}#g" testjob.yaml
+sed -i -e "s#@REPLACE_MACHINE@#${DEVICE_NAME}#g" testjob.yaml
+sed -i -e "s#@REPLACE_URL_PREFIX@#${DEVICE_URL_PREFIX}#g" testjob.yaml
+
+cat testjob.yaml
+
+lava-tool submit-job --block https://agl-jenkins-user@porter.automotivelinux.org testjob.yaml | tee .myjob
+
+MYJOB=`cat .myjob | grep "submitted as job" | sed -e "s#submitted as job id: ##g"`
+
+echo "#### JOBID: $MYJOB #####"
+
+( lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB | tee .status ) || true
+STATUS=`grep "Job Status:" .status | sed -e "s#Job Status: ##g"`
+
+if [ x"Complete" = x"$STATUS"  ] ; then
+    echo "YAY! $STATUS"
+    curl -o plain_output.yaml https://porter.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
+    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
+
+    # cleanup
+    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
+
+    exit 0
+else
+    echo "Nooooooooo! $STATUS"
+    curl -o plain_output.yaml https://porter.automotivelinux.org/scheduler/job/$MYJOB/log_file/plain
+    cat plain_output.yaml | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g'
+
+    # cleanup
+    #ssh -o StrictHostKeyChecking=no jenkins-slave@10.30.72.8 rm -rf /srv/download/AGL/upload/ci/${CHID}/
+
+    exit 1
+fi
diff --git a/jjb/common/include-agl-run-test-snapshot.sh b/jjb/common/include-agl-run-test-snapshot.sh
deleted file mode 100644
index b753708f..00000000
--- a/jjb/common/include-agl-run-test-snapshot.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-# (c) 2016 Jan-Simon Moeller dl9pf(at)gmx.de
-# License GPLv2
-
-################################################################################
-## Run SHORT CI test
-################################################################################
-
-# test currently only for porter, rest WIP
-echo "## ${MACHINE} ##"
-
-
-echo "default keyring config"
-
-mkdir -p ~/.local/share/python_keyring/
-
-cat <<EOF >  ~/.local/share/python_keyring/keyringrc.cfg
-[backend]
-default-keyring=keyring.backends.file.PlaintextKeyring
-EOF
-
-cat <<EOF > ~/.local/token
-$AGLLAVATOKEN
-EOF
-
-lava-tool auth-add --token-file ~/.local/token https://agl-jenkins-user@porter.automotivelinux.org
-
-cat ~/.local/token
-
-cat <<EOF > testjob.yaml
-# Your first LAVA JOB definition for a porter board
-device_type: renesas-porter-uboot
-job_name: renesas-porter-uboot
-
-protocols:
-  lava-xnbd:
-    port: auto
-
-timeouts:
-  job:
-    minutes: 15
-  action:
-    minutes: 5
-  connection:
-    minutes: 2
-priority: medium
-visibility: public
-
-# ACTION_BLOCK
-actions:
-- deploy:
-    to: nbd
-    dtb:
-      url: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/uImage-r8a7791-porter.dtb'
-    kernel:
-      url: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/uImage'
-    initramfs:
-      url: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/initramfs-netboot-image-porter.ext4.gz.u-boot'
-      allow_modify: false
-    nbdroot:
-      url: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/core-image-minimal-porter.ext4'
-    os: debian
-    failure_retry: 2
-
-
-# BOOT_BLOCK
-- boot:
-    method: u-boot
-    commands: nbd
-    type: bootm
-    prompts: ["root@porter:~"]
-    auto_login:
-      login_prompt: "login:"
-      username: root
-
-EOF
-
-#rm ~/.local/token
-
-lava-tool submit-job https://agl-jenkins-user@porter.automotivelinux.org testjob.yaml | tee .myjob
-
-MYJOB=`cat .myjob | sed -e "s#submitted as job id: ##g"`
-
-echo $MYJOB
-
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-lava-tool job-details https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-lava-tool job-output https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-status https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-sleep 10
-lava-tool job-details https://agl-jenkins-user@porter.automotivelinux.org $MYJOB
-
-
-
-# setup 
\ No newline at end of file