SPEC-2962: permit to have more than one board per build

The current way to generate jobs assume that only one board exists per
build.

For handing more than one board per build, we need to introduce a sort
of loop.

So for each build, we will produce a list of boards in BOARDOUT
In the BOARDOUT/$board, we will have:
- an "info" file with all necessary VARIABLES for copying artifacts

So for each board, some artifact will be copied.

Then only one job for the "main" board of the CIBT job will be handled.
The extra boards of a build will need to have their own CIBT, but at
least they dont need to do a real build/copyartifact.

Change-Id: I6c6999e549c7f1216f466410896b3058efc5882d
Signed-off-by: Corentin LABBE <clabbe@baylibre.com>

3 files changed, 118 insertions, 75 deletions

diff --git a/jjb/common/include-agl-run-test-prepare.sh b/jjb/common/include-agl-run-test-prepare.sh
index 56d99c8a..cb5be40e 100644
--- a/jjb/common/include-agl-run-test-prepare.sh
+++ b/jjb/common/include-agl-run-test-prepare.sh
@@ -20,59 +20,98 @@ if [ -e releng-scripts/utils ]; then
     RELENG=$PWD/releng-scripts
 fi
 
-declare -A agl_lava_releng_aliases
-agl_lava_releng_aliases=(
-    [raspberrypi3]="bcm2837-rpi-3-b-32;raspberrypi3"
-    [raspberrypi4]="bcm2711-rpi-4-b;raspberrypi4"
-    [qemux86-64]="qemu;qemux86-64"
-    [qemuarm]="qemu;qemuarm"
-    [qemuarm64]="qemu;qemuarm64"
-    [m3ulcb-nogfx]="r8a7796-m3ulcb;m3ulcb"
-    [h3ulcb-nogfx]="r8a7795-h3ulcb-kf;h3ulcb-kf"
-    [porter-nogfx]="renesas-porter;porter"
-    [dra7xx-evm]="dra7-evm;dra7xx-evm"
-    [intel-corei7-64]="upsquare;upsquare"
-)
+# WARNING: we need gen_info in last, the device of the current JENKINS CIBT
+# example: for x86, the JENKINS CIBT job is for "qemu" and produce artifacts for all x86 boards.
+# so we need to gen_info for qemux86 and upsquare, this content will be used to copy artifacts just after.
+# but the LAVA job need to be generated only for qemu. So we need to have lava_device and releng_device for it.
+export BOARDOUT=$(mktemp --directory)
+case $MACHINE in
+qemux86-64)
+	lava_device=upsquare
+	releng_device=upsquare
+	gen_info || exit 0
+	lava_device=qemu
+	releng_device=qemux86-64
+	gen_info || exit 0
+;;
+raspberrypi4)
+	lava_device=bcm2711-rpi-4-b
+	releng_device=raspberrypi4
+	gen_info || exit 0
+;;
+qemuarm)
+	lava_device=qemu
+	releng_device=qemuarm
+	gen_info || exit 0
+;;
+qemuarm64)
+	lava_device=qemu
+	releng_device=qemuarm64
+	gen_info || exit 0
+;;
+h3ulcb-nogfx)
+	lava_device=r8a7795-h3ulcb-kf
+	releng_device=h3ulcb-kf
+	gen_info || exit 0
+;;
+upsquare)
+	lava_device=upsquare
+	releng_device=upsquare
+	# NO gen_info since, we use artifacts of the qemux86_64 CIBT
+;;
+*)
+	echo "ERROR: unknown machine $MACHINE"
+	exit 0
+;;
+esac
 
-OFS=${IFS}
-IFS=';'
-arr=(${agl_lava_releng_aliases[$MACHINE]})
-IFS=${OFS}
-lava_device=${arr[0]}
-releng_device=${arr[1]}
-# And agl_device=${MACHINE}
+#gen board info in $BOARDOUT
+gen_info() {
+	if [ -z "${lava_device}" -o -z "${releng_device}" ]; then
+		echo "Board not supported: $MACHINE"
+		return 1
+	fi
 
-if [ -z "${lava_device}" -o -z "${releng_device}" ]; then
-    echo "Board not supported: $MACHINE"
-    exit 0
-fi
+	CREATE_ARGS=""
+	CREATE_ARGS+="--machine ${releng_device} "
 
-CREATE_ARGS=""
-CREATE_ARGS+="--machine ${releng_device} "
+	if [[ $TARGETBRANCH ]] && [[ x"master" != x"$TARGETBRANCH" ]] ; then
+		CREATE_ARGS+="--branch $TARGETBRANCH "
+	fi
 
-if [[ $TARGETBRANCH ]] && [[ x"master" != x"$TARGETBRANCH" ]] ; then
-CREATE_ARGS+="--branch $TARGETBRANCH "
-fi
+	# If it's a release build
+	if [[ $RELEASE_BRANCH ]] && [[ $RELEASE_VERSION ]]; then
+		CREATE_ARGS+="--build-type release $RELEASE_BRANCH $RELEASE_VERSION "
+	fi
+	# If it's a CI build
+	if [[ $GERRIT_CHANGE_NUMBER ]] && [[ $GERRIT_PATCHSET_NUMBER ]]; then
+		CREATE_ARGS+="--build-type ci $GERRIT_CHANGE_NUMBER $GERRIT_PATCHSET_NUMBER "
+	fi
 
-# If it's a release build
-if [[ $RELEASE_BRANCH ]] && [[ $RELEASE_VERSION ]]; then
-    CREATE_ARGS+="--build-type release $RELEASE_BRANCH $RELEASE_VERSION "
-fi
-# If it's a CI build
-if [[ $GERRIT_CHANGE_NUMBER ]] && [[ $GERRIT_PATCHSET_NUMBER ]]; then
-    CREATE_ARGS+="--build-type ci $GERRIT_CHANGE_NUMBER $GERRIT_PATCHSET_NUMBER "
-fi
+	$RELENG/utils/job-prereq.py ${CREATE_ARGS}
+	if [ $? -ne 0 ]; then
+		echo "Board not supported by releng-scripts: job-prereq.py"
+		return 1
+	fi
+	# First call to job-prereq suceeded, other should suceed as they use the same args.
+	export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb`
+	export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel`
+	export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd`
+	export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot`
+	# thoses variables (DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT) are only used for copying artifacts later
 
-$RELENG/utils/job-prereq.py ${CREATE_ARGS}
-if [ $? -ne 0 ]; then
-    echo "Board not supported by releng-scripts: job-prereq.py"
-    exit 0
-fi
-# First call to job-prereq suceeded, other should suceed as they use the same args.
-export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb`
-export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel`
-export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd`
-export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot`
+	mkdir -p $BOARDOUT/
+	# We use both lava_device and releng_device for handling name collision (like qemu)
+	BDIRNAME="$lava_device-$releng_device"
+	echo "DEVICE_DTB=$DEVICE_DTB" > $BOARDOUT/$BDIRNAME/info
+	echo "DEVICE_KERNEL=$DEVICE_KERNEL" >> $BOARDOUT/$BDIRNAME/info
+	echo "DEVICE_INITRAMFS=$DEVICE_INITRAMFS" >> $BOARDOUT/$BDIRNAME/info
+	echo "DEVICE_NBDROOT=$DEVICE_NBDROOT" >> $BOARDOUT/$BDIRNAME/info
+	# thoses two variable are only for debugging
+	echo "x_releng_device=$releng_device" >> $BOARDOUT/$BDIRNAME/info
+	echo "x_lava_device=$lava_device" >> $BOARDOUT/$BDIRNAME/info
+	return 0
+}
 
 # echo NEXT is rsync
 #exit 0
diff --git a/jjb/common/include-agl-run-test-rsync-changeid.sh b/jjb/common/include-agl-run-test-rsync-changeid.sh
index e576da04..41fe2b48 100644
--- a/jjb/common/include-agl-run-test-rsync-changeid.sh
+++ b/jjb/common/include-agl-run-test-rsync-changeid.sh
@@ -10,7 +10,6 @@ echo "\n\n\n"
 
 export RSYNCDST="/srv/download/AGL/upload/ci/${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}"
 export RSYNCSRC=$(pwd)/UPLOAD/
-export RSYNCSRCstorage=$(pwd)/UPLOADstorage/
 
 # construct upload folder
 mv UPLOAD UPLOAD2 || true
@@ -18,9 +17,6 @@ rm -rf UPLOAD2 || true
 mkdir -p UPLOAD/${MACHINE}
 export DEST=$(pwd)/UPLOAD/${MACHINE}
 
-export DEST2=$(pwd)/UPLOADstorage/AGL/upload/ci/${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}/
-mkdir -p ${DEST2}
-
 cd $REPODIR
 
 cd output
@@ -30,30 +26,30 @@ ls -alhR tmp/deploy/images
 
 #set -x
 
-echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB}"
-echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-}"
-
-## fixed in releng with --branch
-# workaround for thud until fixed in templates:
-#if [ ! -e tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB} -a -e tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-} ] ; then
-#    cp -avL tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-} tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB}
-#fi
-
-if [ -e tmp/deploy/images/${TARGETMACHINE}/build-info ] ; then
-      cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST}/
-      cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST2}/
-fi
-
-# copy files to $DEST (only if variable is non-empty)
-for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do
-    FILE=$(eval echo \$${i})
-    if [ -n "$FILE" ]; then
-      cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST}/
-      cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST2}/
-    fi
+for board in $(ls $BOARDOUT)
+do
+	. $BOARDOUT/$board/info
+	echo "INFO: copy artifact for $x_releng_device/$x_lava_device"
+	echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB}"
+	echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-}"
+
+	if [ -e tmp/deploy/images/${TARGETMACHINE}/build-info ] ; then
+		cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST}/
+	fi
+
+	# copy files to $DEST (only if variable is non-empty)
+	for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do
+		FILE=$(eval echo \$${i})
+		if [ -n "$FILE" ]; then
+			cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST}/
+		fi
+	done
 done
 set +x
 
+# for the moment, nothing more use this directory, clean it
+rm -r $BOARDOUT
+
 tree $DEST
 ls -alhR $DEST
 
diff --git a/jjb/common/include-agl-run-test-rsync-release.sh b/jjb/common/include-agl-run-test-rsync-release.sh
index abab0f34..7be8bfa3 100644
--- a/jjb/common/include-agl-run-test-rsync-release.sh
+++ b/jjb/common/include-agl-run-test-rsync-release.sh
@@ -25,11 +25,19 @@ cd output
 #pwd
 #ls -alhR tmp/deploy/images
 
-# copy files to $DEST
-for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do
-    eval cp -avL tmp/deploy/images/${TARGETMACHINE}/$(echo "$"${i}) ${DEST}/
+for board in $(ls $BOARDOUT)
+do
+	# copy files to $DEST
+	. $BOARDOUT/$board/info
+	echo "INFO: copy artifact for $x_releng_device/$x_lava_device"
+	for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do
+		eval cp -avL tmp/deploy/images/${TARGETMACHINE}/$(echo "$"${i}) ${DEST}/
+	done
 done
 
+# for the moment, nothing more use this directory, clean it
+rm -r $BOARDOUT
+
 tree $DEST
 ls -alhR $DEST