From 47533a77f8826718e7f79cce01ec9bc1e03bfadb Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 24 Mar 2020 10:08:54 +0100 Subject: SPEC-2962: permit to have more than one board per build The current way to generate jobs assume that only one board exists per build. For handing more than one board per build, we need to introduce a sort of loop. So for each build, we will produce a list of boards in BOARDOUT In the BOARDOUT/$board, we will have: - an "info" file with all necessary VARIABLES for copying artifacts So for each board, some artifact will be copied. Then only one job for the "main" board of the CIBT job will be handled. The extra boards of a build will need to have their own CIBT, but at least they dont need to do a real build/copyartifact. Change-Id: I6c6999e549c7f1216f466410896b3058efc5882d Signed-off-by: Corentin LABBE --- jjb/common/include-agl-run-test-prepare.sh | 133 ++++++++++++++-------- jjb/common/include-agl-run-test-rsync-changeid.sh | 46 ++++---- jjb/common/include-agl-run-test-rsync-release.sh | 14 ++- 3 files changed, 118 insertions(+), 75 deletions(-) (limited to 'jjb') diff --git a/jjb/common/include-agl-run-test-prepare.sh b/jjb/common/include-agl-run-test-prepare.sh index 56d99c8a..cb5be40e 100644 --- a/jjb/common/include-agl-run-test-prepare.sh +++ b/jjb/common/include-agl-run-test-prepare.sh @@ -20,59 +20,98 @@ if [ -e releng-scripts/utils ]; then RELENG=$PWD/releng-scripts fi -declare -A agl_lava_releng_aliases -agl_lava_releng_aliases=( - [raspberrypi3]="bcm2837-rpi-3-b-32;raspberrypi3" - [raspberrypi4]="bcm2711-rpi-4-b;raspberrypi4" - [qemux86-64]="qemu;qemux86-64" - [qemuarm]="qemu;qemuarm" - [qemuarm64]="qemu;qemuarm64" - [m3ulcb-nogfx]="r8a7796-m3ulcb;m3ulcb" - [h3ulcb-nogfx]="r8a7795-h3ulcb-kf;h3ulcb-kf" - [porter-nogfx]="renesas-porter;porter" - [dra7xx-evm]="dra7-evm;dra7xx-evm" - [intel-corei7-64]="upsquare;upsquare" -) +# WARNING: we need gen_info in last, the device of the current JENKINS CIBT +# example: for x86, the JENKINS CIBT job is for "qemu" and produce artifacts for all x86 boards. +# so we need to gen_info for qemux86 and upsquare, this content will be used to copy artifacts just after. +# but the LAVA job need to be generated only for qemu. So we need to have lava_device and releng_device for it. +export BOARDOUT=$(mktemp --directory) +case $MACHINE in +qemux86-64) + lava_device=upsquare + releng_device=upsquare + gen_info || exit 0 + lava_device=qemu + releng_device=qemux86-64 + gen_info || exit 0 +;; +raspberrypi4) + lava_device=bcm2711-rpi-4-b + releng_device=raspberrypi4 + gen_info || exit 0 +;; +qemuarm) + lava_device=qemu + releng_device=qemuarm + gen_info || exit 0 +;; +qemuarm64) + lava_device=qemu + releng_device=qemuarm64 + gen_info || exit 0 +;; +h3ulcb-nogfx) + lava_device=r8a7795-h3ulcb-kf + releng_device=h3ulcb-kf + gen_info || exit 0 +;; +upsquare) + lava_device=upsquare + releng_device=upsquare + # NO gen_info since, we use artifacts of the qemux86_64 CIBT +;; +*) + echo "ERROR: unknown machine $MACHINE" + exit 0 +;; +esac -OFS=${IFS} -IFS=';' -arr=(${agl_lava_releng_aliases[$MACHINE]}) -IFS=${OFS} -lava_device=${arr[0]} -releng_device=${arr[1]} -# And agl_device=${MACHINE} +#gen board info in $BOARDOUT +gen_info() { + if [ -z "${lava_device}" -o -z "${releng_device}" ]; then + echo "Board not supported: $MACHINE" + return 1 + fi -if [ -z "${lava_device}" -o -z "${releng_device}" ]; then - echo "Board not supported: $MACHINE" - exit 0 -fi + CREATE_ARGS="" + CREATE_ARGS+="--machine ${releng_device} " -CREATE_ARGS="" -CREATE_ARGS+="--machine ${releng_device} " + if [[ $TARGETBRANCH ]] && [[ x"master" != x"$TARGETBRANCH" ]] ; then + CREATE_ARGS+="--branch $TARGETBRANCH " + fi -if [[ $TARGETBRANCH ]] && [[ x"master" != x"$TARGETBRANCH" ]] ; then -CREATE_ARGS+="--branch $TARGETBRANCH " -fi + # If it's a release build + if [[ $RELEASE_BRANCH ]] && [[ $RELEASE_VERSION ]]; then + CREATE_ARGS+="--build-type release $RELEASE_BRANCH $RELEASE_VERSION " + fi + # If it's a CI build + if [[ $GERRIT_CHANGE_NUMBER ]] && [[ $GERRIT_PATCHSET_NUMBER ]]; then + CREATE_ARGS+="--build-type ci $GERRIT_CHANGE_NUMBER $GERRIT_PATCHSET_NUMBER " + fi -# If it's a release build -if [[ $RELEASE_BRANCH ]] && [[ $RELEASE_VERSION ]]; then - CREATE_ARGS+="--build-type release $RELEASE_BRANCH $RELEASE_VERSION " -fi -# If it's a CI build -if [[ $GERRIT_CHANGE_NUMBER ]] && [[ $GERRIT_PATCHSET_NUMBER ]]; then - CREATE_ARGS+="--build-type ci $GERRIT_CHANGE_NUMBER $GERRIT_PATCHSET_NUMBER " -fi + $RELENG/utils/job-prereq.py ${CREATE_ARGS} + if [ $? -ne 0 ]; then + echo "Board not supported by releng-scripts: job-prereq.py" + return 1 + fi + # First call to job-prereq suceeded, other should suceed as they use the same args. + export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb` + export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel` + export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd` + export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot` + # thoses variables (DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT) are only used for copying artifacts later -$RELENG/utils/job-prereq.py ${CREATE_ARGS} -if [ $? -ne 0 ]; then - echo "Board not supported by releng-scripts: job-prereq.py" - exit 0 -fi -# First call to job-prereq suceeded, other should suceed as they use the same args. -export DEVICE_DTB=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --dtb` -export DEVICE_KERNEL=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --kernel` -export DEVICE_INITRAMFS=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --initrd` -export DEVICE_NBDROOT=`$RELENG/utils/job-prereq.py ${CREATE_ARGS} --nbdroot` + mkdir -p $BOARDOUT/ + # We use both lava_device and releng_device for handling name collision (like qemu) + BDIRNAME="$lava_device-$releng_device" + echo "DEVICE_DTB=$DEVICE_DTB" > $BOARDOUT/$BDIRNAME/info + echo "DEVICE_KERNEL=$DEVICE_KERNEL" >> $BOARDOUT/$BDIRNAME/info + echo "DEVICE_INITRAMFS=$DEVICE_INITRAMFS" >> $BOARDOUT/$BDIRNAME/info + echo "DEVICE_NBDROOT=$DEVICE_NBDROOT" >> $BOARDOUT/$BDIRNAME/info + # thoses two variable are only for debugging + echo "x_releng_device=$releng_device" >> $BOARDOUT/$BDIRNAME/info + echo "x_lava_device=$lava_device" >> $BOARDOUT/$BDIRNAME/info + return 0 +} # echo NEXT is rsync #exit 0 diff --git a/jjb/common/include-agl-run-test-rsync-changeid.sh b/jjb/common/include-agl-run-test-rsync-changeid.sh index e576da04..41fe2b48 100644 --- a/jjb/common/include-agl-run-test-rsync-changeid.sh +++ b/jjb/common/include-agl-run-test-rsync-changeid.sh @@ -10,7 +10,6 @@ echo "\n\n\n" export RSYNCDST="/srv/download/AGL/upload/ci/${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}" export RSYNCSRC=$(pwd)/UPLOAD/ -export RSYNCSRCstorage=$(pwd)/UPLOADstorage/ # construct upload folder mv UPLOAD UPLOAD2 || true @@ -18,9 +17,6 @@ rm -rf UPLOAD2 || true mkdir -p UPLOAD/${MACHINE} export DEST=$(pwd)/UPLOAD/${MACHINE} -export DEST2=$(pwd)/UPLOADstorage/AGL/upload/ci/${GERRIT_CHANGE_NUMBER}/${GERRIT_PATCHSET_NUMBER}/ -mkdir -p ${DEST2} - cd $REPODIR cd output @@ -30,30 +26,30 @@ ls -alhR tmp/deploy/images #set -x -echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB}" -echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-}" - -## fixed in releng with --branch -# workaround for thud until fixed in templates: -#if [ ! -e tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB} -a -e tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-} ] ; then -# cp -avL tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-} tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB} -#fi - -if [ -e tmp/deploy/images/${TARGETMACHINE}/build-info ] ; then - cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST}/ - cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST2}/ -fi - -# copy files to $DEST (only if variable is non-empty) -for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do - FILE=$(eval echo \$${i}) - if [ -n "$FILE" ]; then - cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST}/ - cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST2}/ - fi +for board in $(ls $BOARDOUT) +do + . $BOARDOUT/$board/info + echo "INFO: copy artifact for $x_releng_device/$x_lava_device" + echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB}" + echo "tmp/deploy/images/${TARGETMACHINE}/${DEVICE_DTB#"$DEVICE_KERNEL"-}" + + if [ -e tmp/deploy/images/${TARGETMACHINE}/build-info ] ; then + cp -avL tmp/deploy/images/${TARGETMACHINE}/build-info ${DEST}/ + fi + + # copy files to $DEST (only if variable is non-empty) + for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do + FILE=$(eval echo \$${i}) + if [ -n "$FILE" ]; then + cp -avL tmp/deploy/images/${TARGETMACHINE}/${FILE} ${DEST}/ + fi + done done set +x +# for the moment, nothing more use this directory, clean it +rm -r $BOARDOUT + tree $DEST ls -alhR $DEST diff --git a/jjb/common/include-agl-run-test-rsync-release.sh b/jjb/common/include-agl-run-test-rsync-release.sh index abab0f34..7be8bfa3 100644 --- a/jjb/common/include-agl-run-test-rsync-release.sh +++ b/jjb/common/include-agl-run-test-rsync-release.sh @@ -25,11 +25,19 @@ cd output #pwd #ls -alhR tmp/deploy/images -# copy files to $DEST -for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do - eval cp -avL tmp/deploy/images/${TARGETMACHINE}/$(echo "$"${i}) ${DEST}/ +for board in $(ls $BOARDOUT) +do + # copy files to $DEST + . $BOARDOUT/$board/info + echo "INFO: copy artifact for $x_releng_device/$x_lava_device" + for i in DEVICE_DTB DEVICE_KERNEL DEVICE_INITRAMFS DEVICE_NBDROOT; do + eval cp -avL tmp/deploy/images/${TARGETMACHINE}/$(echo "$"${i}) ${DEST}/ + done done +# for the moment, nothing more use this directory, clean it +rm -r $BOARDOUT + tree $DEST ls -alhR $DEST -- cgit 1.2.3-korg