adding support for .wic .wic.xz as source for image

Change-Id: I94c09c6105ac06a37cedc04c777dd1372015a02d
Signed-off-by: Dominig ar Foll Intel Open Source <dominig.arfoll@fridu.net>
Reviewed-on: https://gerrit.automotivelinux.org/gerrit/12263
ci-image-build: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>
Tested-by: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>
ci-image-boot-test: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>
Reviewed-by: Jan-Simon Moeller <jsmoeller@linuxfoundation.org>

2 files changed, 120 insertions, 28 deletions

diff --git a/scripts/README-mkefi-agl.md b/scripts/README-mkefi-agl.md
index f415747d8..db95f5ded 100644
--- a/scripts/README-mkefi-agl.md
+++ b/scripts/README-mkefi-agl.md
@@ -3,14 +3,14 @@ This script will install the AGL distribution on a removable device to boot on I
 
 In particular it can create a USB or SD bootable support for (MinnowBoard)[www.minnowboard.org].
 
-Usage:
-
-    mkefi-agl.sh [-v] HDDIMG REMOVABLE_DEVICE
-       -v: verbose debug
-       HDDIMG: the hddimg file to generate the efi disk from
-       REMOVABLE_DEVICE: the block device to write the image to, e.g. /dev/sdh
-
-Example: `mkefi-agl.sh agl-demo-platform-intel-corei7-64.hddimg /dev/sdd`
+Usage: mkefi-agl.sh [-v] IDSK_IMAGE REMOVABLE_DEVICE
+       -v: Verbose debug
+       HDDIMG: The DISK_IMAGE file generated by Yocto the efi disk from
+               Supported formats are .hddimg, .wic .wic.xz
+       REMOVABLE_DEVICE: The block device to write the image to, e.g. /dev/sdh
+ex:
+   mkefi-agl.sh   agl-demo-platform-intel-corei7-64.hddimg /dev/sdd
+   mkefi-agl.sh   agl-demo-platform-intel-corei7-64.wic.xz /dev/sdd
 
 ## Documentation
 Additional documentation: https://wiki.automotivelinux.org/agl-distro/developer_resources_intel
diff --git a/scripts/mkefi-agl.sh b/scripts/mkefi-agl.sh
index b933b190f..e3240a6e5 100755
--- a/scripts/mkefi-agl.sh
+++ b/scripts/mkefi-agl.sh
@@ -26,7 +26,7 @@
 #         - create a grub config with PARTUID to ease boot from various devices automaticaly
 #         - add a UEFI startup.nsh script for autoboot
 #         - does not allocate swap
-#
+#         - accept .hddimg, wic and wic.xz as sources
 
 LANG=C
 
@@ -39,6 +39,10 @@ exec 3>/dev/null
 #
 # 100 Mb for the boot partition
 BOOT_SIZE=100
+# min available space on TMP_DIR for uncompressing xz image in kB e.g. 5G (5000000)
+TMP_SIZE_MIN=5000000
+# TMP_DIR directory use for holding image file for uncompression (e.g. /tmp or $HOME)
+TMP_DIR=/tmp
 
 # Cleanup after die()
 cleanup() {
@@ -47,13 +51,20 @@ cleanup() {
 	unmount $ROOTFS_MNT || error "Failed to unmount $ROOTFS_MNT"
 	unmount $BOOTFS_MNT || error "Failed to unmount $BOOTFS_MNT"
 	unmount $HDDIMG_ROOTFS_MNT || error "Failed to unmount $HDDIMG_ROOTFS_MNT"
-	unmount $HDDIMG_MNT || error "Failed to unmount $HDDIMG_MNT"
+    unmount $HDDIMG_MNT || error "Failed to unmount $HDDIMG_MNT"
+	if [ "$IMG_TYPE" = "DISK" ]; then
+        debug "de-attaching loop devices"
+        for LOOP_DEVICE in `losetup --list |grep $HDDIMG | cut -d" " -f1` ; do
+           losetup -d $LOOP_DEVICE  1>&3 2>&1 || error "Detaching $LOOP_DEVICE from $HDDIMG failled"
+        done   
+     fi
 
 	# Remove the TMPDIR
 	debug "Removing temporary files"
 	if [ -d "$TMPDIR" ]; then
-		rm -rf $TMPDIR || error "Failed to remove $TMPDIR"
+		rm -rf "$TMPDIR" || error "Failed to remove $TMPDIR"
 	fi
+        [ -f "$TMP_DIR/TMP-AGL-wic-image.wic" ] || rm -f $TMP_DIR/TMP-AGL-wic-image.wic
 }
 
 trap 'die "Signal Received, Aborting..."' HUP INT TERM
@@ -95,12 +106,14 @@ usage() {
 	echo "Install AGL on a removable device to boot on IA UEFI based computer"
 	echo "In particular is can create USB or SD bootable support for Minnowboard"
 	echo ""
-	echo "Usage: $(basename $0) [-v] HDDIMG REMOVABLE_DEVICE"
+	echo "Usage: $(basename $0) [-v] IDSK_IMAGE REMOVABLE_DEVICE"
 	echo "       -v: Verbose debug"
-	echo "       HDDIMG: The hddimg file to generate the efi disk from"
+	echo "       HDDIMG: The DISK_IMAGE file generated by Yocto the efi disk from"
+        echo "               Supported formats are .hddimg, .wic .wic.xz"
 	echo "       REMOVABLE_DEVICE: The block device to write the image to, e.g. /dev/sdh"
 	echo "ex:"
 	echo "   mkefi-agl.sh   agl-demo-platform-intel-corei7-64.hddimg /dev/sdd"
+        echo "   mkefi-agl.sh   agl-demo-platform-intel-corei7-64.wic.xz /dev/sdd"
 	exit 1
 }
 
@@ -198,11 +211,55 @@ fi
 if [ ! -e "$HDDIMG" ]; then
 	die "HDDIMG $HDDIMG does not exist"
 fi
+HDDIMG_EXT=${HDDIMG##*.}
+case $HDDIMG_EXT in
+  hddimg)
+     IMG_TYPE="MOUNT"
+     IMG_COMPRESS="NO"
+     debug "Detected: uncompressed image type .hddimg"
+     ;;
+  wic)
+     IMG_TYPE="DISK"
+     IMG_COMPRESS="NO"
+     debug "Detected: uncompressed image type .wic"
+     ;;
+  xz)
+     IMG_TYPE="DISK"
+     IMG_COMPRESS="YES"
+     debug "Detected: xz compressed image type .wic"
+     command -v xz >/dev/null 2>&1 || { die "xz command is not available, pleaes install xz package"; }
+     TMP_SIZE=`df -k $TMP_DIR | awk '/[0-9]%/{print $(NF-2)}'`
+     if [ "$TMP_SIZE" -lt "$TMP_SIZE_MIN" ]; then
+       die "Available space on $TMP_DIR must be at least $TMP_SIZE_MIN kB" 
+     fi
+     printf "Starting decompression of the image. It may take some time ..."
+     xz --decompress --keep --format=auto --force --threads=0 --stdout > $TMP_DIR/TMP-AGL-wic-image.wic  $HDDIMG|| \
+         die "xz command failled: xz --decompress --keep --format=auto --force --threads=0 --stdout > $TMP_DIR/TMP-AGL-wic-image.wic"
+     HDDIMG="$TMP_DIR/TMP-AGL-wic-image.wic"
+     echo "Image uncompressed, starting doing real work ..."
+     ;;
+  *)
+     die "Unsupported image format: $HDDIMG_EXT Supported format are .hddimg .wic wic.xz"
+     ;;
+esac
 
 #
 # Ensure the hddimg is not mounted
 #
-unmount "$HDDIMG" || die "Failed to unmount $HDDIMG"
+debug "will now try to umount /detach previous images"
+case $IMG_TYPE in
+  MOUNT)
+      unmount "$HDDIMG" || die "Failed to unmount $HDDIMG"
+      ;;
+  DISK)
+      [ `losetup --list |grep $HDDIMG | wc -l ` -gt 1 ] && die "Image mounted more than once, manual cleaning required see: losetup --list"
+      debug "ready to attach the wic image to aloop device"
+      LOOP_DEVICE=`losetup --find --show $HDDIMG`  && ( losetup -d $LOOP_DEVICE  1>&3 2>&1 || die "Detaching $LOOP_DEVICE from $HDDIMG failled")
+      ;;
+  *)
+      die "unknown image format $IMG_TYPE"
+      ;;
+esac
 
 #
 # Check if any $DEVICE partitions are mounted
@@ -217,8 +274,7 @@ device_details $(basename $DEVICE)
 echo -n "${INFO}Prepare EFI image on $DEVICE [y/N]?${CLEAR} "
 read RESPONSE
 if [ "$RESPONSE" != "y" ]; then
-	echo "Image creation aborted"
-	exit 0
+	die "Image creation aborted"
 fi
 
 
@@ -326,23 +382,55 @@ mkfs.ext4 -F $ROOTFS -L "ROOT" 1>&3 2>&1 || die "Failed to format $ROOTFS"
 
 
 #
-# Installing to $DEVICE
+# Mounting image file system on loop devices
 #
-debug "Mounting images and device in preparation for installation"
-mount -o loop $HDDIMG $HDDIMG_MNT 1>&3 2>&1 || error "Failed to mount $HDDIMG"
-mount -o loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT 1>&3 2>&1 || error "Failed to mount rootfs.img"
-mount $ROOTFS $ROOTFS_MNT 1>&3 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT"
-mount $BOOTFS $BOOTFS_MNT 1>&3 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT"
+case $IMG_TYPE in
+
+  MOUNT) 
+     debug "Mounting images and device in preparation for installation"
+     mount -o loop $HDDIMG $HDDIMG_MNT 1>&3 2>&1 || die "Failed to mount $HDDIMG"
+     mount -o loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT 1>&3 2>&1 || die "Failed to mount rootfs.img"
+     ;;
+  DISK)
+     debug "Attaching image and mounting partitions then device in preparation for installation"
+     LOOP_DEVICE=`losetup --find` || die "Failled to find an available loop device see: losetup --find"
+     losetup -P $LOOP_DEVICE $HDDIMG  1>&3 2>&1 || die "Attaching $LOOP_DEVICE from $HDDIMG failled"
+     mount "$LOOP_DEVICE"p2  $HDDIMG_ROOTFS_MNT 1>&3 2>&1 || die "Failed to mount $LOOP_DEVICEp1 on $HDDIMG_ROOTFS_MNT"
+     mount "$LOOP_DEVICE"p1  $HDDIMG_MNT       1>&3 2>&1 || die "Failed to mount $LOOP_DEVICEp2 on $HDDIMG_MNT"
+     ;;
+  *)
+      die "unknown image format $IMG_TYPE"
+      ;;
+esac
+# Mount removable device
+     mount $ROOTFS $ROOTFS_MNT 1>&3 2>&1 || die "Failed to mount $ROOTFS on $ROOTFS_MNT"
+     mount $BOOTFS $BOOTFS_MNT 1>&3 2>&1 || die "Failed to mount $BOOTFS on $BOOTFS_MNT"
+
 
 info "Preparing boot partition"
 EFIDIR="$BOOTFS_MNT/EFI/BOOT"
-cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT 1>&3 2>&1 || error "Failed to copy vmlinuz"
-if [ -f $HDDIMG_MNT/initrd ]; then
-  cp $HDDIMG_MNT/initrd $BOOTFS_MNT 1>&3 2>&1 || error "Failed to copy initrd"
+if [ -f $HDDIMG_MNT/vmlinuz ]; then
+   cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT 1>&3 2>&1 || die "Failed to copy vmlinuz"
+   KERNEL_TYPE="VMLINUZ"
+   debug "kernel is vmlinuz"
+fi
+if [ -f $HDDIMG_MNT/bzimage ]; then
+   cp $HDDIMG_MNT/bzimage $BOOTFS_MNT 1>&3 2>&1 || die "Failed to copy vmlinuz"
+   KERNEL_TYPE="BZIMAGE"
+   debug "kernel is bzimage"
 fi
-echo "bootx64.efi" > $BOOTFS_MNT/startup.nsh || error "Failed to create startup.nsh"
+[ -z $KERNEL_TYPE ] && die "Linux kernel type in $HDDIMG is unsupported"
+
+if [ -f $HDDIMG_MNT/initrd ]
+  then
+    echo "initrd detected"
+    cp $HDDIMG_MNT/initrd $BOOTFS_MNT 1>&3 2>&1 || die "Failed to copy initrd"
+  else
+    warn "initrd missing"
+fi
+echo "bootx64.efi" > $BOOTFS_MNT/startup.nsh || die "Failed to create startup.nsh"
 # Copy the efi loader and configs (booti*.efi and grub.cfg if it exists)
-cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT 1>&3 2>&1 || error "Failed to copy EFI dir"
+cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT 1>&3 2>&1 || die "Failed to copy EFI dir"
 # Silently ignore a missing systemd-boot or gummiboot loader dir (we might just be a GRUB image)
 cp -r $HDDIMG_MNT/loader $BOOTFS_MNT 1>&3 2>&1
 
@@ -393,7 +481,7 @@ if [ ! -e $GRUB_CFG ]  && [ ! -e $SYSTEMDBOOT_CFG ] ; then
 	die "No EFI bootloader configuration found"
 fi
 
-info "Copying ROOTFS files (this may take a while)"
+printf "Copying ROOTFS files ... "
 command -v rsync >/dev/null 2>&1 # check if rsync exists
 if [ $DEBUG -eq 1 ] && [ $? -eq 0 ]; then
 	rsync --info=progress2 -h -aHAXW --no-compress  $HDDIMG_ROOTFS_MNT/* $ROOTFS_MNT 1>&3 2>&1 || die "Root FS copy failed"
@@ -401,6 +489,10 @@ else
 	cp -a $HDDIMG_ROOTFS_MNT/* $ROOTFS_MNT 1>&3 2>&1 || die "Root FS copy failed"
 fi
 
+printf "flushing data on removable device. May take a while ... "
+sync --file-system $ROOTFS_MNT
+echo done
+
 # We dont want udev to mount our root device while we're booting...
 if [ -d $ROOTFS_MNT/etc/udev/ ] ; then
 	echo "$TARGET_DEVICE" >> $ROOTFS_MNT/etc/udev/mount.blacklist