distro-manifest-generator: add support for JSON output formatguppy_6.99.1guppy/6.99.16.99.1

Distro build manifest is easier to parse if output is made in a structured
format. Adding JSON output format will allow bindings to read information
more easily.

The files created are now:

On target:
* /etc/platform-info/build (shell format)
* /etc/platform-info/build.json (JSON format)

In image deploy dir (.../tmp/deploy/images/$MACHINE/):
* build-info (shell format)
* build-info.json (JSON format)

In sdk deploy dir (.../tmp/deploy/sdk/):
* poky-agl-<version details>.build-info (shell format)
* poky-agl-<version details>.build-info.json (JSON format)

Bug-AGL: SPEC-720, SPEC-1917

Change-Id: If45d2c5dd96b15ce790aa7f7f97c24f119ad117b
Signed-off-by: Stephane Desneux <stephane.desneux@iot.bzh>

2 files changed, 272 insertions, 90 deletions

diff --git a/meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb b/meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
index 0ef48b272..a3b341230 100644
--- a/meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
+++ b/meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
@@ -1,30 +1,69 @@
 SUMMARY = "Distribution build manifest"
-DESCRIPTION = "The /etc/platform-build-info file contains build manifest (SPEC-720)."
+DESCRIPTION = "The folder /etc/platform-info contains build manifest (SPEC-720)."
 LICENSE = "MIT"
 
 # information distributed by the package is machine specific
 PACKAGE_ARCH = "${MACHINE_ARCH}"
 
 # dependencies of ${DISTRO_MANIFEST_GENERATOR}
-DEPENDS = "coreutils-native bash-native git-native gawk-native sed-native"
+DEPENDS = "coreutils-native bash-native git-native gawk-native sed-native jq-native"
 
 # force a rebuild everytime a build is started
 do_compile[nostamp] = "1"
 
-do_compilestep1 () {
+# borrowed to os-release.bb (output format is very close)
+python do_compilestep1 () {
+   import shutil
+   with open(d.expand('${B}/bbinfo-deploy'),'w') as f:
+      allkeys=[]
+      for field in d.getVar('BUILD_MANIFEST_FIELDS_DEPLOY').split():
+         key='DIST_BB_{0}'.format(field)
+         allkeys.append(key)
+         value=d.getVar(field)
+         if value:
+            f.write('{0}="{1}"\n'.format(key,value))
+      f.write('declare -A BITBAKE_VARS\nBITBAKE_VARS[deploy]="%s"' % ' '.join(allkeys))
+
+   with open(d.expand('${B}/bbinfo-target'),'w') as f:
+      allkeys=[]
+      for field in d.getVar('BUILD_MANIFEST_FIELDS_TARGET').split():
+         key='DIST_BB_{0}'.format(field)
+         allkeys.append(key)
+         value=d.getVar(field)
+         if value:
+            f.write('{0}="{1}"\n'.format(key,value))
+      f.write('declare -A BITBAKE_VARS\nBITBAKE_VARS[target]="%s"' % ' '.join(allkeys))
+
+   with open(d.expand('${B}/bbinfo-sdk'),'w') as f:
+      allkeys=[]
+      for field in d.getVar('BUILD_MANIFEST_FIELDS_SDK').split():
+         key='DIST_BB_{0}'.format(field)
+         allkeys.append(key)
+         value=d.getVar(field)
+         if value:
+            f.write('{0}="{1}"\n'.format(key,value))
+      f.write('declare -A BITBAKE_VARS\nBITBAKE_VARS[sdk]="%s"' % ' '.join(allkeys))
+}
+
+do_compilestep2 () {
     rc=99
-    outfile=${B}/platform-build-info
+    outfile=${B}/build-info
     if [ -x "${DISTRO_MANIFEST_GENERATOR}" -a -f "${DISTRO_SETUP_MANIFEST}" ]; then
-        ${DISTRO_MANIFEST_GENERATOR} ${DISTRO_SETUP_MANIFEST} deploy >${outfile}-deploy
-        rc1=$?
-        ${DISTRO_MANIFEST_GENERATOR} ${DISTRO_SETUP_MANIFEST} target >${outfile}-target
-        rc2=$?
-        ${DISTRO_MANIFEST_GENERATOR} ${DISTRO_SETUP_MANIFEST} sdk >${outfile}-sdk
-        rc=$?
-
-        if [ "$rc1" -ne 0 -o "$rc2" -ne 0 -o "$rc3" -ne 0 ]; then
-            rc=98
-        fi
+		rc=0
+		for	format in bash json; do
+			if [ "$format" = "json" ]; then
+				ext=".json"
+			else
+				ext=""
+			fi
+			for mode in deploy target sdk; do
+				${DISTRO_MANIFEST_GENERATOR} -m $mode -f $format -s ${B}/bbinfo-${mode} ${DISTRO_SETUP_MANIFEST} >${outfile}-${mode}${ext}
+				rc=$?
+				if [ $rc -ne 0 ]; then
+					break
+				fi
+			done
+		done
     else
         if [ -z "${DISTRO_MANIFEST_GENERATOR}" ]; then
             echo "The name of the generation script is not defined."
@@ -48,30 +87,9 @@ do_compilestep1 () {
     return $rc
 }
 
-# borrowed to os-release.bb (output format is very close)
-python do_compilestep2 () {
-   import shutil
-   with open(d.expand('${B}/platform-build-info-deploy'),'a') as f:
-      for field in d.getVar('BUILD_MANIFEST_FIELDS_DEPLOY').split():
-         value=d.getVar(field)
-         if value:
-            f.write('DIST_BB_{0}="{1}"\n'.format(field,value))
-
-   with open(d.expand('${B}/platform-build-info-target'),'a') as f:
-      for field in d.getVar('BUILD_MANIFEST_FIELDS_TARGET').split():
-         value=d.getVar(field)
-         if value:
-            f.write('DIST_BB_{0}="{1}"\n'.format(field,value))
-
-   with open(d.expand('${B}/platform-build-info-sdk'),'a') as f:
-      for field in d.getVar('BUILD_MANIFEST_FIELDS_SDK').split():
-         value=d.getVar(field)
-         if value:
-            f.write('DIST_BB_{0}="{1}"\n'.format(field,value))
-}
-do_compilestep2[vardeps] += " ${BUILD_MANIFEST_FIELDS_DEPLOY}"
-do_compilestep2[vardeps] += " ${BUILD_MANIFEST_FIELDS_TARGET}"
-do_compilestep2[vardeps] += " ${BUILD_MANIFEST_FIELDS_SDK}"
+do_compilestep1[vardeps] += " ${BUILD_MANIFEST_FIELDS_DEPLOY}"
+do_compilestep1[vardeps] += " ${BUILD_MANIFEST_FIELDS_TARGET}"
+do_compilestep1[vardeps] += " ${BUILD_MANIFEST_FIELDS_SDK}"
 
 # combine the two steps
 python do_compile() {
@@ -81,12 +99,19 @@ python do_compile() {
 
 do_install () {
     # install in target dir
-    install -d ${D}${sysconfdir}
-    install -m 0644 platform-build-info-target ${D}${sysconfdir}/platform-build-info
+    install -d ${D}${sysconfdir}/platform-info
+    install -m 0644 build-info-target ${D}${sysconfdir}/platform-info/build
+    install -m 0644 build-info-target.json ${D}${sysconfdir}/platform-info/build.json
 
     # also copy in deploy dir
     install -d ${DEPLOY_DIR_IMAGE}
-    install -m 0644 platform-build-info-deploy ${DEPLOY_DIR_IMAGE}/platform-build-info
+    install -m 0644 build-info-deploy ${DEPLOY_DIR_IMAGE}/build-info
+    install -m 0644 build-info-deploy.json ${DEPLOY_DIR_IMAGE}/build-info.json
+
+    # copy into sdk deploy dir
+    install -d ${DEPLOY_DIR}/sdk
+    install -m 0644 build-info-sdk ${DEPLOY_DIR}/sdk/${SDK_NAME}.build-info
+    install -m 0644 build-info-sdk.json ${DEPLOY_DIR}/sdk/${SDK_NAME}.build-info.json
 
     # and copy to nativesdk package
     # TODO
diff --git a/scripts/distro-manifest-generator.sh b/scripts/distro-manifest-generator.sh
index 9a910e5a5..834cde248 100755
--- a/scripts/distro-manifest-generator.sh
+++ b/scripts/distro-manifest-generator.sh
@@ -26,20 +26,132 @@
 #
 ################################################################################
 
+mode=deploy
+manifest=
+verbose=0
+format=bash
+sourcefile=
+
+function info() { echo "$@" >&2; }
+function error() { echo "$BASH_SOURCE: $@" >&2; }
+function out() { echo -n "$@"; }
+function out_object() {
+	# expected stdin stream is:
+	# --------------
+	# key
+	# value
+	# key
+	# value
+	# ...
+	# --------------
+	local sep=""
+	local k
+	case $format in
+		bash)
+			while read x; do
+				[[ -z "$k" ]] && { k="$x"; continue; }
+				out "$sep${k}="
+				out_value "$x"
+				sep=$'\n'
+				k=
+			done
+			out "$sep"
+			;;
+		json)
+			out "{"
+			while read x; do
+				[[ -z "$k" ]] && { k="$x"; continue; }
+				out "$sep\"${k}\":"
+				out_value "$x"
+				sep=","
+				k=
+			done
+			out "}"
+			;;
+	esac
+}
+
+function out_array() {
+	# expected stdin stream is:
+	# --------------
+	# value
+	# value
+	# ...
+	# --------------
+	local sep=""
+	case $format in
+		bash)
+			while read x; do
+				out "$sep"
+				out_value "$x"
+				sep=" "
+			done
+			;;
+		json)
+			out "["
+			while read x; do
+				out $sep
+				out_value "$x"
+				sep=","
+			done
+			out "]"
+			;;
+	esac
+}
+
+function out_value() {
+	# string
+	# number
+	# object
+	# array
+	# 'true'
+	# 'false'
+	# 'null'
+
+	x=$1
+
+	# litterals
+	if [[ "$x" =~ ^(true|false|null)$ ]]; then
+		out "$x"
+	# number
+	elif [[ "$x" =~ ^[+-]?[0-9]+(\.[0-9]+)?$ ]]; then
+		out "$x"
+	# object
+	elif [[ "$x" =~ ^\{.*\}$ ]]; then
+		out "$x"
+	# array
+	elif [[ "$x" =~ ^\[.*\]$ ]]; then
+		out "$x"
+	# string
+	else
+		out "\"$(sed 's/\("\)/\\\1/g' <<<$x)\""
+	fi
+}
+
+function out_comment() {
+	case $format in
+		bash)
+			[[ "$verbose" == 1 ]] && echo "# $@" || true
+			;;
+		json)
+			;;
+	esac
+}
+
 function _getgitmanifest() {
 	# this function takes the setup.manifest generated by setup script and uses DIST_METADIR
 	# to analyze git repos
 
 	local manifest=$1 mode=$2
-	[[ -f $manifest ]] && source $manifest || { echo "$BASH_SOURCE: Invalid setup manifest '$manifest'" >&2; return 1; }
+	[[ -f $manifest ]] && source $manifest || { error "Invalid setup manifest '$manifest'"; return 1; }
 	[[ ! -d "$DIST_METADIR" ]] && {
-		echo "$BASH_SOURCE: Invalid meta directory. Check variable DIST_METADIR in manifest file '$manifest'." >&2
-		echo "$BASH_SOURCE: Also, check directory '$DIST_METADIR'." >&2
+		error "Invalid meta directory. Check variable DIST_METADIR in manifest file '$manifest'."
+		error "$BASH_SOURCE: Also, check directory '$DIST_METADIR'."
 		return 2
 	}
 	local GIT=$(which git) REALPATH=$(which realpath)
-	[[ ! -x $GIT ]] && { echo "$BASH_SOURCE: Unable to find git command in $PATH." >&2; return 3; }
-	[[ ! -x $REALPATH ]] && { echo "$BASH_SOURCE: Unable to find realpath command in $PATH." >&2; return 4; }
+	[[ ! -x $GIT ]] && { error "$BASH_SOURCE: Unable to find git command in $PATH."; return 3; }
+	[[ ! -x $REALPATH ]] && { error "$BASH_SOURCE: Unable to find realpath command in $PATH."; return 4; }
 
 	local gitrepo gitrev metagitdir sep=""
 	DIST_LAYERS=""
@@ -58,6 +170,18 @@ function _getgitmanifest() {
 	# layers checksum
 	DIST_LAYERS_MD5=$(echo $DIST_LAYERS|md5sum -|awk '{print $1;}')
 
+	# in json, transform layers in an object, features in array
+	[[ "$format" == "json" ]] && {
+		DIST_FEATURES=$(for x in $DIST_FEATURES; do
+			echo $x
+		done | out_array)
+		DIST_LAYERS=$(for x in $DIST_LAYERS; do
+			echo ${x%%:*}
+			echo ${x#*:}
+		done | out_object)
+	}
+
+
 	# compute build hash
 	DIST_BUILD_HASH="F${DIST_FEATURES_MD5:0:8}-L${DIST_LAYERS_MD5:0:8}"
 	DIST_BUILD_ID="${DIST_DISTRO_NAME}-${DIST_MACHINE}-F${DIST_FEATURES_MD5:0:8}-L${DIST_LAYERS_MD5:0:8}"
@@ -81,66 +205,99 @@ function _getgitmanifest() {
 	EXTRA_VARS[target]="DIST_LAYERS DIST_BUILD_HASH DIST_BUILD_ID"
 	EXTRA_VARS[sdk]="DIST_LAYERS DIST_BUILD_HASH DIST_BUILD_ID"
 
-	echo "# setup variables in $mode mode"
-	for x in ${SETUP_VARS[$mode]}; do
-		echo "$x=\"${!x}\""
-	done
-	echo
+	# BITBAKE_VARS may be defined from external file to source (--source arg)
+	# this is used to dump extra vars from inside bitbake recipe
 
-	echo "# set by $BASH_SOURCE"
-	for x in ${EXTRA_VARS[$mode]}; do
-		echo "$x=\"${!x}\""
-	done
+	{ for x in ${SETUP_VARS[$mode]} ${EXTRA_VARS[$mode]} ${BITBAKE_VARS[$mode]}; do
+		k=$x
+		[[ "$format" == "json" ]] && {
+			k=${k#DIST_} # remove prefix
+			k=${k,,*} # to lower case
+		}
+		echo $k
+		echo ${!x}
+	done } | out_object
 }
 
 function getmanifest() {
 	local rc=0
-	echo "# DISTRO BUILD MANIFEST"
-	echo
+	out_comment "DISTRO BUILD MANIFEST"
+	out_comment
 
 	# add layers manifest
-	echo "# ----- this fragment has been generated by $BASH_SOURCE"
+	out_comment "----- this fragment has been generated by $BASH_SOURCE"
 	_getgitmanifest $1 $2 || rc=$?
-	echo "# ------------ end of $BASH_SOURCE fragment --------"
+	out_comment "------------ end of $BASH_SOURCE fragment --------"
 
 	return $rc
 }
 
+function __usage() {
+	cat <<EOF >&2
+Usage: $BASH_SOURCE [-v|--verbose] [-f|--format <fmt>] [-m|--mode <mode>] [-s|--source <file>] <setup_manifest_file>
+   Options:
+      -v|--verbose: generate comments in the output file
+      -s|--source: extra file to source (get extra variables generated from bitbake recipe)
+      -f|--format: specify output format: 'bash' or 'json'
+      -m|--mode: specify the destination for the generated manifest
+         'deploy' : for the tmp/deploy/images/* directories
+         'target' : for the manifest to be installed inside a target image
+         'sdk'    : for the manifest to be installed inside the SDK
+
+   <setup_manifest_file> is the input manifest generated from setup script
+EOF
+}
+
 set -e
 
-verbose=0
-if [[ "$1" =~ ^(-v|--verbose)$ ]]; then
-	shift
-	verbose=1
-fi
-
-if [[ -f "$1" ]]; then
-	manifest=$1
-	shift
-
-	# default mode
-	mode=${1:-deploy}
-	case $mode in
-		deploy|target|sdk) mode=$mode;;
-		*) echo "Invalid mode specified. Allow modes are: deploy target sdk"; exit 42;;
+tmp=$(getopt -o h,v,m:,f:,s: --long help,verbose,mode:,format:,source: -n "$BASH_SOURCE" -- "$@") || {
+	error "Invalid arguments."
+	__usage
+	exit 1
+}
+eval set -- $tmp
+
+while true; do
+	case "$1" in
+		-h|--help) __usage; exit 0;;
+		-v|--verbose) verbose=1; shift ;;
+		-f|--format) format=$2; shift 2;;
+		-m|--mode) mode=$2; shift 2;;
+		-s|--source) sourcefile=$2; shift 2;;
+		--) shift; break;;
+		*) fatal "Internal error";;
 	esac
+done
 
-	getmanifest $manifest $mode | { [[ $verbose == 1 ]] && cat || sed -e 's/#.*$//g;/^\s*$/d'; }
-	exit ${PIPESTATUS[0]}
-else
-	cat <<EOF >&2
-Usage: $0 [-v|--verbose] <setup_manifest_file> [mode]
-   Options:
-      -v|--verbose: generate comments in the output file
+manifest=$1
+shift
+[[ ! -f "$manifest" ]] && { __usage; exit 1; }
 
-   <setup_manifest_file> is generated from setup script in the specified build dir
+case $mode in
+	deploy|target|sdk) ;;
+	*) error "Invalid mode specified. Allowed modes are: 'deploy', 'target', 'sdk'"; __usage; exit 42;;
+esac
 
-   [mode] specifies the destination for the generated manifest
-   Accepted values:
-      'deploy' : for the tmp/deploy/images/* directories
-      'target' : for the manifest to be installed inside a target image
-      'sdk'    : for the manifest to be installed inside the SDK
-EOF
-	exit 56
-fi
+case $format in
+	bash|json) ;;
+	*) error "Invalid format specified. Allowed formats are 'json' or 'bash'"; __usage; exit 43;;
+esac
+
+info "Generating manifest: mode=$mode format=$format manifest=$manifest"
+[[ -f "$sourcefile" ]] && {
+	info "Sourcing file $sourcefile"
+	. $sourcefile
+	# this may define extra vars: to be taken into account BITBAKE_VARS must be defined
+}
+
+[[ "$format" == "json" ]] && {
+	# if jq is present, use it to format json output
+	jq=$(which jq || true)
+	[[ -n "$jq" ]] && {
+		getmanifest $manifest $mode | $jq ""
+		exit ${PIPESTATUS[0]}
+	}
+}
+
+getmanifest $manifest $mode