Adds 'image_url' JSON parameter to an SDK entry

Adds a link to installable target image corresponding to the current
SDK. The easest way to implement this was to manage a pair of suffix
for earch source of SDKs (AGL or IoT.bzh), The input data are
refactored as following: each source has an URL prefix, a list of
directories (possibly including wildcard) to be browsed, relative to
prefix URL, and a pair of suffixes : one for SDK, one for ilage
(relative to prefix + directory).

When an SDK installation file is found, the corresponding image is
searched by replacing the SDK suffix by the image one, requesting this
URL, and grepping for a filename matching agl-demo-plaform-crosssdk-*
With different possible extensions:
- .vmdk(.xz) in case of qEMU image, or
- .rootfs.wic(.xz), when avaible, or
- .rrotfs.tar(.xz)

If no matching image file is found, the image_url is set to the
directory URL containing the image file candidates.

If requesting the image directory URL content leads to server error
(mainly error 404 Not Found, but also any other error), the image_url
value will be set to null in the JSON output.

Makes the trace system more flexible, using an verbose -v|vv option.

KNOWN ISSUES:
    - '-nogfx' images are not named agl-demo-plaform-crosssdk-* on the
      AGL server. image_url is set to the image directory URL for such
      SDKs/images
    - on IoT.bzh, older images directories are named 'image' while
      newer ones are named 'images'. Advantage given to newer images,
      this lead to have older SDKs' image_url set to null.

Change-Id: I8c029d3ebbdef1a096c5cb225fd40405e91b312d
Signed-off-by: Johann CAHIER <johann.cahier@iot.bzh>

1 files changed, 169 insertions, 66 deletions

diff --git a/scripts/sdks/agl/_build-sdks-json.sh b/scripts/sdks/agl/_build-sdks-json.sh
index 1357d63..6539c5b 100755
--- a/scripts/sdks/agl/_build-sdks-json.sh
+++ b/scripts/sdks/agl/_build-sdks-json.sh
@@ -17,102 +17,172 @@
 # limitations under the License.
 ###########################################################################
 
-SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL"
-SDK_AGL_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS"
+# script_name=$(basename ${0})
+CURL="curl -f -s --connect-timeout 10"
 
-# Define urls where SDKs can be downloaded
-DOWNLOADABLE_URLS="
-    ${SDK_AGL_BASEURL}/snapshots/master/latest/*/deploy/sdk
+### Define urls where SDKs can be downloaded / AGL part
+SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL/"
+DOWNLOADABLE_AGL_URLS="
+    snapshots/master/latest/*
 
-    ${SDK_AGL_BASEURL}/release/dab/4.0.3/*/deploy/sdk
+    release/dab/4.0.3/*
 
-    ${SDK_AGL_BASEURL}/release/eel/5.1.0/*/deploy/sdk
-    ${SDK_AGL_BASEURL}/release/eel/latest/*/deploy/sdk
+    release/eel/5.1.0/*
+    release/eel/latest/*
 
-    ${SDK_AGL_BASEURL}/release/flounder/6.0.0/*/deploy/sdk
-    ${SDK_AGL_BASEURL}/release/flounder/latest/*/deploy/sdk
+    release/flounder/6.0.0/*
+    release/flounder/latest/*
+"
+#     release/dab/4.0.2/*
+#
+#     release/eel/5.0.0/*
+#     release/eel/5.0.1/*
+#     release/eel/5.0.2/*
+#     release/eel/5.0.3/*
+#     release/eel/5.1.0/*
+#     release/eel/latest/
+#
+#     release/flounder/5.99.6/*
 
-    ${SDK_AGL_IOTBZH_BASEURL}/images/dab-m3ulcb/*/sdk
+SDK_AGL_URL_SUFFIX="/deploy/sdk/"
+IMG_AGL_URL_SUFFIX="/deploy/images/"
+#######################################
 
-    ${SDK_AGL_IOTBZH_BASEURL}/images/eel-intel-corei7-64/*/sdk
-    ${SDK_AGL_IOTBZH_BASEURL}/images/eel-m3ulcb/*/sdk
+### Define urls where SDKs can be downloaded / IoT.bzh part
+SDK_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS/images/"
+DOWNLOADABLE_IOTBZH_URLS="
+    dab-m3ulcb/*
 
-    ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-h3ulcb/*/sdk
-    ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-m3ulcb/*/sdk
+    eel-intel-corei7-64/*
+    eel-m3ulcb/*
 
-    ${SDK_AGL_IOTBZH_BASEURL}/images/master-m3ulcb/*/sdk
+    flounder-h3ulcb/*
+    flounder-m3ulcb/*
+    flounder-qemux86-64/*
+
+    master-m3ulcb/*
 "
-###
+
+SDK_IOTBZH_URL_SUFFIX="/sdk/"
+IMG_IOTBZH_URL_SUFFIX="/images/"
+#######################################
 
 
 # Compute full urls list  (parse '*' characters)
-urls=""
-for url in $(echo $DOWNLOADABLE_URLS); do
+function expand_url { # url, prefix, suffix
+    local result=""
+    url=$2$1
     if [[ "$url" = *"*"* ]]; then
         bUrl=$(echo $url | cut -d'*' -f 1)
         eUrl=$(echo $url | cut -d'*' -f 2)
         dirs=$(curl -s ${bUrl} | grep '\[DIR\]' | grep -oP  'href="[^"]*"' | cut -d'"' -f 2)
         for dir in $(echo $dirs); do
-            urls="$urls ${bUrl::-1}/${dir::-1}/${eUrl:1}"
+            result="$result ${bUrl::-1}/${dir::-1}${eUrl:1}"
         done
     else
-        urls="$urls $url"
-    fi
-done
-
-# Compute list of available/installable SDKs
-sdksList=" "
-for url in $(echo $urls); do
-    htmlPage=$(curl -s --connect-timeout 10 "${url}/")
-    files=$(echo ${htmlPage} | egrep -o 'href="[^"]*.sh"' | cut -d '"' -f 2)
-    if [ "$?" != "0" ] || [ "${files}" = "" ]; then
-        echo " IGNORED ${url}: no valid files found"
-        continue
+        result="$url"
     fi
 
-    for sdkFile in $(echo ${files}); do
+    echo $result
+}
 
-		# Ignored silent some known files
-		[[ "${sdkFile}" = "agl-sdk-latest.sh" ]] && continue
+function dbg_error()  { >&2 echo "ERROR:" $@; }
+function dbg_warning(){ [ $DEBUG -ge 1 ] && { >&2 echo "WARNING:" $@; } ; }
+function dbg_info()   { [ $DEBUG -ge 1 ] && { >&2 echo "INFO:" $@; } ; }
+function dbg_debug()   { [ $DEBUG -ge 2 ] && { >&2 echo "DEBUG:" $@; } ; }
 
-        # assume that sdk name follow this format :
-        #  _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh
-        # for example:
-        #  poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh
+function count() { echo $#; }
 
-        [[ "${sdkFile}" != *"crosssdk"* ]] && { echo " IGNORED ${sdkFile}, not a valid sdk file"; echo " (url: ${url})"; continue; }
+function first() { echo $1; }
 
-        echo "Processing ${sdkFile}"
-        profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/')
-        version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/')
-        arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/')
 
-        endUrl=${url#$SDK_AGL_BASEURL}
-        if [ "${url::14}" = "http://iot.bzh" ]; then
-            endUrl=${url#$SDK_AGL_IOTBZH_BASEURL}
-            name=$(echo "IoTbzh-$(echo ${endUrl} | cut -d'/' -f3,4 | sed s:/:-:g)-${version}")
-        elif [ "${endUrl::4}" = "http" ]; then
-            name="${profile}_${arch}_${version}"
+function search_image_url {
+    htmlPage=$(${CURL} ${1}/)
+    if [[ $? == 0 ]]; then # curl request without error, at least URL is OK
+        if [ "${2}" = "qemux86-64" ]; then
+            image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.vmdk(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
         else
-            name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f2,3,4,5)" | sed s:/:-:g)
+            image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.rootfs\.wic(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
+            # search for tar.xz if no wic found...
+            if [ $(count $image_name) -eq 0 ]; then
+                image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.tar\.xz"')) | sed -E 's/.*href="([^"]*)".*/\1/')
+            fi
         fi
+    else
+        dbg_warning "could not list content of ${1}, HTTP request failed."
+        image_name="null"
+    fi
 
-        # Distringuish qemux86-64 and corei7-64
-        if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then
-            arch="qemux86-64"
-        fi
+    # return string result
+    if [[ ${image_name} = "null" ]]; then
+        echo "null" # to be inserted as-is in the JSON output
+    else
+        echo "\"${1}/${image_name}\"" # if no image file found, but URL is valid, lets point to the directory
+    fi
+}
+
+# Given an URL, generate the corresponding JSON chunk
+function generate_json {
+    url=$1
+    SDK_URL=$1$2
+    IMG_URL=$1$3
+
+    arch=""
 
+    htmlPage=$(${CURL} "${SDK_URL}")
+    files=$(echo ${htmlPage} | egrep -o 'href="[^"]*crosssdk[^"]*\.sh"' | cut -d '"' -f 2)
+    if [ "$?" != "0" ] || [ "${files}" = "" ]; then
+        dbg_warning " IGNORED ${url}: no valid files found"
+        return
+    fi
+
+    # assume that sdk name follow this format :
+    #  _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh
+    # for example:
+    #  poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh
+    if [ $(count files) -gt 1 ]
+    then
+        dbg_warning " IGNORE ${url}: multiple shell script containing 'crosssdk' in their names"
+        return
+    fi
+    sdkFile=${files}
+    dbg_info "Processing SDK: ${sdkFile} (from ${url})"
+    profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/')
+    version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/')
+    arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/')
+
+    endUrl=${url#$SDK_AGL_BASEURL}
+    if [ "${url::14}" = "http://iot.bzh" ]; then
+        endUrl=${url#$SDK_IOTBZH_BASEURL}
+        name=$(echo "IoTbzh-$(echo ${endUrl} | sed s:/:-:g)-${version}")
+        board=""
+    elif [ "${endUrl::4}" = "http" ]; then
+        name="${profile}_${arch}_${version}"
+        board=""
+    else
+        name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f1,2,3,4)" | sed s:/:-:g)
+        board=$(echo ${endUrl} | cut -d'/' -f4)
+        board=${board%-nogfx}
+    fi
+
+    # Distringuish qemux86-64 and corei7-64
+    if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then
+        arch="qemux86-64"
+    fi
+
+    [ "${profile}" = "" ] && { dbg_error "profile is not set" ; return ; }
+    [ "${version}" = "" ] && { dbg_error "version is not set" ; return ; }
+    [ "${arch}" = "" ] && { dbg_error "arch is not set" ; return ; }
+    [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; }
 
-        [ "${profile}" = "" ] && { echo " ERROR: profile not set" continue; }
-        [ "${version}" = "" ] && { echo " ERROR: version not set" continue; }
-        [ "${arch}" = "" ] && { echo " ERROR: arch not set" continue; }
-        [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; }
+    sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)"
+    sdkSize="$(echo "${htmlPage}" |egrep -o  "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)"
+    md5sum="$(wget -q -O - ${SDK_URL}/${sdkFile/.sh/.md5} |cut -d' ' -f1)"
 
-        sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)"
-        sdkSize="$(echo "${htmlPage}" |egrep -o  "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)"
-        md5sum="$(wget -q -O - ${url}/${sdkFile/.sh/.md5} |cut -d' ' -f1)"
+    image_url=$(search_image_url ${IMG_URL}${board} ${arch})
 
-        read -r -d '' res <<- EndOfMessage
+
+    read -r -d '' res <<- EndOfMessage
 {
     "name":         "${name}",
     "description":  "AGL SDK ${arch} (version ${version})",
@@ -120,7 +190,8 @@ for url in $(echo $urls); do
     "version":      "${version}",
     "arch":         "${arch}",
     "path":         "",
-    "url":          "${url}/${sdkFile}",
+    "url":          "${SDK_URL}${sdkFile}",
+    "image_url":    ${image_url},
     "status":       "Not Installed",
     "date":         "${sdkDate}",
     "size":         "${sdkSize}",
@@ -129,11 +200,43 @@ for url in $(echo $urls); do
 },
 EndOfMessage
 
-        sdksList="${sdksList}${res}"
+    sdksList="${sdksList}${res}"
+}
+
+DEBUG=0
+if [[ ${1:0:2} = "-v" ]] ; then
+    verbose=$(echo -s ${1} | sed -E "s/-([v]+).*/\1/")
+    DEBUG=$(echo ${verbose} | wc -c)
+    echo "Set verbosity level to : ${DEBUG}"
+    shift
+fi
+
+# Compute list of available/installable SDKs
+sdksList=" "
+for base_url in $DOWNLOADABLE_AGL_URLS
+do
+    dbg_debug "Look into ${base_url}"
+    urls="$(expand_url $base_url $SDK_AGL_BASEURL)"
+    for url in $urls
+    do
+        dbg_debug "URL to check: ${url}"
+        generate_json $url $SDK_AGL_URL_SUFFIX $IMG_AGL_URL_SUFFIX
     done
 done
 
-OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%m").json
+for base_url in $DOWNLOADABLE_IOTBZH_URLS
+do
+    dbg_debug "Look into ${base_url}"
+    urls="$(expand_url $base_url $SDK_IOTBZH_BASEURL)"
+    for url in $urls
+    do
+        dbg_debug "URL to check: ${url}"
+        generate_json $url $SDK_IOTBZH_URL_SUFFIX $IMG_IOTBZH_URL_SUFFIX
+    done
+done
+
+
+OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%M").json
 
 echo "[" > ${OUT_FILE}
 echo "${sdksList::-1}" >> ${OUT_FILE}