From ecef0475a0a9d51e1a8842fbf0e0b9a948565cbd Mon Sep 17 00:00:00 2001 From: Johann CAHIER Date: Fri, 19 Oct 2018 10:41:14 +0200 Subject: Adds 'image_url' JSON parameter to an SDK entry Adds a link to installable target image corresponding to the current SDK. The easest way to implement this was to manage a pair of suffix for earch source of SDKs (AGL or IoT.bzh), The input data are refactored as following: each source has an URL prefix, a list of directories (possibly including wildcard) to be browsed, relative to prefix URL, and a pair of suffixes : one for SDK, one for ilage (relative to prefix + directory). When an SDK installation file is found, the corresponding image is searched by replacing the SDK suffix by the image one, requesting this URL, and grepping for a filename matching agl-demo-plaform-crosssdk-* With different possible extensions: - .vmdk(.xz) in case of qEMU image, or - .rootfs.wic(.xz), when avaible, or - .rrotfs.tar(.xz) If no matching image file is found, the image_url is set to the directory URL containing the image file candidates. If requesting the image directory URL content leads to server error (mainly error 404 Not Found, but also any other error), the image_url value will be set to null in the JSON output. Makes the trace system more flexible, using an verbose -v|vv option. KNOWN ISSUES: - '-nogfx' images are not named agl-demo-plaform-crosssdk-* on the AGL server. image_url is set to the image directory URL for such SDKs/images - on IoT.bzh, older images directories are named 'image' while newer ones are named 'images'. Advantage given to newer images, this lead to have older SDKs' image_url set to null. Change-Id: I8c029d3ebbdef1a096c5cb225fd40405e91b312d Signed-off-by: Johann CAHIER --- scripts/sdks/agl/_build-sdks-json.sh | 235 +++++++++++++++++++++++++---------- 1 file changed, 169 insertions(+), 66 deletions(-) diff --git a/scripts/sdks/agl/_build-sdks-json.sh b/scripts/sdks/agl/_build-sdks-json.sh index 1357d63..6539c5b 100755 --- a/scripts/sdks/agl/_build-sdks-json.sh +++ b/scripts/sdks/agl/_build-sdks-json.sh @@ -17,102 +17,172 @@ # limitations under the License. ########################################################################### -SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL" -SDK_AGL_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS" +# script_name=$(basename ${0}) +CURL="curl -f -s --connect-timeout 10" -# Define urls where SDKs can be downloaded -DOWNLOADABLE_URLS=" - ${SDK_AGL_BASEURL}/snapshots/master/latest/*/deploy/sdk +### Define urls where SDKs can be downloaded / AGL part +SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL/" +DOWNLOADABLE_AGL_URLS=" + snapshots/master/latest/* - ${SDK_AGL_BASEURL}/release/dab/4.0.3/*/deploy/sdk + release/dab/4.0.3/* - ${SDK_AGL_BASEURL}/release/eel/5.1.0/*/deploy/sdk - ${SDK_AGL_BASEURL}/release/eel/latest/*/deploy/sdk + release/eel/5.1.0/* + release/eel/latest/* - ${SDK_AGL_BASEURL}/release/flounder/6.0.0/*/deploy/sdk - ${SDK_AGL_BASEURL}/release/flounder/latest/*/deploy/sdk + release/flounder/6.0.0/* + release/flounder/latest/* +" +# release/dab/4.0.2/* +# +# release/eel/5.0.0/* +# release/eel/5.0.1/* +# release/eel/5.0.2/* +# release/eel/5.0.3/* +# release/eel/5.1.0/* +# release/eel/latest/ +# +# release/flounder/5.99.6/* - ${SDK_AGL_IOTBZH_BASEURL}/images/dab-m3ulcb/*/sdk +SDK_AGL_URL_SUFFIX="/deploy/sdk/" +IMG_AGL_URL_SUFFIX="/deploy/images/" +####################################### - ${SDK_AGL_IOTBZH_BASEURL}/images/eel-intel-corei7-64/*/sdk - ${SDK_AGL_IOTBZH_BASEURL}/images/eel-m3ulcb/*/sdk +### Define urls where SDKs can be downloaded / IoT.bzh part +SDK_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS/images/" +DOWNLOADABLE_IOTBZH_URLS=" + dab-m3ulcb/* - ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-h3ulcb/*/sdk - ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-m3ulcb/*/sdk + eel-intel-corei7-64/* + eel-m3ulcb/* - ${SDK_AGL_IOTBZH_BASEURL}/images/master-m3ulcb/*/sdk + flounder-h3ulcb/* + flounder-m3ulcb/* + flounder-qemux86-64/* + + master-m3ulcb/* " -### + +SDK_IOTBZH_URL_SUFFIX="/sdk/" +IMG_IOTBZH_URL_SUFFIX="/images/" +####################################### # Compute full urls list (parse '*' characters) -urls="" -for url in $(echo $DOWNLOADABLE_URLS); do +function expand_url { # url, prefix, suffix + local result="" + url=$2$1 if [[ "$url" = *"*"* ]]; then bUrl=$(echo $url | cut -d'*' -f 1) eUrl=$(echo $url | cut -d'*' -f 2) dirs=$(curl -s ${bUrl} | grep '\[DIR\]' | grep -oP 'href="[^"]*"' | cut -d'"' -f 2) for dir in $(echo $dirs); do - urls="$urls ${bUrl::-1}/${dir::-1}/${eUrl:1}" + result="$result ${bUrl::-1}/${dir::-1}${eUrl:1}" done else - urls="$urls $url" - fi -done - -# Compute list of available/installable SDKs -sdksList=" " -for url in $(echo $urls); do - htmlPage=$(curl -s --connect-timeout 10 "${url}/") - files=$(echo ${htmlPage} | egrep -o 'href="[^"]*.sh"' | cut -d '"' -f 2) - if [ "$?" != "0" ] || [ "${files}" = "" ]; then - echo " IGNORED ${url}: no valid files found" - continue + result="$url" fi - for sdkFile in $(echo ${files}); do + echo $result +} - # Ignored silent some known files - [[ "${sdkFile}" = "agl-sdk-latest.sh" ]] && continue +function dbg_error() { >&2 echo "ERROR:" $@; } +function dbg_warning(){ [ $DEBUG -ge 1 ] && { >&2 echo "WARNING:" $@; } ; } +function dbg_info() { [ $DEBUG -ge 1 ] && { >&2 echo "INFO:" $@; } ; } +function dbg_debug() { [ $DEBUG -ge 2 ] && { >&2 echo "DEBUG:" $@; } ; } - # assume that sdk name follow this format : - # _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh - # for example: - # poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh +function count() { echo $#; } - [[ "${sdkFile}" != *"crosssdk"* ]] && { echo " IGNORED ${sdkFile}, not a valid sdk file"; echo " (url: ${url})"; continue; } +function first() { echo $1; } - echo "Processing ${sdkFile}" - profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/') - version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/') - arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/') - endUrl=${url#$SDK_AGL_BASEURL} - if [ "${url::14}" = "http://iot.bzh" ]; then - endUrl=${url#$SDK_AGL_IOTBZH_BASEURL} - name=$(echo "IoTbzh-$(echo ${endUrl} | cut -d'/' -f3,4 | sed s:/:-:g)-${version}") - elif [ "${endUrl::4}" = "http" ]; then - name="${profile}_${arch}_${version}" +function search_image_url { + htmlPage=$(${CURL} ${1}/) + if [[ $? == 0 ]]; then # curl request without error, at least URL is OK + if [ "${2}" = "qemux86-64" ]; then + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.vmdk(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/') else - name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f2,3,4,5)" | sed s:/:-:g) + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.rootfs\.wic(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/') + # search for tar.xz if no wic found... + if [ $(count $image_name) -eq 0 ]; then + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.tar\.xz"')) | sed -E 's/.*href="([^"]*)".*/\1/') + fi fi + else + dbg_warning "could not list content of ${1}, HTTP request failed." + image_name="null" + fi - # Distringuish qemux86-64 and corei7-64 - if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then - arch="qemux86-64" - fi + # return string result + if [[ ${image_name} = "null" ]]; then + echo "null" # to be inserted as-is in the JSON output + else + echo "\"${1}/${image_name}\"" # if no image file found, but URL is valid, lets point to the directory + fi +} + +# Given an URL, generate the corresponding JSON chunk +function generate_json { + url=$1 + SDK_URL=$1$2 + IMG_URL=$1$3 + + arch="" + htmlPage=$(${CURL} "${SDK_URL}") + files=$(echo ${htmlPage} | egrep -o 'href="[^"]*crosssdk[^"]*\.sh"' | cut -d '"' -f 2) + if [ "$?" != "0" ] || [ "${files}" = "" ]; then + dbg_warning " IGNORED ${url}: no valid files found" + return + fi + + # assume that sdk name follow this format : + # _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh + # for example: + # poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh + if [ $(count files) -gt 1 ] + then + dbg_warning " IGNORE ${url}: multiple shell script containing 'crosssdk' in their names" + return + fi + sdkFile=${files} + dbg_info "Processing SDK: ${sdkFile} (from ${url})" + profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/') + version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/') + arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/') + + endUrl=${url#$SDK_AGL_BASEURL} + if [ "${url::14}" = "http://iot.bzh" ]; then + endUrl=${url#$SDK_IOTBZH_BASEURL} + name=$(echo "IoTbzh-$(echo ${endUrl} | sed s:/:-:g)-${version}") + board="" + elif [ "${endUrl::4}" = "http" ]; then + name="${profile}_${arch}_${version}" + board="" + else + name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f1,2,3,4)" | sed s:/:-:g) + board=$(echo ${endUrl} | cut -d'/' -f4) + board=${board%-nogfx} + fi + + # Distringuish qemux86-64 and corei7-64 + if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then + arch="qemux86-64" + fi + + [ "${profile}" = "" ] && { dbg_error "profile is not set" ; return ; } + [ "${version}" = "" ] && { dbg_error "version is not set" ; return ; } + [ "${arch}" = "" ] && { dbg_error "arch is not set" ; return ; } + [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; } - [ "${profile}" = "" ] && { echo " ERROR: profile not set" continue; } - [ "${version}" = "" ] && { echo " ERROR: version not set" continue; } - [ "${arch}" = "" ] && { echo " ERROR: arch not set" continue; } - [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; } + sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)" + sdkSize="$(echo "${htmlPage}" |egrep -o "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+" |cut -d'>' -f7 |cut -d'<' -f1)" + md5sum="$(wget -q -O - ${SDK_URL}/${sdkFile/.sh/.md5} |cut -d' ' -f1)" - sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)" - sdkSize="$(echo "${htmlPage}" |egrep -o "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+" |cut -d'>' -f7 |cut -d'<' -f1)" - md5sum="$(wget -q -O - ${url}/${sdkFile/.sh/.md5} |cut -d' ' -f1)" + image_url=$(search_image_url ${IMG_URL}${board} ${arch}) - read -r -d '' res <<- EndOfMessage + + read -r -d '' res <<- EndOfMessage { "name": "${name}", "description": "AGL SDK ${arch} (version ${version})", @@ -120,7 +190,8 @@ for url in $(echo $urls); do "version": "${version}", "arch": "${arch}", "path": "", - "url": "${url}/${sdkFile}", + "url": "${SDK_URL}${sdkFile}", + "image_url": ${image_url}, "status": "Not Installed", "date": "${sdkDate}", "size": "${sdkSize}", @@ -129,11 +200,43 @@ for url in $(echo $urls); do }, EndOfMessage - sdksList="${sdksList}${res}" + sdksList="${sdksList}${res}" +} + +DEBUG=0 +if [[ ${1:0:2} = "-v" ]] ; then + verbose=$(echo -s ${1} | sed -E "s/-([v]+).*/\1/") + DEBUG=$(echo ${verbose} | wc -c) + echo "Set verbosity level to : ${DEBUG}" + shift +fi + +# Compute list of available/installable SDKs +sdksList=" " +for base_url in $DOWNLOADABLE_AGL_URLS +do + dbg_debug "Look into ${base_url}" + urls="$(expand_url $base_url $SDK_AGL_BASEURL)" + for url in $urls + do + dbg_debug "URL to check: ${url}" + generate_json $url $SDK_AGL_URL_SUFFIX $IMG_AGL_URL_SUFFIX done done -OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%m").json +for base_url in $DOWNLOADABLE_IOTBZH_URLS +do + dbg_debug "Look into ${base_url}" + urls="$(expand_url $base_url $SDK_IOTBZH_BASEURL)" + for url in $urls + do + dbg_debug "URL to check: ${url}" + generate_json $url $SDK_IOTBZH_URL_SUFFIX $IMG_IOTBZH_URL_SUFFIX + done +done + + +OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%M").json echo "[" > ${OUT_FILE} echo "${sdksList::-1}" >> ${OUT_FILE} -- cgit 1.2.3-korg