diff options
| -rwxr-xr-x | scripts/sdks/agl/_build-sdks-json.sh | 235 |
1 files changed, 169 insertions, 66 deletions
diff --git a/scripts/sdks/agl/_build-sdks-json.sh b/scripts/sdks/agl/_build-sdks-json.sh index 1357d63..6539c5b 100755 --- a/scripts/sdks/agl/_build-sdks-json.sh +++ b/scripts/sdks/agl/_build-sdks-json.sh @@ -17,102 +17,172 @@ # limitations under the License. ########################################################################### -SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL" -SDK_AGL_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS" +# script_name=$(basename ${0}) +CURL="curl -f -s --connect-timeout 10" -# Define urls where SDKs can be downloaded -DOWNLOADABLE_URLS=" - ${SDK_AGL_BASEURL}/snapshots/master/latest/*/deploy/sdk +### Define urls where SDKs can be downloaded / AGL part +SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL/" +DOWNLOADABLE_AGL_URLS=" + snapshots/master/latest/* - ${SDK_AGL_BASEURL}/release/dab/4.0.3/*/deploy/sdk + release/dab/4.0.3/* - ${SDK_AGL_BASEURL}/release/eel/5.1.0/*/deploy/sdk - ${SDK_AGL_BASEURL}/release/eel/latest/*/deploy/sdk + release/eel/5.1.0/* + release/eel/latest/* - ${SDK_AGL_BASEURL}/release/flounder/6.0.0/*/deploy/sdk - ${SDK_AGL_BASEURL}/release/flounder/latest/*/deploy/sdk + release/flounder/6.0.0/* + release/flounder/latest/* +" +# release/dab/4.0.2/* +# +# release/eel/5.0.0/* +# release/eel/5.0.1/* +# release/eel/5.0.2/* +# release/eel/5.0.3/* +# release/eel/5.1.0/* +# release/eel/latest/ +# +# release/flounder/5.99.6/* - ${SDK_AGL_IOTBZH_BASEURL}/images/dab-m3ulcb/*/sdk +SDK_AGL_URL_SUFFIX="/deploy/sdk/" +IMG_AGL_URL_SUFFIX="/deploy/images/" +####################################### - ${SDK_AGL_IOTBZH_BASEURL}/images/eel-intel-corei7-64/*/sdk - ${SDK_AGL_IOTBZH_BASEURL}/images/eel-m3ulcb/*/sdk +### Define urls where SDKs can be downloaded / IoT.bzh part +SDK_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS/images/" +DOWNLOADABLE_IOTBZH_URLS=" + dab-m3ulcb/* - ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-h3ulcb/*/sdk - ${SDK_AGL_IOTBZH_BASEURL}/images/flounder-m3ulcb/*/sdk + eel-intel-corei7-64/* + eel-m3ulcb/* - ${SDK_AGL_IOTBZH_BASEURL}/images/master-m3ulcb/*/sdk + flounder-h3ulcb/* + flounder-m3ulcb/* + flounder-qemux86-64/* + + master-m3ulcb/* " -### + +SDK_IOTBZH_URL_SUFFIX="/sdk/" +IMG_IOTBZH_URL_SUFFIX="/images/" +####################################### # Compute full urls list (parse '*' characters) -urls="" -for url in $(echo $DOWNLOADABLE_URLS); do +function expand_url { # url, prefix, suffix + local result="" + url=$2$1 if [[ "$url" = *"*"* ]]; then bUrl=$(echo $url | cut -d'*' -f 1) eUrl=$(echo $url | cut -d'*' -f 2) dirs=$(curl -s ${bUrl} | grep '\[DIR\]' | grep -oP 'href="[^"]*"' | cut -d'"' -f 2) for dir in $(echo $dirs); do - urls="$urls ${bUrl::-1}/${dir::-1}/${eUrl:1}" + result="$result ${bUrl::-1}/${dir::-1}${eUrl:1}" done else - urls="$urls $url" - fi -done - -# Compute list of available/installable SDKs -sdksList=" " -for url in $(echo $urls); do - htmlPage=$(curl -s --connect-timeout 10 "${url}/") - files=$(echo ${htmlPage} | egrep -o 'href="[^"]*.sh"' | cut -d '"' -f 2) - if [ "$?" != "0" ] || [ "${files}" = "" ]; then - echo " IGNORED ${url}: no valid files found" - continue + result="$url" fi - for sdkFile in $(echo ${files}); do + echo $result +} - # Ignored silent some known files - [[ "${sdkFile}" = "agl-sdk-latest.sh" ]] && continue +function dbg_error() { >&2 echo "ERROR:" $@; } +function dbg_warning(){ [ $DEBUG -ge 1 ] && { >&2 echo "WARNING:" $@; } ; } +function dbg_info() { [ $DEBUG -ge 1 ] && { >&2 echo "INFO:" $@; } ; } +function dbg_debug() { [ $DEBUG -ge 2 ] && { >&2 echo "DEBUG:" $@; } ; } - # assume that sdk name follow this format : - # _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh - # for example: - # poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh +function count() { echo $#; } - [[ "${sdkFile}" != *"crosssdk"* ]] && { echo " IGNORED ${sdkFile}, not a valid sdk file"; echo " (url: ${url})"; continue; } +function first() { echo $1; } - echo "Processing ${sdkFile}" - profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/') - version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/') - arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/') - endUrl=${url#$SDK_AGL_BASEURL} - if [ "${url::14}" = "http://iot.bzh" ]; then - endUrl=${url#$SDK_AGL_IOTBZH_BASEURL} - name=$(echo "IoTbzh-$(echo ${endUrl} | cut -d'/' -f3,4 | sed s:/:-:g)-${version}") - elif [ "${endUrl::4}" = "http" ]; then - name="${profile}_${arch}_${version}" +function search_image_url { + htmlPage=$(${CURL} ${1}/) + if [[ $? == 0 ]]; then # curl request without error, at least URL is OK + if [ "${2}" = "qemux86-64" ]; then + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.vmdk(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/') else - name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f2,3,4,5)" | sed s:/:-:g) + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.rootfs\.wic(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/') + # search for tar.xz if no wic found... + if [ $(count $image_name) -eq 0 ]; then + image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.tar\.xz"')) | sed -E 's/.*href="([^"]*)".*/\1/') + fi fi + else + dbg_warning "could not list content of ${1}, HTTP request failed." + image_name="null" + fi - # Distringuish qemux86-64 and corei7-64 - if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then - arch="qemux86-64" - fi + # return string result + if [[ ${image_name} = "null" ]]; then + echo "null" # to be inserted as-is in the JSON output + else + echo "\"${1}/${image_name}\"" # if no image file found, but URL is valid, lets point to the directory + fi +} + +# Given an URL, generate the corresponding JSON chunk +function generate_json { + url=$1 + SDK_URL=$1$2 + IMG_URL=$1$3 + + arch="" + htmlPage=$(${CURL} "${SDK_URL}") + files=$(echo ${htmlPage} | egrep -o 'href="[^"]*crosssdk[^"]*\.sh"' | cut -d '"' -f 2) + if [ "$?" != "0" ] || [ "${files}" = "" ]; then + dbg_warning " IGNORED ${url}: no valid files found" + return + fi + + # assume that sdk name follow this format : + # _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh + # for example: + # poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh + if [ $(count files) -gt 1 ] + then + dbg_warning " IGNORE ${url}: multiple shell script containing 'crosssdk' in their names" + return + fi + sdkFile=${files} + dbg_info "Processing SDK: ${sdkFile} (from ${url})" + profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/') + version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/') + arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/') + + endUrl=${url#$SDK_AGL_BASEURL} + if [ "${url::14}" = "http://iot.bzh" ]; then + endUrl=${url#$SDK_IOTBZH_BASEURL} + name=$(echo "IoTbzh-$(echo ${endUrl} | sed s:/:-:g)-${version}") + board="" + elif [ "${endUrl::4}" = "http" ]; then + name="${profile}_${arch}_${version}" + board="" + else + name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f1,2,3,4)" | sed s:/:-:g) + board=$(echo ${endUrl} | cut -d'/' -f4) + board=${board%-nogfx} + fi + + # Distringuish qemux86-64 and corei7-64 + if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then + arch="qemux86-64" + fi + + [ "${profile}" = "" ] && { dbg_error "profile is not set" ; return ; } + [ "${version}" = "" ] && { dbg_error "version is not set" ; return ; } + [ "${arch}" = "" ] && { dbg_error "arch is not set" ; return ; } + [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; } - [ "${profile}" = "" ] && { echo " ERROR: profile not set" continue; } - [ "${version}" = "" ] && { echo " ERROR: version not set" continue; } - [ "${arch}" = "" ] && { echo " ERROR: arch not set" continue; } - [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; } + sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)" + sdkSize="$(echo "${htmlPage}" |egrep -o "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)" + md5sum="$(wget -q -O - ${SDK_URL}/${sdkFile/.sh/.md5} |cut -d' ' -f1)" - sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)" - sdkSize="$(echo "${htmlPage}" |egrep -o "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)" - md5sum="$(wget -q -O - ${url}/${sdkFile/.sh/.md5} |cut -d' ' -f1)" + image_url=$(search_image_url ${IMG_URL}${board} ${arch}) - read -r -d '' res <<- EndOfMessage + + read -r -d '' res <<- EndOfMessage { "name": "${name}", "description": "AGL SDK ${arch} (version ${version})", @@ -120,7 +190,8 @@ for url in $(echo $urls); do "version": "${version}", "arch": "${arch}", "path": "", - "url": "${url}/${sdkFile}", + "url": "${SDK_URL}${sdkFile}", + "image_url": ${image_url}, "status": "Not Installed", "date": "${sdkDate}", "size": "${sdkSize}", @@ -129,11 +200,43 @@ for url in $(echo $urls); do }, EndOfMessage - sdksList="${sdksList}${res}" + sdksList="${sdksList}${res}" +} + +DEBUG=0 +if [[ ${1:0:2} = "-v" ]] ; then + verbose=$(echo -s ${1} | sed -E "s/-([v]+).*/\1/") + DEBUG=$(echo ${verbose} | wc -c) + echo "Set verbosity level to : ${DEBUG}" + shift +fi + +# Compute list of available/installable SDKs +sdksList=" " +for base_url in $DOWNLOADABLE_AGL_URLS +do + dbg_debug "Look into ${base_url}" + urls="$(expand_url $base_url $SDK_AGL_BASEURL)" + for url in $urls + do + dbg_debug "URL to check: ${url}" + generate_json $url $SDK_AGL_URL_SUFFIX $IMG_AGL_URL_SUFFIX done done -OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%m").json +for base_url in $DOWNLOADABLE_IOTBZH_URLS +do + dbg_debug "Look into ${base_url}" + urls="$(expand_url $base_url $SDK_IOTBZH_BASEURL)" + for url in $urls + do + dbg_debug "URL to check: ${url}" + generate_json $url $SDK_IOTBZH_URL_SUFFIX $IMG_IOTBZH_URL_SUFFIX + done +done + + +OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%M").json echo "[" > ${OUT_FILE} echo "${sdksList::-1}" >> ${OUT_FILE} |