summaryrefslogtreecommitdiffstats
path: root/scripts/sdks/agl/_build-sdks-json.sh
blob: 7b67b86f5801ec43de0e669727a5309e482ba7e5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
#!/bin/bash
###########################################################################
# Copyright 2017-2018 IoT.bzh
#
# author: Sebastien Douheret <sebastien@iot.bzh>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###########################################################################

# script_name=$(basename ${0})
CURL="curl -f -s --connect-timeout 10"

### Define urls where SDKs can be downloaded / AGL part
SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL/"
DOWNLOADABLE_AGL_URLS="
    snapshots/master/latest/*

    release/dab/4.0.3/*

    release/eel/5.1.0/*
    release/eel/latest/*

    release/flounder/6.0.3/*
    release/flounder/latest/*

    release/guppy/6.99.2/*
    release/guppy/latest/*
"
#     release/dab/4.0.2/*
#
#     release/eel/5.0.0/*
#     release/eel/5.0.1/*
#     release/eel/5.0.2/*
#     release/eel/5.0.3/*
#     release/eel/5.1.0/*
#     release/eel/latest/
#
#     release/flounder/5.99.6/*

SDK_AGL_URL_SUFFIX="/deploy/sdk/"
IMG_AGL_URL_SUFFIX="/deploy/images/"
#######################################

### Define urls where SDKs can be downloaded / IoT.bzh part
SDK_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS/images/"
DOWNLOADABLE_IOTBZH_URLS="
    dab-m3ulcb/*

    eel-intel-corei7-64/*
    eel-m3ulcb/*

    flounder-h3ulcb/*
    flounder-m3ulcb/*
    flounder-qemux86-64/*

    master-m3ulcb/*
"

SDK_IOTBZH_URL_SUFFIX="/sdk/"
IMG_IOTBZH_URL_SUFFIX="/images/"
#######################################


# Compute full urls list  (parse '*' characters)
function expand_url { # url, prefix, suffix
    local result=""
    url=$2$1
    if [[ "$url" = *"*"* ]]; then
        bUrl=$(echo $url | cut -d'*' -f 1)
        eUrl=$(echo $url | cut -d'*' -f 2)
        dirs=$(curl -s ${bUrl} | grep '\[DIR\]' | grep -oP  'href="[^"]*"' | cut -d'"' -f 2)
        for dir in $(echo $dirs); do
            result="$result ${bUrl::-1}/${dir::-1}${eUrl:1}"
        done
    else
        result="$url"
    fi

    echo $result
}

function dbg_error()  { >&2 echo "ERROR:" $@; }
function dbg_warning(){ [ $DEBUG -ge 1 ] && { >&2 echo "WARNING:" $@; } ; }
function dbg_info()   { [ $DEBUG -ge 1 ] && { >&2 echo "INFO:" $@; } ; }
function dbg_debug()   { [ $DEBUG -ge 2 ] && { >&2 echo "DEBUG:" $@; } ; }

function count() { echo $#; }

function first() { echo $1; }


function search_image_url {
    htmlPage=$(${CURL} ${1}/)
    if [[ $? == 0 ]]; then # curl request without error, at least URL is OK
        if [ "${2}" = "qemux86-64" ]; then
            image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.vmdk(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
        else
            image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.rootfs\.wic(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
            # search for tar.xz if no wic found...
            if [ $(count $image_name) -eq 0 ]; then
                image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.tar\.xz"')) | sed -E 's/.*href="([^"]*)".*/\1/')
            fi
        fi
    else
        dbg_warning "could not list content of ${1}, HTTP request failed."
        image_name="null"
    fi

    # return string result
    if [[ ${image_name} = "null" ]]; then
        echo "null" # to be inserted as-is in the JSON output
    else
        echo "\"${1}/${image_name}\"" # if no image file found, but URL is valid, lets point to the directory
    fi
}

# Given an URL, generate the corresponding JSON chunk
function generate_json {
    url=$1
    SDK_URL=$1$2
    IMG_URL=$1$3

    arch=""

    htmlPage=$(${CURL} "${SDK_URL}")
    files=$(echo ${htmlPage} | egrep -o 'href="[^"]*crosssdk[^"]*\.sh"' | cut -d '"' -f 2)
    if [ "$?" != "0" ] || [ "${files}" = "" ]; then
        dbg_warning " IGNORED ${url}: no valid files found"
        return
    fi

    # assume that sdk name follow this format :
    #  _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh
    # for example:
    #  poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh
    if [ $(count files) -gt 1 ]
    then
        dbg_warning " IGNORE ${url}: multiple shell script containing 'crosssdk' in their names"
        return
    fi
    sdkFile=${files}
    dbg_info "Processing SDK: ${sdkFile} (from ${url})"
    profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/')
    version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/')
    arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/')

    endUrl=${url#$SDK_AGL_BASEURL}
    if [ "${url::14}" = "http://iot.bzh" ]; then
        endUrl=${url#$SDK_IOTBZH_BASEURL}
        name=$(echo "IoTbzh-$(echo ${endUrl} | sed s:/:-:g)-${version}")
        board=""
    elif [ "${endUrl::4}" = "http" ]; then
        name="${profile}_${arch}_${version}"
        board=""
    else
        name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f1,2,3,4)" | sed s:/:-:g)
        board=$(echo ${endUrl} | cut -d'/' -f4)
        board=${board%-nogfx}
    fi

    # Distringuish qemux86-64 and corei7-64
    if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then
        arch="qemux86-64"
    fi

    [ "${profile}" = "" ] && { dbg_error "profile is not set" ; return ; }
    [ "${version}" = "" ] && { dbg_error "version is not set" ; return ; }
    [ "${arch}" = "" ] && { dbg_error "arch is not set" ; return ; }
    [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; }

    sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)"
    sdkSize="$(echo "${htmlPage}" |egrep -o  "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)"
    md5sum="$(wget -q -O - ${SDK_URL}/${sdkFile/.sh/.md5} |cut -d' ' -f1)"

    image_url=$(search_image_url ${IMG_URL}${board} ${arch})


    read -r -d '' res <<- EndOfMessage
{
    "name":         "${name}",
    "description":  "AGL SDK ${arch} (version ${version})",
    "profile":      "${profile}",
    "version":      "${version}",
    "arch":         "${arch}",
    "path":         "",
    "url":          "${SDK_URL}${sdkFile}",
    "image_url":    ${image_url},
    "status":       "Not Installed",
    "date":         "${sdkDate}",
    "size":         "${sdkSize}",
    "md5sum":       "${md5sum}",
    "setupFile":    ""
},
EndOfMessage

    sdksList="${sdksList}${res}"
}

DEBUG=0
if [[ ${1:0:2} = "-v" ]] ; then
    verbose=$(echo -s ${1} | sed -E "s/-([v]+).*/\1/")
    DEBUG=$(echo ${verbose} | wc -c)
    echo "Set verbosity level to : ${DEBUG}"
    shift
fi

# Compute list of available/installable SDKs
sdksList=" "
for base_url in $DOWNLOADABLE_AGL_URLS
do
    dbg_debug "Look into ${base_url}"
    urls="$(expand_url $base_url $SDK_AGL_BASEURL)"
    for url in $urls
    do
        dbg_debug "URL to check: ${url}"
        generate_json $url $SDK_AGL_URL_SUFFIX $IMG_AGL_URL_SUFFIX
    done
done

for base_url in $DOWNLOADABLE_IOTBZH_URLS
do
    dbg_debug "Look into ${base_url}"
    urls="$(expand_url $base_url $SDK_IOTBZH_BASEURL)"
    for url in $urls
    do
        dbg_debug "URL to check: ${url}"
        generate_json $url $SDK_IOTBZH_URL_SUFFIX $IMG_IOTBZH_URL_SUFFIX
    done
done


OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%M").json

echo "[" > ${OUT_FILE}
echo "${sdksList::-1}" >> ${OUT_FILE}
echo "]" >> ${OUT_FILE}

echo "SDKs list successfully saved in ${OUT_FILE}"