blob: b7c393d219974d8444b13af9a35ec53e2c4ebbc7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
# (c) 2017 Kevin Hilman <khilman@baylibre.com>
# License GPLv2
#
# Submit LAVA YAML job file (default testjob.yaml) to first available LAVA lab
# with matching device-type
#
#JOB_FILE=${1:-testjob.yaml}
JOB_FILE=testjob.yaml
if [ ! -e $JOB_FILE ]; then
JOB_FILE=$1
fi
if [ ! -e $JOB_FILE ]; then
echo "ERROR: LAVA job file $JOB_FILE not present."
exit 123
fi
JOB_BASE=$(basename $JOB_FILE .yaml)
# Need to hack the LAVA device-type name in the job file
JOB_FILE_NEW="${JOB_BASE}_${LAVA_LAB}.yaml"
cat $JOB_FILE | sed "s/device_type: $releng_device$/device_type: $lava_device/" > $JOB_FILE_NEW
#
# LAVA job submit, get job ID from lava-tool output
#
JOB_STATUS="${JOB_BASE}_${LAVA_LAB}.status"
if [ -z "$lab" ];then
echo "WARNING: lab is empty fallback to agl lab"
lab=agl
fi
job_id=$(lavacli -i $lab jobs submit $JOB_FILE_NEW)
if [ $? -ne 0 ]; then
echo "ERROR: job submission error"
exit 123
fi
# Printing the job URL in the log
echo "THe job id is: ${job_id}"
JOB_URL="${url}scheduler/job/${job_id}"
echo ""
echo ""
echo "Submitted as job: $JOB_URL"
echo ""
echo "####################################################"
echo ""
cat << EOF > .msg
Submitted as job: $JOB_URL
EOF
# escape
sed -i -e "s/ /\\\ /g" .msg
#### echo back to gerrit
#set -x
eval ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} -n NONE --message='"$(cat .msg)"' || true
#set +x
# use lavacli wait job instead of the polling.
lavacli -i $lab jobs wait $job_id --timeout 7200 || true
lavacli -i $lab jobs show $job_id --yaml > $JOB_STATUS
state=$(grep ^state: $JOB_STATUS| cut -d' ' -f2)
if [ "$state" == "Finished" ]; then
status=$(grep ^health: $JOB_STATUS| cut -d' ' -f2)
echo "LAVA job $job_id completed with status: $status"
fi
echo "####"
echo "#### Start: Output from LAVA job $job_id ####"
echo "####"
JOB_OUTPUT="${JOB_BASE}_output.yaml"
JOB_LOG="${JOB_BASE}_output.log"
curl -s -o $JOB_OUTPUT $lava_url/scheduler/job/$job_id/log_file/plain
cat $JOB_OUTPUT | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' | tee $JOB_LOG
cat $JOB_OUTPUT | grep '"error_msg":' | sed -e 's#.*"error_msg": "##g' -e 's#", ".*##g' | tee -a $JOB_LOG
echo "####"
echo "#### End: Output from LAVA job $job_id ####"
echo "####"
handle_skiplist() {
if [ -z "${GERRIT_HOST}" ]; then
export GERRIT_HOST="gerrit.automotivelinux.org"
fi
SKIPLIST=skiplist
# detect job type (apps vs non-apps) and get skiplist
if [ ! -z "$(echo $JOB_NAME | grep 'ci-platform')" ];then
git clone https://${GERRIT_HOST}/gerrit/src/qa-testdefinitions.git || true
if [ ! -e qa-testdefinitions ];then
echo "ERROR: fail to get qa-testdefinitions"
return 0
fi
SKIPDIR=qa-testdefinitions/skiplists/
SKIPFILE=common
CK_REPO_NAME="qa-testdefinitions"
else
git clone https://${GERRIT_HOST}/gerrit/${GERRIT_PROJECT}.git || true
# GERRIT_PROJECT could have a / in it
CK_REPO_NAME="$(basename ${GERRIT_PROJECT})"
if [ ! -e $CK_REPO_NAME ];then
echo "ERROR: fail to get $GERRIT_PROJECT"
return 0
fi
SKIPDIR=${CK_REPO_NAME}/
SKIPFILE=.aglci
fi
for skipfile in $SKIPFILE $MACHINE ${GERRIT_PROJECT}
do
if [ -e $SKIPDIR/$skipfile ];then
echo "DEBUG: SKIPLIST: Found $SKIPDIR/$skipfile"
cat $SKIPDIR/$skipfile >> $SKIPLIST
fi
done
if [ ! -e "$SKIPLIST" ];then
echo "DEBUG: $SKIPLIST does not exist"
touch $SKIPLIST
return 0
fi
if [ ! -s "$SKIPLIST" ];then
echo "DEBUG: $SKIPLIST is empty"
return 0
fi
}
handle_result() {
# example of testline: * lava.lava-test-retry [fail]
while read testline
do
# Check if fail are "normal/accepted"
TESTNAME=$(echo "$testline" | cut -d' ' -f2)
# always skip lava.read-feedback (alone)
if [ x"lava.read-feedback" = x"$TESTNAME" ] ; then
continue
fi
if [ -z "$(grep $TESTNAME $SKIPLIST)" ];then
echo "DEBUG: $TESTNAME not in skiplist"
GOODJOB=0
else
echo "DEBUG: $TESTNAME in skiplist"
fi
done < job-result.fail
# clean skiplist
rm --one-file-system -rf "${CK_REPO_NAME}"
return 0
}
# Analyze jobs results
lavacli -i $lab results $job_id | tee job-result
set -x
# GOODJOB is equal to 0 if any test fail (and not present in a skiplist), 1 if all test are success
GOODJOB=1
grep '\[fail\]$' job-result > job-result.fail || true
if [ -s job-result.fail ];then
handle_skiplist
handle_result
fi
set -x
LINES_PERCENT=$(grep LINES_PERCENT= $JOB_LOG | sed -e 's#LINES_PERCENT=##g' | head -1)
BRANCHES_PERCENT=$(grep BRANCHES_PERCENT= $JOB_LOG | sed -e 's#LINES_PERCENT=##g' | head -1)
MSG_FAIL="CI-Image-Boot-Test fail on ${MACHINE}. Coverage: Lines ${LINES_PERCENT:-0}% , Branches ${BRANCHES_PERCENT:-0}%
${JOB_URL}"
MSG_SUCCESS="CI-Image-Boot-Test success on ${MACHINE}. Coverage: Lines ${LINES_PERCENT:-0}% , Branches ${BRANCHES_PERCENT:-0}%
${JOB_URL}"
MSG_BUILD_LOG="${BUILD_URL}consoleFull"
MSG_JOBFAIL="Failed jobs:
$(cat job-result.fail)"
set +x
if [ $GOODJOB -eq 0 ];then
#construct message
cat << EOF > .msg
$MSG_FAIL
Build log available at: $MSG_BUILD_LOG
$MSG_JOBFAIL
EOF
# escape
sed -i -e "s/ /\\\ /g" .msg
# send -1
eval ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} --verified +1 --label ci-image-boot-test=-1 -n NONE --message='"$(cat .msg)"' || true
else
#construct message
cat << EOF > .msg
$MSG_SUCCESS
Build log available at: $MSG_BUILD_LOG
EOF
# escape
sed -i -e "s/ /\\\ /g" .msg
# send +1
eval ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} --verified +1 --label ci-image-boot-test=+1 -n NONE --message='"$(cat .msg)"' || true
fi
set +x
# after one successful submit, we're done
if [ x"$status" = x"Complete" ]; then
exit 0
fi
#exit 123
# for now do not fail
exit 0
|
