aboutsummaryrefslogtreecommitdiffstats
path: root/jjb/common/include-agl-lava-jobs-submit.sh
blob: 824821303f9fd2043beab719439bd7068612de38 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
# (c) 2017 Kevin Hilman <khilman@baylibre.com>
# License GPLv2
#
# Submit LAVA YAML job file (default testjob.yaml) to first available LAVA lab
# with matching device-type
#
#JOB_FILE=${1:-testjob.yaml}
JOB_FILE=testjob.yaml

if [ ! -e $JOB_FILE ]; then
    JOB_FILE=$1
fi

if [ ! -e $JOB_FILE ]; then
    echo "ERROR: LAVA job file $JOB_FILE not present."
    exit 1
fi
JOB_BASE=$(basename $JOB_FILE .yaml)

# Need to hack the LAVA device-type name in the job file
JOB_FILE_NEW="${JOB_BASE}_${LAVA_LAB}.yaml"
cat $JOB_FILE | sed "s/device_type: $releng_device$/device_type: $lava_device/" > $JOB_FILE_NEW

#
# LAVA job submit, get job ID from lava-tool output
#
JOB_STATUS="${JOB_BASE}_${LAVA_LAB}.status"

job_id=$(lavacli -i $lab jobs submit $JOB_FILE_NEW)
if [ $? -ne 0 ]; then
	echo "ERROR: job submission error"
	exit 1
fi
# Printing the job URL in the log
echo "THe job id is: ${job_id}"
JOB_URL="${url}scheduler/job/${job_id}"

echo ""
echo ""
echo "Submitted as job: $JOB_URL"
echo ""
echo "####################################################"
echo ""

# use lavacli wait job instead of the polling.
lavacli -i agl jobs wait $job_id --timeout 7200
lavacli -i agl jobs show $job_id --yaml > $JOB_STATUS
state=$(grep ^state: $JOB_STATUS| cut -d' ' -f2)
if [ "$state" == "Finished" ]; then
        status=$(grep ^health: $JOB_STATUS| cut -d' ' -f2)
        echo "LAVA job $job_id completed with status: $status"
fi

echo "####"
echo "#### Start: Output from LAVA job $job_id ####"
echo "####"

JOB_OUTPUT="${JOB_BASE}_output.yaml"
JOB_LOG="${JOB_BASE}_output.log"
curl -s -o $JOB_OUTPUT $lava_url/scheduler/job/$job_id/log_file/plain
cat $JOB_OUTPUT | grep '"target",' | sed -e 's#- {"dt".*"lvl".*"msg":."##g' -e 's#"}$##g' | tee $JOB_LOG
cat $JOB_OUTPUT | grep '"error_msg":' | sed -e 's#.*"error_msg": "##g' -e 's#", ".*##g' | tee -a $JOB_LOG

echo "####"
echo "#### End: Output from LAVA job $job_id ####"
echo "####"

handle_skiplist() {
	SKIPLIST=skiplist
	# detect job type (apps vs non-apps) and get skiplist
	echo $JOB_NAME | grep -q 'ci-platform'
	if [ $? -eq 0 ];then
		git clone https://${GERRIT_HOST}/gerrit/src/qa-testdefinitions.git
		if [ $? -ne ];then
			echo "ERROR: fail to get qa-testdefinitions"
			return 1
		fi
		SKIPDIR=qa-testdefinitions/skiplists/
		SKIPFILE=common
		CK_REPO_NAME="qa-testdefinitions"
	else
		git clone https://${GERRIT_HOST}/gerrit/${GERRIT_PROJECT}.git
		if [ $? -ne ];then
			echo "ERROR: fail to get $GERRIT_PROJECT"
			return 1
		fi
		SKIPDIR=${GERRIT_PROJECT}/
		SKIPFILE=.aglci
		CK_REPO_NAME="${GERRIT_PROJECT}"
	fi
	for skipfile in $SKIPFILE $MACHINE ${GERRIT_PROJECT}
	do
		if [ -e $SKIPDIR/$skipfile ];then
			echo "DEBUG: SKIPLIST: Found $SKIPDIR/$skipfile"
			cat $SKIPDIR/$skipfile >> $SKIPLIST
		fi
	done

	if [ -e "$SKIPLIST" ];then
		echo "DEBUG: $SKIPLIST does not exists"
		return 0
	fi

	if [ ! -s "$SKIPLIST" ];then
		echo "DEBUG: $SKIPLIST is empty"
		return 0
	fi

	# example of testline: * lava.lava-test-retry [fail]
	while read testline
	do
		# Check if fail are "normal/accepted"
		TESTNAME=$(echo $testline | cut -d' ' -f2)
		grep -q $TESTNAME $SKIPLIST
		if [ $? -ne 0 ];then
			echo "DEBUG: $TESNAME not in skiplist"
			GOODJOB=0
		else
			echo "DEBUG: $TESNAME in skiplist"
		fi
	done < job-result.fail
	# clean skiplist
	rm --one-file-system -rf "${CK_REPO_NAME}"
	return 0
}

# Analyze jobs results
lavacli results $job_id | tee job-result

# GOODJOB is equal to 0 if any test fail (and not present in a skiplist), 1 if all test are success
GOODJOB=1
grep '\[fail\]$' job-result > job-result.fail || true

if [ -s job-result.fail ];then
	handle_skiplist
fi

if [ $GOODJOB -eq 0 ];then
	# send -1
	ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} --verified +1 --label ci-image-boot-test=-1 -n NONE --message='CI-Image-Boot-Test\ fail' || true
else
	# send +1
	ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p 29418 agl-jobbuilder@gerrit.automotivelinux.org gerrit review ${GERRIT_CHANGE_NUMBER},${GERRIT_PATCHSET_NUMBER} --verified +1 --label ci-image-boot-test=+1 -n NONE --message='CI-Image-Boot-Test\ success' || true
fi

# after one successful submit, we're done
if [ x"$status" = x"Complete" ]; then
    exit 0
fi

#exit 1
# for now do not fail
exit 0