Skip to content
This repository was archived by the owner on Aug 5, 2022. It is now read-only.

Commit aa51d87

Browse files
mythimythi
authored
Merge pull request #102 from okartau/try-parallel-tests
CI Jenkinsfile,tester-exec: streamline, fix issue of lost test results
2 parents adb4c8e + 15346e8 commit aa51d87

File tree

2 files changed

+73
-71
lines changed

2 files changed

+73
-71
lines changed

‎Jenkinsfile‎

Lines changed: 64 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -122,63 +122,47 @@ try {
122122
def test_runs = [:]
123123
for(int i = 0; i < test_devices.size(); i++) {
124124
def test_device = test_devices[i]
125-
// only if built for machine that this tester wants
126-
if ( target_machine == mapping["${test_device}"] ) {
127-
// testinfo_data may contain multiple lines stating different images
128-
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
129-
for (int m = 0; m < separated_testinfo.length; m++) {
130-
def one_image_testinfo = separated_testinfo[m]
131-
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
132-
test_runs["test_${m}_${test_device}"] = {
133-
node('refkit-tester') {
134-
deleteDir() // clean workspace
135-
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
136-
writeFile file: 'tester-exec.sh', text: tester_script
137-
// append newline so that tester-exec.sh can parse it using "read"
138-
one_image_testinfo += "\n"
139-
// write testinfo file on this tester for this image, one line per tester
140-
writeFile file: "testinfo.csv", text: one_image_testinfo
141-
String[] one_testinfo_elems = one_image_testinfo.split(",")
142-
def img = one_testinfo_elems[0]
143-
try {
144-
withEnv(["CI_BUILD_ID=${ci_build_id}",
145-
"MACHINE=${mapping["${test_device}"]}",
146-
"TEST_DEVICE=${test_device}" ]) {
147-
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
148-
}
149-
} catch (Exception e) {
150-
throw e
151-
} finally {
152-
// read tests summary prepared by tester-exec.sh
153-
// Here one tester adds it's summary piece to the global buffer.
154-
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
155-
archiveArtifacts allowEmptyArchive: true,
156-
artifacts: '**/*.log, **/*.xml, **/aft-results*.tar.bz2'
125+
// only if built for machine that this tester wants
126+
if ( target_machine == mapping["${test_device}"] ) {
127+
// testinfo_data may contain multiple lines stating different images
128+
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
129+
for (int m = 0; m < separated_testinfo.length; m++) {
130+
def one_image_testinfo = separated_testinfo[m]
131+
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
132+
test_runs["test_${m}_${test_device}"] = {
133+
node('refkit-tester') {
134+
deleteDir() // clean workspace
135+
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
136+
writeFile file: 'tester-exec.sh', text: tester_script
137+
// append newline so that tester-exec.sh can parse it using "read"
138+
one_image_testinfo += "\n"
139+
// create testinfo.csv on this tester describing one image
140+
writeFile file: "testinfo.csv", text: one_image_testinfo
141+
def img = one_image_testinfo.split(",")[0]
142+
try {
143+
withEnv(["CI_BUILD_ID=${ci_build_id}",
144+
"MACHINE=${mapping["${test_device}"]}",
145+
"TEST_DEVICE=${test_device}" ]) {
146+
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
157147
}
158-
step([$class: 'XUnitPublisher',
159-
testTimeMargin: '3000',
160-
thresholdMode: 1,
161-
thresholds: [
162-
[$class: 'FailedThreshold',
163-
failureNewThreshold: '0',
164-
failureThreshold: '0',
165-
unstableNewThreshold: '99999',
166-
unstableThreshold: '99999'],
167-
[$class: 'SkippedThreshold',
168-
failureNewThreshold: '99999',
169-
failureThreshold: '99999',
170-
unstableNewThreshold: '99999',
171-
unstableThreshold: '99999']],
172-
tools: [[$class: 'JUnitType',
173-
deleteOutputFiles: true,
174-
failIfNotNew: true,
175-
pattern: 'TEST-*.xml',
176-
skipNoTestFiles: false,
177-
stopProcessingIfError: true]]])
178-
} // node
179-
} // test_runs =
180-
} // for m
181-
} // if target_machine == mapping
148+
} catch (Exception e) {
149+
throw e
150+
} finally {
151+
// read tests summary prepared by tester-exec.sh
152+
// Here one tester adds it's summary piece to the global buffer.
153+
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
154+
archiveArtifacts allowEmptyArchive: true,
155+
artifacts: '*.log, *.xml'
156+
}
157+
// without locking we may lose tester result set(s)
158+
// if testers run xunit step in nearly same time
159+
lock(resource: "step-xunit") {
160+
step_xunit()
161+
}
162+
} // node
163+
} // test_runs =
164+
} // for
165+
} // if target_machine == mapping
182166
} // for i
183167
stage('Parallel test run') {
184168
set_gh_status_pending(is_pr, 'Testing')
@@ -300,3 +284,27 @@ def set_gh_status_pending(is_pr, _msg) {
300284
setGitHubPullRequestStatus state: 'PENDING', context: "${env.JOB_NAME}", message: "${_msg}"
301285
}
302286
}
287+
288+
def step_xunit() {
289+
step([$class: 'XUnitPublisher',
290+
testTimeMargin: '3000',
291+
thresholdMode: 1,
292+
thresholds: [
293+
[$class: 'FailedThreshold',
294+
failureNewThreshold: '0',
295+
failureThreshold: '0',
296+
unstableNewThreshold: '99999',
297+
unstableThreshold: '99999'],
298+
[$class: 'SkippedThreshold',
299+
failureNewThreshold: '99999',
300+
failureThreshold: '99999',
301+
unstableNewThreshold: '99999',
302+
unstableThreshold: '99999']],
303+
tools: [
304+
[$class: 'JUnitType',
305+
deleteOutputFiles: true,
306+
failIfNotNew: true,
307+
pattern: 'TEST-*.xml',
308+
skipNoTestFiles: false,
309+
stopProcessingIfError: true]]])
310+
}

‎docker/tester-exec.sh‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ testimg() {
2121
_IMG_NAME=$1
2222
TEST_SUITE_FILE=$2
2323
TEST_CASES_FILE=$3
24-
_IMG_NAME_MACHINE=${_IMG_NAME}-${MACHINE}
2524

2625
# Get test suite
2726
wget ${_WGET_OPTS} ${TEST_SUITE_FOLDER_URL}/${_IMG_NAME}/${TEST_SUITE_FILE}
@@ -34,21 +33,16 @@ testimg() {
3433
cp $HOME/.config.ini.wlan ${_WLANCONF}
3534
chmod 644 ${_WLANCONF}
3635

37-
FN_BASE=${_IMG_NAME_MACHINE}-${CI_BUILD_ID}
38-
FILENAME=${FN_BASE}.wic
39-
FILENAME_BMAP=${FILENAME}.bmap
40-
FILENAME_XZ=${FILENAME}.xz
41-
FILENAME_ZIP=${FILENAME}.zip
42-
36+
FILENAME=${_IMG_NAME}-${MACHINE}-${CI_BUILD_ID}.wic
4337
set +e
44-
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_BMAP}
45-
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_XZ} -O - | unxz - > ${FILENAME}
38+
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap
39+
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME}
4640
if [ ! -s ${FILENAME} ]; then
47-
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_ZIP}
48-
if [ -s ${FILENAME_ZIP} ]; then
49-
unzip ${FILENAME_ZIP}
41+
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip
42+
if [ -s ${FILENAME}.zip ]; then
43+
unzip ${FILENAME}.zip
5044
else
51-
echo "ERROR: No file ${FILENAME_XZ} or ${FILENAME_ZIP} found, can not continue."
45+
echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue."
5246
exit 1
5347
fi
5448
fi
@@ -76,6 +70,8 @@ testimg() {
7670
daft ${DEVICE} ${FILENAME} --record
7771
AFT_EXIT_CODE=$?
7872

73+
# delete symlinks, these point outside of local set and are useless
74+
find . -type l -print -delete
7975
# modify names inside TEST-*.xml files to contain device and img_name
8076
# as these get shown in same xUnit results table in Jenkins
8177
sed -e "s/name=\"oeqa/name=\"${DEVICE}.${_IMG_NAME}.oeqa/g" -i TEST-*.xml
@@ -107,8 +103,6 @@ testimg() {
107103
echo " Run rate:${run_rate}% Pass rate of total:${pass_rate_of_total}% Pass rate of exec:${pass_rate_of_exec}%" >> $sumfile
108104
fi
109105
echo "-------------------------------------------------------------------" >> $sumfile
110-
# combine artifacts into single file for easier download
111-
tar c --ignore-failed-read results* *.xml *.log | bzip2 -c9 > aft-results_${DEVICE}_${_IMG_NAME}_${TEST_SUITE_FILE}.tar.bz2
112106
set -e
113107

114108
return ${AFT_EXIT_CODE}

0 commit comments

Comments
 (0)