Commit 4b441fc4 authored by Raphael Defosseux's avatar Raphael Defosseux

CI: better handling of test failures

  -- IperfNoS1 when eNB crashed
  -- LTE-M module failed attachment
  -- LTE-M module pinging
Signed-off-by: Raphael Defosseux's avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent ac711d9a
......@@ -521,6 +521,35 @@ class SSHConnection():
# here add a check if git clone or git fetch went smoothly
self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
self.command('git config user.name "OAI Jenkins"', '\$', 5)
self.command('ls *.txt', '\$', 5)
result = re.search('LAST_BUILD_INFO', str(self.ssh.before))
if result is not None:
mismatch = False
self.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
result = re.search(self.ranCommitID, str(self.ssh.before))
if result is None:
mismatch = True
self.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
if (self.ranAllowMerge):
result = re.search('YES', str(self.ssh.before))
if result is None:
mismatch = True
self.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
if self.ranTargetBranch == '':
result = re.search('develop', str(self.ssh.before))
else:
result = re.search(self.ranTargetBranch, str(self.ssh.before))
if result is None:
mismatch = True
else:
result = re.search('NO', str(self.ssh.before))
if result is None:
mismatch = True
if not mismatch:
self.close()
self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
return
self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
# if the commit ID is provided use it to point to it
if self.ranCommitID != '':
......@@ -549,10 +578,22 @@ class SSHConnection():
self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5)
self.close()
if buildStatus:
# Generating a BUILD INFO file
self.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
self.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
if (self.ranAllowMerge):
self.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
if self.ranTargetBranch == '':
self.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
else:
self.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
else:
self.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
self.close()
self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
else:
self.close()
logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE')
self.CreateHtmlTabFooter(False)
......@@ -1081,7 +1122,7 @@ class SSHConnection():
html_queue = SimpleQueue()
self.checkDevTTYisUnlocked()
if attach_status:
html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Completed in ' + str(attach_cnt+4) + ' seconds'
html_cell = '<pre style="background-color:white">CAT-M module Attachment Completed in ' + str(attach_cnt+4) + ' seconds'
if (nRSRQ is not None) and (nRSRP is not None):
html_cell += '\n RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB'
html_cell += '\n RSRP = ' + str(-140+nRSRP) + ' dBm</pre>'
......@@ -1090,9 +1131,11 @@ class SSHConnection():
html_queue.put(html_cell)
self.CreateHtmlTestRowQueue('N/A', 'OK', 1, html_queue)
else:
html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Failed</pre>'
logging.error('\u001B[1m CAT-M module Attachment Failed\u001B[0m')
html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>'
html_queue.put(html_cell)
self.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)
self.AutoTerminateUEandeNB()
def PingCatM(self):
if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
......@@ -1103,7 +1146,7 @@ class SSHConnection():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.prematureExit = True
self.AutoTerminateUEandeNB()
return
try:
statusQueue = SimpleQueue()
......@@ -1122,6 +1165,8 @@ class SSHConnection():
if result is not None:
moduleIPAddr = result.group('ipaddr')
else:
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
return
ping_time = re.findall("-c (\d+)",str(self.ping_args))
device_id = 'catm'
......@@ -2353,15 +2398,16 @@ class SSHConnection():
os.kill(os.getppid(),signal.SIGUSR1)
def IperfNoS1(self):
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
Usage()
sys.exit('Insufficient Parameter')
check_eNB = True
check_OAI_UE = True
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
if (pStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
Usage()
sys.exit('Insufficient Parameter')
return
server_on_enb = re.search('-R', str(self.iperf_args))
if server_on_enb is not None:
iServerIPAddr = self.eNBIPAddress
......@@ -4399,7 +4445,11 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
while cnt < SSH.repeatCounts[0] and SSH.prematureExit:
SSH.prematureExit = False
for test_case_id in todo_tests:
if SSH.prematureExit:
break
for test in all_tests:
if SSH.prematureExit:
break
id = test.get('id')
if test_case_id != id:
continue
......@@ -4480,10 +4530,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
SSH.Perform_X2_Handover()
else:
sys.exit('Invalid action')
if SSH.prematureExit:
break
if SSH.prematureExit:
break
cnt += 1
if cnt == SSH.repeatCounts[0] and SSH.prematureExit:
logging.debug('Testsuite failed ' + str(cnt) + ' time(s)')
......
......@@ -25,10 +25,23 @@
<htmlTabName>EPC-Start</htmlTabName>
<htmlTabIcon>log-in</htmlTabIcon>
<TestCaseRequestedList>
010101 090101
050101 060101 070101
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="010101">
<class>Build_eNB</class>
<desc>Build eNB (USRP)</desc>
<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
</testCase>
<testCase id="090101">
<class>Build_OAI_UE</class>
<desc>Build OAI UE</desc>
<Build_OAI_UE_args>-w USRP --UE</Build_OAI_UE_args>
</testCase>
<testCase id="050101">
<class>Initialize_HSS</class>
<desc>Initialize HSS</desc>
......
......@@ -24,7 +24,7 @@
<htmlTabRef>test-05-tm1-nos1-tunnel</htmlTabRef>
<htmlTabName>Test-05MHz-TM1-noS1-tunnel</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>2</repeatCount>
<repeatCount>4</repeatCount>
<TestCaseRequestedList>
030201 090109
030101 000001 090101 000002 040501 040502 000001 040601 040602 040641 040642 000001 090109 030201
......
......@@ -25,10 +25,16 @@
<htmlTabName>EPC-Closure</htmlTabName>
<htmlTabIcon>log-out</htmlTabIcon>
<TestCaseRequestedList>
040202
050201 060201 070201
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="040202">
<class>Terminate_CatM_module</class>
<desc>Terminate CAT-M Module</desc>
</testCase>
<testCase id="050201">
<class>Terminate_HSS</class>
<desc>Terminate HSS</desc>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment