Commit 0a4cfc21 authored by Raphael Defosseux's avatar Raphael Defosseux
Browse files

CI: scenarios that are known as unstable won't be seen as failures


Signed-off-by: Raphael Defosseux's avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent fb957484
......@@ -72,6 +72,9 @@ class OaiCiTest():
self.ADBCentralized = True
self.testCase_id = ''
self.testXMLfiles = []
self.testUnstable = False
self.testMinStableId = '999999'
self.testStabilityPointReached = False
self.desc = ''
self.ping_args = ''
self.ping_packetloss_threshold = ''
......@@ -211,7 +214,7 @@ class OaiCiTest():
logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', CONST.ALL_PROCESSES_OK, 'OAI UE')
HTML.CreateHtmlTabFooter(False)
sys.exit(1)
self.ConditionalExit()
def CheckFlexranCtrlInstallation(self,RAN,EPC):
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '':
......@@ -335,7 +338,7 @@ class OaiCiTest():
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', pStatus)
HTML.CreateHtmlTabFooter(False)
sys.exit(1)
self.ConditionalExit()
UE_prefix = ''
else:
UE_prefix = 'NR '
......@@ -719,7 +722,7 @@ class OaiCiTest():
logging.debug('Using the OAI EPC HSS: not implemented yet')
HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
HTML.CreateHtmlTabFooter(False)
sys.exit(1)
self.ConditionalExit()
else:
SSH.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5)
result = re.search('Allocated ipv4 addr: (?P<ipaddr>[0-9\.]+) from Pool', SSH.getBefore())
......@@ -1046,7 +1049,7 @@ class OaiCiTest():
if (pStatus < 0):
HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
HTML.CreateHtmlTabFooter(False)
sys.exit(1)
self.ConditionalExit()
multi_jobs = []
for device_id in self.UEDevices:
p = Process(target = self.RebootUE_common, args = (device_id,))
......@@ -1248,7 +1251,7 @@ class OaiCiTest():
if (pStatus < 0):
HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
HTML.CreateHtmlTabFooter(False)
sys.exit(1)
self.ConditionalExit()
multi_jobs = []
lock = Lock()
status_queue = SimpleQueue()
......@@ -3169,6 +3172,12 @@ class OaiCiTest():
HTML.CpuMHz[idx]=CpuMHz
SSH.close()
def ConditionalExit(self):
if self.testUnstable:
if self.testStabilityPointReached or self.testMinStableId == '999999':
sys.exit(0)
sys.exit(1)
def ShowTestID(self):
logging.debug('\u001B[1m----------------------------------------\u001B[0m')
logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m')
......
......@@ -64,6 +64,10 @@ class HTMLManagement():
self.htmlTabIcons = []
self.testXMLfiles = []
self.testUnstable = False
self.testMinStableId = '999999'
self.testStabilityPointReached = False
self.htmleNBFailureMsg = ''
self.htmlUEFailureMsg = ''
......@@ -230,13 +234,20 @@ class HTMLManagement():
def CreateHtmlTabFooter(self, passStatus):
if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
testOkEvenIfUnstable = False
if self.testUnstable and not passStatus:
if self.testStabilityPointReached or self.testMinStableId == '999999':
testOkEvenIfUnstable = True
self.htmlFile = open('test_results.html', 'a')
self.htmlFile.write(' <tr>\n')
self.htmlFile.write(' <th bgcolor = "#33CCFF" colspan=3>Final Tab Status</th>\n')
if passStatus:
self.htmlFile.write(' <th bgcolor = "green" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">PASS <span class="glyphicon glyphicon-ok"></span> </font></th>\n')
else:
self.htmlFile.write(' <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
if testOkEvenIfUnstable:
self.htmlFile.write(' <th bgcolor = "orange" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">KNOWN UNSTABLE SCENARIO <span class="glyphicon glyphicon-exclamation-sign"></span> </font></th>\n')
else:
self.htmlFile.write(' <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL <span class="glyphicon glyphicon-remove"></span> </font></th>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' </table>\n')
self.htmlFile.write(' </div>\n')
......@@ -246,7 +257,10 @@ class HTMLManagement():
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
subprocess.run(cmd, shell=True)
else:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
if testOkEvenIfUnstable:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-exclamation-sign\"><\/span>/' test_results.html"
else:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
subprocess.run(cmd, shell=True)
self.htmlFooterCreated = False
......
......@@ -494,6 +494,9 @@ elif re.match('^FinalizeHtml$', mode, re.IGNORECASE):
CiTestObj.RetrieveSystemVersion('UE',HTML,RAN)
HTML.CreateHtmlFooter(CiTestObj.finalStatus)
elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE):
logging.debug('\u001B[1m----------------------------------------\u001B[0m')
logging.debug('\u001B[1m Starting Scenario \u001B[0m')
logging.debug('\u001B[1m----------------------------------------\u001B[0m')
if re.match('^TesteNB$', mode, re.IGNORECASE):
if RAN.eNBIPAddress == '' or RAN.ranRepository == '' or RAN.ranBranch == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '' or EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '' or CiTestObj.ADBIPAddress == '' or CiTestObj.ADBUserName == '' or CiTestObj.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
......@@ -529,7 +532,14 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
HTML.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
HTML.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0'))
repeatCount = xmlRoot.findtext('repeatCount',default='1')
testStability = xmlRoot.findtext('TestUnstable',default='False')
CiTestObj.repeatCounts.append(int(repeatCount))
if testStability == 'True':
CiTestObj.testUnstable = True
HTML.testUnstable = True
CiTestObj.testMinStableId = xmlRoot.findtext('TestMinId',default='999999')
HTML.testMinStableId = CiTestObj.testMinStableId
logging.debug('Test is tagged as Unstable -- starting from TestID ' + str(CiTestObj.testMinStableId))
all_tests=xmlRoot.findall('testCase')
exclusion_tests=exclusion_tests.split()
......@@ -686,11 +696,19 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
HTML=ldpc.Run_PhySim(HTML,CONST,id)
else:
sys.exit('Invalid class (action) from xml')
if not RAN.prematureExit:
if CiTestObj.testCase_id == CiTestObj.testMinStableId:
logging.debug('Scenario has reached minimal stability point')
CiTestObj.testStabilityPointReached = True
HTML.testStabilityPointReached = True
CiTestObj.FailReportCnt += 1
if CiTestObj.FailReportCnt == CiTestObj.repeatCounts[0] and RAN.prematureExit:
logging.debug('Testsuite failed ' + str(CiTestObj.FailReportCnt) + ' time(s)')
HTML.CreateHtmlTabFooter(False)
sys.exit('Failed Scenario')
if CiTestObj.testUnstable and (CiTestObj.testStabilityPointReached or CiTestObj.testMinStableId == '999999'):
logging.debug('Scenario has reached minimal stability point -- Not a Failure')
else:
sys.exit('Failed Scenario')
else:
logging.info('Testsuite passed after ' + str(CiTestObj.FailReportCnt) + ' time(s)')
HTML.CreateHtmlTabFooter(True)
......
......@@ -25,6 +25,7 @@
<htmlTabName>Test-LTE-M-10MHz-TM1</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>1</repeatCount>
<TestUnstable>True</TestUnstable>
<TestCaseRequestedList>
030201
040102
......
......@@ -26,6 +26,8 @@
<htmlTabName>Test-05MHz-TM2</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>2</repeatCount>
<TestUnstable>True</TestUnstable>
<TestMinId>040608</TestMinId>
<TestCaseRequestedList>
030201
040101
......
......@@ -24,6 +24,8 @@
<htmlTabRef>test-10-tm1</htmlTabRef>
<htmlTabName>Test-10MHz-TM1</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<TestUnstable>True</TestUnstable>
<TestMinId>040615</TestMinId>
<TestCaseRequestedList>
030201
040101
......
......@@ -25,6 +25,8 @@
<htmlTabName>Test-20MHz-TM1</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>4</repeatCount>
<TestUnstable>True</TestUnstable>
<TestMinId>040623</TestMinId>
<TestCaseRequestedList>
030201
040101
......
......@@ -24,6 +24,8 @@
<htmlTabRef>test-20-tm1-defSched</htmlTabRef>
<htmlTabName>Test-20MHz-TM1-default-scheduler</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<TestUnstable>True</TestUnstable>
<TestMinId>040522</TestMinId>
<TestCaseRequestedList>
030201
040101
......
......@@ -25,6 +25,8 @@
<htmlTabName>Test-10MHz</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>2</repeatCount>
<TestUnstable>True</TestUnstable>
<TestMinId>040612</TestMinId>
<TestCaseRequestedList>
030201
040101
......
......@@ -26,6 +26,7 @@
<htmlTabName>Test-X2-HO-FDD-05MHz-TM1</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<repeatCount>2</repeatCount>
<TestUnstable>True</TestUnstable>
<TestCaseRequestedList>
030201 030202
040101
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment