From 65002b7fb127a62fbeeec92489b73f4f811bc573 Mon Sep 17 00:00:00 2001
From: Raphael Defosseux <raphael.defosseux@eurecom.fr>
Date: Thu, 18 Jul 2019 13:51:21 +0200
Subject: [PATCH] CI: adding support to repeat scenario in case of failure.    
 nice when the tested feature is known to be unstable     CDRX testing is now
 separated from normal FDD 10MHz testing

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
---
 ci-scripts/main.py                            | 269 +++++++++---------
 .../enb_usrp210_band7_test_10mhz_tm1.xml      |  36 ---
 .../enb_usrp210_band7_test_10mhz_tm1_cdrx.xml |  90 ++++++
 .../xml_files/f1_usrp210_band7_test_05mhz.xml |   3 +-
 .../xml_files/f1_usrp210_band7_test_10mhz.xml |   3 +-
 .../xml_files/f1_usrp210_band7_test_20mhz.xml |   3 +-
 .../multi_rru_band38_test_10mhz_tm1.xml       |   1 +
 .../xml_files/ue_band20_test_10mhz_orange.xml |   1 +
 .../xml_files/ue_band20_test_10mhz_sfr.xml    |   1 +
 9 files changed, 236 insertions(+), 171 deletions(-)
 create mode 100644 ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1_cdrx.xml

diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index 22631a742f..fd8d8c9b6c 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -87,6 +87,7 @@ logging.basicConfig(
 #-----------------------------------------------------------
 class SSHConnection():
 	def __init__(self):
+		self.prematureExit = False
 		self.ranRepository = ''
 		self.ranBranch = ''
 		self.ranAllowMerge = False
@@ -146,6 +147,7 @@ class SSHConnection():
 		self.htmlTabRefs = []
 		self.htmlTabNames = []
 		self.htmlTabIcons = []
+		self.repeatCounts = []
 		self.finalStatus = False
 		self.OsVersion = ''
 		self.KernelVersion = ''
@@ -745,7 +747,6 @@ class SSHConnection():
 				doLoop = False
 				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
 				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
-				self.CreateHtmlTabFooter(False)
 				# In case of T tracer recording, we need to kill tshark on EPC side
 				result = re.search('T_stdout', str(self.Initialize_eNB_args))
 				if result is not None:
@@ -761,7 +762,8 @@ class SSHConnection():
 						copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
 						if (copyin_res == 0):
 							self.copyout(lIpAddr, lUserName, lPassWord, self.EPC_PcapFileName, lSourcePath + '/cmake_targets/.')
-				sys.exit(1)
+				self.prematureExit = True
+				return
 			else:
 				self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
 				if rruCheck:
@@ -938,8 +940,6 @@ class SSHConnection():
 			self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
 			logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 
 	def checkDevTTYisUnlocked(self):
 		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
@@ -1090,8 +1090,8 @@ class SSHConnection():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			self.prematureExit = True
+			return
 		try:
 			statusQueue = SimpleQueue()
 			lock = Lock()
@@ -1173,8 +1173,6 @@ class SSHConnection():
 			else:
 				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
 				self.AutoTerminateUEandeNB()
-				self.CreateHtmlTabFooter(False)
-				sys.exit(1)
 		except:
 			os.kill(os.getppid(),signal.SIGUSR1)
 
@@ -1235,8 +1233,7 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		multi_jobs = []
 		status_queue = SimpleQueue()
 		lock = Lock()
@@ -1254,9 +1251,8 @@ class SSHConnection():
 
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
-			self.CreateHtmlTabFooter(False)
 			self.AutoTerminateUEandeNB()
-			sys.exit(1)
+			return
 		else:
 			attach_status = True
 			html_queue = SimpleQueue()
@@ -1285,8 +1281,6 @@ class SSHConnection():
 			else:
 				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
 				self.AutoTerminateUEandeNB()
-				self.CreateHtmlTabFooter(False)
-				sys.exit(1)
 
 	def DetachUE_common(self, device_id):
 		try:
@@ -1307,8 +1301,7 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		multi_jobs = []
 		cnt = 0
 		for device_id in self.UEDevices:
@@ -1560,8 +1553,6 @@ class SSHConnection():
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow(htmlOptions, 'KO', ALL_PROCESSES_OK)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 		else:
 			check_status = True
 			html_queue = SimpleQueue()
@@ -1578,8 +1569,6 @@ class SSHConnection():
 			else:
 				self.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
 				self.AutoTerminateUEandeNB()
-				self.CreateHtmlTabFooter(False)
-				sys.exit(1)
 
 	def GetAllUEIPAddresses(self):
 		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
@@ -1725,8 +1714,7 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
 		if ping_from_eNB is not None:
 			if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '':
@@ -1829,14 +1817,12 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		ueIpStatus = self.GetAllUEIPAddresses()
 		if (ueIpStatus < 0):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		multi_jobs = []
 		i = 0
 		lock = Lock()
@@ -1854,8 +1840,6 @@ class SSHConnection():
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 		else:
 			ping_status = True
 			html_queue = SimpleQueue()
@@ -1873,8 +1857,6 @@ class SSHConnection():
 			else:
 				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
 				self.AutoTerminateUEandeNB()
-				self.CreateHtmlTabFooter(False)
-				sys.exit(1)
 
 	def Iperf_ComputeTime(self):
 		result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
@@ -2352,8 +2334,6 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
 			Usage()
 			sys.exit('Insufficient Parameter')
@@ -2443,8 +2423,6 @@ class SSHConnection():
 		else:
 			self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 
 	def Iperf(self):
 		result = re.search('noS1', str(self.Initialize_eNB_args))
@@ -2463,14 +2441,12 @@ class SSHConnection():
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		ueIpStatus = self.GetAllUEIPAddresses()
 		if (ueIpStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
+			return
 		multi_jobs = []
 		i = 0
 		ue_num = len(self.UEIPAddresses)
@@ -2489,8 +2465,6 @@ class SSHConnection():
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
 			self.AutoTerminateUEandeNB()
-			self.CreateHtmlTabFooter(False)
-			sys.exit(1)
 		else:
 			iperf_status = True
 			iperf_noperf = False
@@ -2513,8 +2487,6 @@ class SSHConnection():
 			else:
 				self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
 				self.AutoTerminateUEandeNB()
-				self.CreateHtmlTabFooter(False)
-				sys.exit(1)
 
 	def CheckProcessExist(self, check_eNB, check_OAI_UE):
 		multi_jobs = []
@@ -2834,10 +2806,17 @@ class SSHConnection():
 			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
 			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
 			self.htmleNBFailureMsg += rrcMsg + '\n'
-		if cdrxActivationMessageCount > 0:
-			rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			self.htmleNBFailureMsg += rrcMsg + '\n'
+		if self.eNBOptions[int(self.eNB_instance)] != '':
+			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
+			if res1 is not None:
+				if cdrxActivationMessageCount > 0:
+					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
+					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
+					self.htmleNBFailureMsg += rrcMsg + '\n'
+				else:
+					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
+					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
+					self.htmleNBFailureMsg += rrcMsg + '\n'
 		if rachCanceledProcedure > 0:
 			rachMsg = 'eNB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
 			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
@@ -2852,6 +2831,7 @@ class SSHConnection():
 					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
 					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
 					self.htmleNBFailureMsg += rruMsg + '\n'
+					self.prematureExit = True
 					return ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
 		if foundSegFault:
 			logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
@@ -3121,8 +3101,8 @@ class SSHConnection():
 				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
 				if (logStatus < 0):
 					self.CreateHtmlTestRow('N/A', 'KO', logStatus)
-					self.CreateHtmlTabFooter(False)
-					sys.exit(1)
+					self.preamtureExit = True
+					return
 				else:
 					self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
 			else:
@@ -3261,8 +3241,6 @@ class SSHConnection():
 				if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
 					self.Initialize_OAI_UE_args = ''
 					self.AutoTerminateUEandeNB()
-					self.CreateHtmlTabFooter(False)
-					sys.exit(1)
 			else:
 				logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
 				self.htmlUEFailureMsg = '<b>' + ueAction + ' Completed</b>\n' + self.htmlUEFailureMsg
@@ -3288,6 +3266,7 @@ class SSHConnection():
 			self.ShowTestID()
 			self.eNB_instance = '0'
 			self.TerminateeNB()
+		self.prematureExit = True
 
 	def IdleSleep(self):
 		time.sleep(self.idle_sleep_time)
@@ -3548,6 +3527,7 @@ class SSHConnection():
 				pillMsg = '    <li><a data-toggle="pill" href="#'
 				pillMsg += self.htmlTabRefs[count]
 				pillMsg += '">'
+				pillMsg += '__STATE_' + self.htmlTabNames[count] + '__'
 				pillMsg += self.htmlTabNames[count]
 				pillMsg += ' <span class="glyphicon glyphicon-'
 				pillMsg += self.htmlTabIcons[count]
@@ -3601,6 +3581,14 @@ class SSHConnection():
 			self.htmlFile.write('      </tr>\n')
 			self.htmlFile.write('  </table>\n')
 			self.htmlFile.write('  </div>\n')
+			self.htmlFile.close()
+			time.sleep(1)
+			if passStatus:
+				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
+				subprocess.run(cmd, shell=True)
+			else:
+				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
+				subprocess.run(cmd, shell=True)
 		self.htmlFooterCreated = False
 
 	def CreateHtmlFooter(self, passStatus):
@@ -4178,6 +4166,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='')
 	if (SSH.nbTestXMLfiles == 1):
 		SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
+		SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0'))
+		repeatCount = xmlRoot.findtext('repeatCount',default='1')
+		SSH.repeatCounts.append(int(repeatCount))
 	all_tests=xmlRoot.findall('testCase')
 
 	exclusion_tests=exclusion_tests.split()
@@ -4219,88 +4210,102 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 
 	SSH.CreateHtmlTabHeader()
 
-	for test_case_id in todo_tests:
-		for test in all_tests:
-			id = test.get('id')
-			if test_case_id != id:
-				continue
-			SSH.testCase_id = id
-			SSH.desc = test.findtext('desc')
-			action = test.findtext('class')
-			if (CheckClassValidity(action, id) == False):
-				continue
-			SSH.ShowTestID()
-			GetParametersFromXML(action)
-			if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE':
-				if (SSH.ADBIPAddress != 'none'):
-					terminate_ue_flag = False
-					SSH.GetAllUEDevices(terminate_ue_flag)
-			if action == 'Build_eNB':
-				SSH.BuildeNB()
-			elif action == 'WaitEndBuild_eNB':
-				SSH.WaitBuildeNBisFinished()
-			elif action == 'Initialize_eNB':
-				SSH.InitializeeNB()
-			elif action == 'Terminate_eNB':
-				SSH.TerminateeNB()
-			elif action == 'Initialize_UE':
-				SSH.InitializeUE()
-			elif action == 'Terminate_UE':
-				SSH.TerminateUE()
-			elif action == 'Attach_UE':
-				SSH.AttachUE()
-			elif action == 'Detach_UE':
-				SSH.DetachUE()
-			elif action == 'DataDisable_UE':
-				SSH.DataDisableUE()
-			elif action == 'DataEnable_UE':
-				SSH.DataEnableUE()
-			elif action == 'CheckStatusUE':
-				SSH.CheckStatusUE()
-			elif action == 'Build_OAI_UE':
-				SSH.BuildOAIUE()
-			elif action == 'Initialize_OAI_UE':
-				SSH.InitializeOAIUE()
-			elif action == 'Terminate_OAI_UE':
-				SSH.TerminateOAIUE()
-			elif action == 'Initialize_CatM_module':
-				SSH.InitializeCatM()
-			elif action == 'Terminate_CatM_module':
-				SSH.TerminateCatM()
-			elif action == 'Attach_CatM_module':
-				SSH.AttachCatM()
-			elif action == 'Detach_CatM_module':
-				SSH.TerminateCatM()
-			elif action == 'Ping_CatM_module':
-				SSH.PingCatM()
-			elif action == 'Ping':
-				SSH.Ping()
-			elif action == 'Iperf':
-				SSH.Iperf()
-			elif action == 'Reboot_UE':
-				SSH.RebootUE()
-			elif action == 'Initialize_HSS':
-				SSH.InitializeHSS()
-			elif action == 'Terminate_HSS':
-				SSH.TerminateHSS()
-			elif action == 'Initialize_MME':
-				SSH.InitializeMME()
-			elif action == 'Terminate_MME':
-				SSH.TerminateMME()
-			elif action == 'Initialize_SPGW':
-				SSH.InitializeSPGW()
-			elif action == 'Terminate_SPGW':
-				SSH.TerminateSPGW()
-			elif action == 'Initialize_FlexranCtrl':
-				SSH.InitializeFlexranCtrl()
-			elif action == 'Terminate_FlexranCtrl':
-				SSH.TerminateFlexranCtrl()
-			elif action == 'IdleSleep':
-				SSH.IdleSleep()
-			else:
-				sys.exit('Invalid action')
-
-	SSH.CreateHtmlTabFooter(True)
+	cnt = 0
+	SSH.prematureExit = True
+	while cnt < SSH.repeatCounts[0] and SSH.prematureExit:
+		SSH.prematureExit = False
+		for test_case_id in todo_tests:
+			for test in all_tests:
+				id = test.get('id')
+				if test_case_id != id:
+					continue
+				SSH.testCase_id = id
+				SSH.desc = test.findtext('desc')
+				action = test.findtext('class')
+				if (CheckClassValidity(action, id) == False):
+					continue
+				SSH.ShowTestID()
+				GetParametersFromXML(action)
+				if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE':
+					if (SSH.ADBIPAddress != 'none'):
+						terminate_ue_flag = False
+						SSH.GetAllUEDevices(terminate_ue_flag)
+				if action == 'Build_eNB':
+					SSH.BuildeNB()
+				elif action == 'WaitEndBuild_eNB':
+					SSH.WaitBuildeNBisFinished()
+				elif action == 'Initialize_eNB':
+					SSH.InitializeeNB()
+				elif action == 'Terminate_eNB':
+					SSH.TerminateeNB()
+				elif action == 'Initialize_UE':
+					SSH.InitializeUE()
+				elif action == 'Terminate_UE':
+					SSH.TerminateUE()
+				elif action == 'Attach_UE':
+					SSH.AttachUE()
+				elif action == 'Detach_UE':
+					SSH.DetachUE()
+				elif action == 'DataDisable_UE':
+					SSH.DataDisableUE()
+				elif action == 'DataEnable_UE':
+					SSH.DataEnableUE()
+				elif action == 'CheckStatusUE':
+					SSH.CheckStatusUE()
+				elif action == 'Build_OAI_UE':
+					SSH.BuildOAIUE()
+				elif action == 'Initialize_OAI_UE':
+					SSH.InitializeOAIUE()
+				elif action == 'Terminate_OAI_UE':
+					SSH.TerminateOAIUE()
+				elif action == 'Initialize_CatM_module':
+					SSH.InitializeCatM()
+				elif action == 'Terminate_CatM_module':
+					SSH.TerminateCatM()
+				elif action == 'Attach_CatM_module':
+					SSH.AttachCatM()
+				elif action == 'Detach_CatM_module':
+					SSH.TerminateCatM()
+				elif action == 'Ping_CatM_module':
+					SSH.PingCatM()
+				elif action == 'Ping':
+					SSH.Ping()
+				elif action == 'Iperf':
+					SSH.Iperf()
+				elif action == 'Reboot_UE':
+					SSH.RebootUE()
+				elif action == 'Initialize_HSS':
+					SSH.InitializeHSS()
+				elif action == 'Terminate_HSS':
+					SSH.TerminateHSS()
+				elif action == 'Initialize_MME':
+					SSH.InitializeMME()
+				elif action == 'Terminate_MME':
+					SSH.TerminateMME()
+				elif action == 'Initialize_SPGW':
+					SSH.InitializeSPGW()
+				elif action == 'Terminate_SPGW':
+					SSH.TerminateSPGW()
+				elif action == 'Initialize_FlexranCtrl':
+					SSH.InitializeFlexranCtrl()
+				elif action == 'Terminate_FlexranCtrl':
+					SSH.TerminateFlexranCtrl()
+				elif action == 'IdleSleep':
+					SSH.IdleSleep()
+				else:
+					sys.exit('Invalid action')
+				if SSH.prematureExit:
+					break
+			if SSH.prematureExit:
+				break
+		cnt += 1
+	if cnt == SSH.repeatCounts[0] and SSH.prematureExit:
+		logging.debug('Testsuite failed ' + str(cnt) + ' time(s)')
+		SSH.CreateHtmlTabFooter(False)
+		sys.exit('Failed Scenario')
+	else:
+		logging.info('Testsuite passed after ' + str(cnt) + ' time(s)')
+		SSH.CreateHtmlTabFooter(True)
 else:
 	Usage()
 	sys.exit('Invalid mode')
diff --git a/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1.xml b/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
index 48ebdaef69..1ee223d1c7 100644
--- a/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
+++ b/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
@@ -28,7 +28,6 @@
  030201
  040101
  030111 040301 040511 040613 040614 040615 040616 040617 040651 040652 040653 040654 040401 040201 030201
- 030112 040302 040512 040612 040650 040401 040201 030201
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
@@ -38,12 +37,6 @@
 		<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf --eNBs.[0].component_carriers.[0].drx_Config_present prRelease</Initialize_eNB_args>
 	</testCase>
 
-	<testCase id="030112">
-		<class>Initialize_eNB</class>
-		<desc>Initialize eNB (FDD/Band7/10MHz) with CDRX</desc>
-		<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf</Initialize_eNB_args>
-	</testCase>
-
 	<testCase id="030201">
 		<class>Terminate_eNB</class>
 		<desc>Terminate eNB</desc>
@@ -64,12 +57,6 @@
 		<desc>Attach UE</desc>
 	</testCase>
 
-	<testCase id="040302">
-		<class>Attach_UE</class>
-		<desc>Attach UE</desc>
-		<nbMaxUEtoAttach>1</nbMaxUEtoAttach>
-	</testCase>
-
 	<testCase id="040401">
 		<class>Detach_UE</class>
 		<desc>Detach UE</desc>
@@ -82,21 +69,6 @@
 		<ping_packetloss_threshold>5</ping_packetloss_threshold>
 	</testCase>
 
-	<testCase id="040512">
-		<class>Ping</class>
-		<desc>ping (10MHz - 20 sec)</desc>
-		<ping_args>-c 20</ping_args>
-		<ping_packetloss_threshold>5</ping_packetloss_threshold>
-	</testCase>
-
-	<testCase id="040612">
-		<class>Iperf</class>
-		<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
-		<iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
-		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
-		<iperf_profile>balanced</iperf_profile>
-	</testCase>
-
 	<testCase id="040613">
 		<class>Iperf</class>
 		<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
@@ -137,14 +109,6 @@
 		<iperf_profile>balanced</iperf_profile>
 	</testCase>
 
-	<testCase id="040650">
-		<class>Iperf</class>
-		<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
-		<iperf_args>-u -b 20M -t 30 -i 1 -R</iperf_args>
-		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
-		<iperf_profile>balanced</iperf_profile>
-	</testCase>
-
 	<testCase id="040651">
 		<class>Iperf</class>
 		<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
diff --git a/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1_cdrx.xml b/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1_cdrx.xml
new file mode 100644
index 0000000000..f62c31dd5a
--- /dev/null
+++ b/ci-scripts/xml_files/enb_usrp210_band7_test_10mhz_tm1_cdrx.xml
@@ -0,0 +1,90 @@
+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+	<htmlTabRef>test-10-cdrx-tm1</htmlTabRef>
+	<htmlTabName>Test-10MHz-CDRX-TM1</htmlTabName>
+	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
+	<TestCaseRequestedList>
+ 030201
+ 040101
+ 030112 040302 040512 040612 040650 040401 040201 030201
+	</TestCaseRequestedList>
+	<TestCaseExclusionList></TestCaseExclusionList>
+
+	<testCase id="030112">
+		<class>Initialize_eNB</class>
+		<desc>Initialize eNB (FDD/Band7/10MHz) with CDRX</desc>
+		<Initialize_eNB_args>-O ci-scripts/conf_files/enb.band7.tm1.50PRB.usrpb210.conf  --eNBs.[0].component_carriers.[0].drx_Config_present prSetup</Initialize_eNB_args>
+	</testCase>
+
+	<testCase id="030201">
+		<class>Terminate_eNB</class>
+		<desc>Terminate eNB</desc>
+	</testCase>
+
+	<testCase id="040101">
+		<class>Initialize_UE</class>
+		<desc>Initialize UE</desc>
+	</testCase>
+
+	<testCase id="040201">
+		<class>Terminate_UE</class>
+		<desc>Terminate UE</desc>
+	</testCase>
+
+	<testCase id="040302">
+		<class>Attach_UE</class>
+		<desc>Attach UE</desc>
+		<nbMaxUEtoAttach>1</nbMaxUEtoAttach>
+	</testCase>
+
+	<testCase id="040401">
+		<class>Detach_UE</class>
+		<desc>Detach UE</desc>
+	</testCase>
+
+	<testCase id="040512">
+		<class>Ping</class>
+		<desc>ping (10MHz - 20 sec)</desc>
+		<ping_args>-c 20</ping_args>
+		<ping_packetloss_threshold>5</ping_packetloss_threshold>
+	</testCase>
+
+	<testCase id="040612">
+		<class>Iperf</class>
+		<desc>iperf (10MHz - DL/30Mbps/UDP)(30 sec)(balanced profile)</desc>
+		<iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>balanced</iperf_profile>
+	</testCase>
+
+	<testCase id="040650">
+		<class>Iperf</class>
+		<desc>iperf (10MHz - UL/20Mbps/UDP)(30 sec)(balanced profile)</desc>
+		<iperf_args>-u -b 20M -t 30 -i 1 -R</iperf_args>
+		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
+		<iperf_profile>balanced</iperf_profile>
+	</testCase>
+
+</testCaseList>
diff --git a/ci-scripts/xml_files/f1_usrp210_band7_test_05mhz.xml b/ci-scripts/xml_files/f1_usrp210_band7_test_05mhz.xml
index 4dadbbdcba..fc14f5f7ff 100644
--- a/ci-scripts/xml_files/f1_usrp210_band7_test_05mhz.xml
+++ b/ci-scripts/xml_files/f1_usrp210_band7_test_05mhz.xml
@@ -22,8 +22,9 @@
 -->
 <testCaseList>
 	<htmlTabRef>test-f1-05</htmlTabRef>
-	<htmlTabName>Test-F1-CU/DU-05MHz</htmlTabName>
+	<htmlTabName>Test-F1-CU-DU-05MHz</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  030201
  040101
diff --git a/ci-scripts/xml_files/f1_usrp210_band7_test_10mhz.xml b/ci-scripts/xml_files/f1_usrp210_band7_test_10mhz.xml
index 115f3c648a..8d028df433 100644
--- a/ci-scripts/xml_files/f1_usrp210_band7_test_10mhz.xml
+++ b/ci-scripts/xml_files/f1_usrp210_band7_test_10mhz.xml
@@ -22,8 +22,9 @@
 -->
 <testCaseList>
 	<htmlTabRef>test-f1-10</htmlTabRef>
-	<htmlTabName>Test-F1-CU/DU-10MHz</htmlTabName>
+	<htmlTabName>Test-F1-CU-DU-10MHz</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  030201
  040101
diff --git a/ci-scripts/xml_files/f1_usrp210_band7_test_20mhz.xml b/ci-scripts/xml_files/f1_usrp210_band7_test_20mhz.xml
index 790afe2a71..061b87294b 100644
--- a/ci-scripts/xml_files/f1_usrp210_band7_test_20mhz.xml
+++ b/ci-scripts/xml_files/f1_usrp210_band7_test_20mhz.xml
@@ -22,8 +22,9 @@
 -->
 <testCaseList>
 	<htmlTabRef>test-f1-20</htmlTabRef>
-	<htmlTabName>Test-F1-CU/DU-20MHz</htmlTabName>
+	<htmlTabName>Test-F1-CU-DU-20MHz</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  030201
  040101
diff --git a/ci-scripts/xml_files/multi_rru_band38_test_10mhz_tm1.xml b/ci-scripts/xml_files/multi_rru_band38_test_10mhz_tm1.xml
index 70fba65f05..7658f8a0e5 100644
--- a/ci-scripts/xml_files/multi_rru_band38_test_10mhz_tm1.xml
+++ b/ci-scripts/xml_files/multi_rru_band38_test_10mhz_tm1.xml
@@ -24,6 +24,7 @@
 	<htmlTabRef>test-multi-rru-10</htmlTabRef>
 	<htmlTabName>Test-Multi-RRU-10MHz</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  030211 030212 030213
  030111 030112 030113
diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
index 19d08d5119..d3e36f603e 100644
--- a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
+++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
@@ -24,6 +24,7 @@
 	<htmlTabRef>test-10mhz-orange</htmlTabRef>
 	<htmlTabName>Test-10Mhz-Orange</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>	
+	<repeatCount>4</repeatCount>	
 	<TestCaseRequestedList>
  090109
  090110 000001 090109
diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
index 50912d3458..3f6441e06e 100644
--- a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
+++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
@@ -24,6 +24,7 @@
 	<htmlTabRef>test-10mHz-sfr</htmlTabRef>
 	<htmlTabName>Test-10MHz-SFR</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>4</repeatCount>
 	<TestCaseRequestedList>
  090109
  090111 000001 090109
-- 
GitLab