diff --git a/ci-scripts/NU_cls_oaicitest.py b/ci-scripts/NU_cls_oaicitest.py index 2cd78b77bd4dc1d22db5481af5b838df9d104cb0..99951671f750700d5a8ad43cb144138e1b788882 100755 --- a/ci-scripts/NU_cls_oaicitest.py +++ b/ci-scripts/NU_cls_oaicitest.py @@ -55,7 +55,8 @@ import helpreadme as HELP import constants as CONST import sshconnection - +import cls_module_ue +import cls_ci_ueinfra #class defining the multi Ue infrastrucure #----------------------------------------------------------- @@ -130,6 +131,7 @@ class OaiCiTest(): self.iperf_packetloss_threshold = '' self.iperf_profile = '' self.iperf_options = '' + self.iperf_direction = '' self.nbMaxUEtoAttach = -1 self.UEDevices = [] self.UEDevicesStatus = [] @@ -158,6 +160,8 @@ class OaiCiTest(): self.clean_repository = True self.air_interface='' self.expectedNbOfConnectedUEs = 0 + self.ue_id = '' #used for module identification + self.ue_trace ='' #used to enable QLog trace for Module UE, passed to Module UE object at InitializeUE() def BuildOAIUE(self,HTML): @@ -175,7 +179,7 @@ class OaiCiTest(): ue_prefix = '' result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: - full_ran_repo_name = self.ranRepository + full_ran_repo_name = self.ranRepository.replace('git/', 'git') else: full_ran_repo_name = self.ranRepository + '.git' SSH.command('mkdir -p ' + self.UESourceCodePath, '\$', 5) @@ -218,7 +222,7 @@ class OaiCiTest(): # if the commit ID is provided use it to point to it if self.ranCommitID != '': - SSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + SSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier if self.ranAllowMerge: @@ -365,29 +369,50 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def InitializeUE(self,HTML,COTS_UE): - if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - multi_jobs = [] - i = 0 - for device_id in self.UEDevices: - p = Process(target = self.InitializeUE_common, args = (device_id,i,COTS_UE,)) - p.daemon = True - p.start() - multi_jobs.append(p) - i += 1 - for job in multi_jobs: - job.join() - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - - + def InitializeUE(self,HTML,RAN,EPC, COTS_UE, InfraUE,ue_trace): + if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB + if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + multi_jobs = [] + i = 0 + for device_id in self.UEDevices: + p = Process(target = self.InitializeUE_common, args = (device_id,i,COTS_UE,)) + p.daemon = True + p.start() + multi_jobs.append(p) + i += 1 + for job in multi_jobs: + job.join() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + else: #if an ID is specified, it is a module from the yaml infrastructure file + #RH + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.ue_trace=ue_trace + is_module=Module_UE.CheckCMProcess() + if is_module: + Module_UE.EnableTrace() + time.sleep(5) + Module_UE.Command("wup") + logging.debug("Waiting for IP address to be assigned") + time.sleep(20) + logging.debug("Retrieve IP address") + status=Module_UE.GetModuleIPAddress() + if status==0: + HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) + logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) + else: #status==-1 failed to retrieve IP address + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return - def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE): + def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') + + if self.air_interface == 'lte-uesoftmodem': result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args)) if result is None: @@ -404,13 +429,13 @@ class OaiCiTest(): SSH = sshconnection.SSHConnection() SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword) # b2xx_fx3_utils reset procedure - SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 60) + SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 90) result = re.search('type: b200', SSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') SSH.command('echo ' + self.UEPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware - SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 60) + SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 90) result = re.search('type: n3xx', str(SSH.getBefore())) if result is not None: logging.debug('Found a N3xx device --> resetting it') @@ -564,7 +589,7 @@ class OaiCiTest(): SSH.command('ifconfig oaitun_ue1', '\$', 4) SSH.command('ifconfig oaitun_ue1', '\$', 4) # ifconfig output is different between ubuntu 16 and ubuntu 18 - result = re.search('inet addr:1|inet 1', SSH.getBefore()) + result = re.search('inet addr:[0-9]|inet [0-9]', SSH.getBefore()) if result is not None: logging.debug('\u001B[1m oaitun_ue1 interface is mounted and configured\u001B[0m') tunnelInterfaceStatus = True @@ -606,7 +631,7 @@ class OaiCiTest(): HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced' HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE') logging.error('\033[91mInitialize OAI UE Failed! \033[0m') - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def checkDevTTYisUnlocked(self): SSH = sshconnection.SSHConnection() @@ -684,7 +709,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) self.checkDevTTYisUnlocked() - def AttachCatM(self,HTML,RAN,COTS_UE,EPC): + def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -757,9 +782,9 @@ class OaiCiTest(): html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>' html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) - def PingCatM(self,HTML,RAN,EPC,COTS_UE): + def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -768,7 +793,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return try: statusQueue = SimpleQueue() @@ -789,7 +814,7 @@ class OaiCiTest(): moduleIPAddr = result.group('ipaddr') else: HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_time = re.findall("-c (\d+)",str(self.ping_args)) device_id = 'catm' @@ -853,7 +878,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) except: os.kill(os.getppid(),signal.SIGUSR1) @@ -945,64 +970,82 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def AttachUE(self,HTML,RAN,EPC,COTS_UE): - if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - check_eNB = True - check_OAI_UE = False - pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) - if (pStatus < 0): - HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) - return - multi_jobs = [] - status_queue = SimpleQueue() - lock = Lock() - nb_ue_to_connect = 0 - for device_id in self.UEDevices: - if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach): - self.UEDevicesStatus[nb_ue_to_connect] = CONST.UE_STATUS_ATTACHING - p = Process(target = self.AttachUE_common, args = (device_id, status_queue, lock,nb_ue_to_connect,COTS_UE,)) - p.daemon = True - p.start() - multi_jobs.append(p) - nb_ue_to_connect = nb_ue_to_connect + 1 - for job in multi_jobs: - job.join() - - if (status_queue.empty()): - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) - return - else: - attach_status = True - html_queue = SimpleQueue() - while (not status_queue.empty()): - count = status_queue.get() - if (count < 0): - attach_status = False - device_id = status_queue.get() - message = status_queue.get() - if (count < 0): - html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>' - else: - html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + ' in ' + str(count + 2) + ' seconds</pre>' - html_queue.put(html_cell) - if (attach_status): - cnt = 0 - while cnt < len(self.UEDevices): - if self.UEDevicesStatus[cnt] == CONST.UE_STATUS_ATTACHING: - self.UEDevicesStatus[cnt] = CONST.UE_STATUS_ATTACHED - cnt += 1 - HTML.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue) - result = re.search('T_stdout', str(RAN.Initialize_eNB_args)) - if result is not None: - logging.debug('Waiting 5 seconds to fill up record file') - time.sleep(5) + def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): + if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB + if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) + if (pStatus < 0): + HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return + multi_jobs = [] + status_queue = SimpleQueue() + lock = Lock() + nb_ue_to_connect = 0 + for device_id in self.UEDevices: + if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach): + self.UEDevicesStatus[nb_ue_to_connect] = CONST.UE_STATUS_ATTACHING + p = Process(target = self.AttachUE_common, args = (device_id, status_queue, lock,nb_ue_to_connect,COTS_UE,)) + p.daemon = True + p.start() + multi_jobs.append(p) + nb_ue_to_connect = nb_ue_to_connect + 1 + for job in multi_jobs: + job.join() + + if (status_queue.empty()): + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return else: - HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + attach_status = True + html_queue = SimpleQueue() + while (not status_queue.empty()): + count = status_queue.get() + if (count < 0): + attach_status = False + device_id = status_queue.get() + message = status_queue.get() + if (count < 0): + html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>' + else: + html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + ' in ' + str(count + 2) + ' seconds</pre>' + html_queue.put(html_cell) + if (attach_status): + cnt = 0 + while cnt < len(self.UEDevices): + if self.UEDevicesStatus[cnt] == CONST.UE_STATUS_ATTACHING: + self.UEDevicesStatus[cnt] = CONST.UE_STATUS_ATTACHED + cnt += 1 + HTML.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue) + result = re.search('T_stdout', str(RAN.Initialize_eNB_args)) + if result is not None: + logging.debug('Waiting 5 seconds to fill up record file') + time.sleep(5) + else: + HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + + else: #if an ID is specified, it is a module from the yaml infrastructure file + #Attention, as opposed to InitializeUE, the connect manager process is not checked as it is supposed to be active already + #only 1- module wakeup, 2- check IP address + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.Command("wup") + logging.debug("Waiting for IP address to be assigned") + time.sleep(20) + logging.debug("Retrieve IP address") + status=Module_UE.GetModuleIPAddress() + if status==0: + HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) + logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) + else: #status==-1 failed to retrieve IP address + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return def DetachUE_common(self, device_id, idx,COTS_UE): try: @@ -1026,37 +1069,44 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def DetachUE(self,HTML,RAN,EPC,COTS_UE): - if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - check_eNB = True - check_OAI_UE = False - pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) - if (pStatus < 0): - HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) - return - multi_jobs = [] - cnt = 0 - for device_id in self.UEDevices: - self.UEDevicesStatus[cnt] = CONST.UE_STATUS_DETACHING - p = Process(target = self.DetachUE_common, args = (device_id,cnt,COTS_UE,)) - p.daemon = True - p.start() - multi_jobs.append(p) - cnt += 1 - for job in multi_jobs: - job.join() - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - result = re.search('T_stdout', str(RAN.Initialize_eNB_args)) - if result is not None: - logging.debug('Waiting 5 seconds to fill up record file') - time.sleep(5) - cnt = 0 - while cnt < len(self.UEDevices): - self.UEDevicesStatus[cnt] = CONST.UE_STATUS_DETACHED - cnt += 1 + def DetachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): + if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB + if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) + if (pStatus < 0): + HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return + multi_jobs = [] + cnt = 0 + for device_id in self.UEDevices: + self.UEDevicesStatus[cnt] = CONST.UE_STATUS_DETACHING + p = Process(target = self.DetachUE_common, args = (device_id,cnt,COTS_UE,)) + p.daemon = True + p.start() + multi_jobs.append(p) + cnt += 1 + for job in multi_jobs: + job.join() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + result = re.search('T_stdout', str(RAN.Initialize_eNB_args)) + if result is not None: + logging.debug('Waiting 5 seconds to fill up record file') + time.sleep(5) + cnt = 0 + while cnt < len(self.UEDevices): + self.UEDevicesStatus[cnt] = CONST.UE_STATUS_DETACHED + cnt += 1 + else:#if an ID is specified, it is a module from the yaml infrastructure file + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.Command("detach") + HTML.CreateHtmlTestRow('NA', 'OK', CONST.ALL_PROCESSES_OK) + + def RebootUE_common(self, device_id): try: @@ -1304,7 +1354,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE): + def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -1351,7 +1401,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: check_status = True html_queue = SimpleQueue() @@ -1367,7 +1417,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def GetAllUEIPAddresses(self): SSH = sshconnection.SSHConnection() @@ -1438,14 +1488,20 @@ class OaiCiTest(): statusQueue.put(message) lock.release() - def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC): + def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE): try: SSH = sshconnection.SSHConnection() # Launch ping on the EPC side (true for ltebox and old open-air-cn) # But for OAI-Rel14-CUPS, we launch from python executor launchFromEpc = True + launchFromModule = False if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): launchFromEpc = False + #if module, ping from module to EPC + if self.ue_id!='': + launchFromEpc = False + launchfromModule = True + ping_time = re.findall("-c (\d+)",str(self.ping_args)) if launchFromEpc: @@ -1463,21 +1519,39 @@ class OaiCiTest(): #copy the ping log file to have it locally for analysis (ping stats) SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') else: - #ping log file is on the python executor - cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 > ping_' + self.testCase_id + '_' + device_id + '.log' - message = cmd + '\n' - logging.debug(cmd) - ret = subprocess.run(cmd, shell=True) - ping_status = ret.returncode - #copy the ping log file to an other folder for log collection (source and destination are EPC) - SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts') - #copy the ping log file to have it locally for analysis (ping stats) - logging.debug(EPC.SourceCodePath + 'ping_' + self.testCase_id + '_' + device_id + '.log') - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath +'/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') + if launchfromModule == False: + #ping log file is on the python executor + cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 > ping_' + self.testCase_id + '_' + device_id + '.log' + message = cmd + '\n' + logging.debug(cmd) + ret = subprocess.run(cmd, shell=True) + ping_status = ret.returncode + #copy the ping log file to an other folder for log collection (source and destination are EPC) + SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts') + #copy the ping log file to have it locally for analysis (ping stats) + logging.debug(EPC.SourceCodePath + 'ping_' + self.testCase_id + '_' + device_id + '.log') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath +'/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') + + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + #cat is executed on EPC + SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) + else: #launch from Module + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + #target address is different depending on EPC type + if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + Target = EPC.MmeIPAddress + else: + Target = EPC.IPAddress + #ping from module NIC rather than IP address to make sure round trip is over the air + cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',int(ping_time[0])*1.5) + #copy the ping log file to have it locally for analysis (ping stats) + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + + #cat is executed locally + SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5) + ping_status=0 - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - #cat is executed on EPC - SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) # TIMEOUT CASE if ping_status < 0: message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!' @@ -1568,14 +1642,14 @@ class OaiCiTest(): html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - def PingNoS1(self,HTML,RAN,EPC,COTS_UE): + def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH=sshconnection.SSHConnection() check_eNB = True check_OAI_UE = True pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args)) if ping_from_eNB is not None: @@ -1661,10 +1735,10 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def Ping(self,HTML,RAN,EPC,COTS_UE): + def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.PingNoS1(HTML,RAN,EPC,COTS_UE) + self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -1677,20 +1751,32 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) - return - ueIpStatus = self.GetAllUEIPAddresses() - if (ueIpStatus < 0): - HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return + + if self.ue_id=="": + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra['dummy']) #RH, temporary, we need a dummy Module_UE object to pass to Ping_common + ueIpStatus = self.GetAllUEIPAddresses() + if (ueIpStatus < 0): + HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return + else: + self.UEIPAddresses=[] + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.GetModuleIPAddress() + self.UEIPAddresses.append(Module_UE.UEIPAddress) + logging.debug(self.UEIPAddresses) multi_jobs = [] i = 0 lock = Lock() status_queue = SimpleQueue() for UE_IPAddress in self.UEIPAddresses: - device_id = self.UEDevices[i] - p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,)) + if self.ue_id=="": + device_id = self.UEDevices[i] + else: + device_id = Module_UE.ID + "-" + Module_UE.Kind + p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,)) p.daemon = True p.start() multi_jobs.append(p) @@ -1700,7 +1786,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: ping_status = True html_queue = SimpleQueue() @@ -1717,7 +1803,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def Iperf_ComputeTime(self): result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args)) @@ -1865,8 +1951,8 @@ class OaiCiTest(): else: return -2 - def Iperf_analyzeV2Server(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options): - if (not os.path.isfile('iperf_server_' + self.testCase_id + '_' + device_id + '.log')): + def Iperf_analyzeV2Server(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options, filename,type): + if (not os.path.isfile(filename)): self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log') return # Computing the requested bandwidth in float @@ -1891,14 +1977,18 @@ class OaiCiTest(): req_bandwidth = '%.1f Gbits/sec' % req_bw req_bw = req_bw * 1000000000 - server_file = open('iperf_server_' + self.testCase_id + '_' + device_id + '.log', 'r') + server_file = open(filename, 'r') br_sum = 0.0 ji_sum = 0.0 pl_sum = 0 ps_sum = 0 row_idx = 0 for line in server_file.readlines(): - result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line)) + if type==0: + result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line)) + else: + result = re.search('^\[ \d\].+ +(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)\/(?P<sentPack>[0-9]+)', str(line)) + if result is not None: bitrate = result.group('bitrate') jitter = result.group('jitter') @@ -2128,12 +2218,95 @@ class OaiCiTest(): SSH.command('docker cp prod-trf-gen:/iperf-2.0.5/iperf_server_' + self.testCase_id + '_' + device_id + '.log ' + EPC.SourceCodePath + '/scripts', '\$', 5) SSH.close() SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath+ '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.') - self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options) + filename='iperf_server_' + self.testCase_id + '_' + device_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options,filename,0) # in case of OAI-UE if (device_id == 'OAI-UE'): SSH.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/iperf_' + self.testCase_id + '_' + device_id + '.log', '.') SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts') + + def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): + SSH = sshconnection.SSHConnection() + #RH temporary quick n dirty for test + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'echo ' + EPC.Password + ' | sudo -S ip link set dev tun5 mtu 1358' + SSH.command(cmd,'\$',5) + SSH.close() + + + #kill iperf processes before (in case there are still some remaining) + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + + + iperf_time = self.Iperf_ComputeTime() + if self.iperf_direction=="DL": + logging.debug("Iperf for Module in DL mode detected") + #server side UE + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.close() + #client side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',int(iperf_time)*5.0) + SSH.close() + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + #send for analysis + filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) + + elif self.iperf_direction=="UL":#does not work at the moment + logging.debug("Iperf for Module in UL mode detected") + #server side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.close() + + #client side UE + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0) + SSH.close() + + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + #send for analysis + filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) + else : + logging.debug("Incorrect or missing IPERF direction in XML") + + #kill iperf processes after to be clean + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + return + def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC): try: SSH = sshconnection.SSHConnection() @@ -2230,8 +2403,13 @@ class OaiCiTest(): # Launch the IPERF client on the EPC side for DL (true for ltebox and old open-air-cn # But for OAI-Rel14-CUPS, we launch from python executor launchFromEpc = True + launchFromModule = False if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): launchFromEpc = False + #if module + if self.ue_id!='' and self.iperf : + launchFromEpc = False + launchfromModule = True # When using a docker-based deployment, IPERF client shall be launched from trf container launchFromTrfContainer = False if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): @@ -2335,7 +2513,8 @@ class OaiCiTest(): subprocess.run(cmd, shell=True) except: pass - self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options) + filename='iperf_server_' + self.testCase_id + '_' + device_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options,filename,0) # in case of OAI UE: if (device_id == 'OAI-UE'): @@ -2348,7 +2527,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def IperfNoS1(self,HTML,RAN,EPC,COTS_UE): + def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH = sshconnection.SSHConnection() if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '': HELP.GenericHelp(CONST.Version) @@ -2358,7 +2537,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return server_on_enb = re.search('-R', str(self.iperf_args)) if server_on_enb is not None: @@ -2423,7 +2602,8 @@ class OaiCiTest(): if (os.path.isfile('iperf_server_' + self.testCase_id + '.log')): os.remove('iperf_server_' + self.testCase_id + '.log') SSH.copyin(iServerIPAddr, iServerUser, iServerPasswd, '/tmp/tmp_iperf_server_' + self.testCase_id + '.log', 'iperf_server_' + self.testCase_id + '_OAI-UE.log') - self.Iperf_analyzeV2Server(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options) + filename='iperf_server_' + self.testCase_id + '_OAI-UE.log' + self.Iperf_analyzeV2Server(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options,filename,0) # copying on the EPC server for logCollection if (clientStatus == -1): @@ -2456,12 +2636,12 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) - def Iperf(self,HTML,RAN,EPC,COTS_UE): + def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.IperfNoS1(HTML,RAN,EPC,COTS_UE) + self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -2474,15 +2654,24 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) - return - ueIpStatus = self.GetAllUEIPAddresses() - if (ueIpStatus < 0): - logging.debug('going here') - HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return + if self.ue_id=="":#is not a module, follow legacy code + ueIpStatus = self.GetAllUEIPAddresses() + if (ueIpStatus < 0): + HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + return + else: #is a module + self.UEIPAddresses=[] + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.GetModuleIPAddress() + self.UEIPAddresses.append(Module_UE.UEIPAddress) + + + + self.dummyIperfVersion = '2.0.10' #cmd = 'iperf --version' #logging.debug(cmd + '\n') @@ -2499,9 +2688,15 @@ class OaiCiTest(): ue_num = len(self.UEIPAddresses) lock = Lock() status_queue = SimpleQueue() + logging.debug(self.UEIPAddresses) for UE_IPAddress in self.UEIPAddresses: device_id = self.UEDevices[i] - p = Process(target = self.Iperf_common, args = (lock,UE_IPAddress,device_id,i,ue_num,status_queue,EPC,)) + #special quick and dirty treatment for modules, iperf to be restructured + if self.ue_id!="": #is module + device_id = Module_UE.ID + "-" + Module_UE.Kind + p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE,)) + else: #legacy code + p = Process(target = self.Iperf_common, args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, )) p.daemon = True p.start() multi_jobs.append(p) @@ -2511,7 +2706,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfaUE) else: iperf_status = True iperf_noperf = False @@ -2533,7 +2728,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def CheckProcessExist(self, check_eNB, check_OAI_UE,RAN,EPC): multi_jobs = [] @@ -2938,22 +3133,34 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def TerminateUE(self,HTML,COTS_UE): - terminate_ue_flag = False - self.GetAllUEDevices(terminate_ue_flag) - multi_jobs = [] - i = 0 - for device_id in self.UEDevices: - p = Process(target= self.TerminateUE_common, args = (device_id,i,COTS_UE,)) - p.daemon = True - p.start() - multi_jobs.append(p) - i += 1 - for job in multi_jobs: - job.join() - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + def TerminateUE(self,HTML,COTS_UE,InfraUE,ue_trace): + if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB + terminate_ue_flag = False + self.GetAllUEDevices(terminate_ue_flag) + multi_jobs = [] + i = 0 + for device_id in self.UEDevices: + p = Process(target= self.TerminateUE_common, args = (device_id,i,COTS_UE,)) + p.daemon = True + p.start() + multi_jobs.append(p) + i += 1 + for job in multi_jobs: + job.join() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + else: #if an ID is specified, it is a module from the yaml infrastructure file + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.ue_trace=ue_trace + Module_UE.Command("detach") + Module_UE.DisableTrace() + Module_UE.DisableCM() + archive_destination=Module_UE.LogCollect() + if Module_UE.ue_trace=='yes': + HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK) + else: + HTML.CreateHtmlTestRow('QLog trace is disabled', 'OK', CONST.ALL_PROCESSES_OK) - def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC): + def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC, InfraUE): SSH = sshconnection.SSHConnection() SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword) SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) @@ -2994,11 +3201,11 @@ class OaiCiTest(): # Not an error then if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m') HTML.htmlUEFailureMsg='<b>' + ueAction + ' Completed</b>\n' + HTML.htmlUEFailureMsg @@ -3042,21 +3249,21 @@ class OaiCiTest(): logging.info('\u001B[1m' + UELogFile + ' Not found \u001B[0m') #HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC): + def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE): if (self.ADBIPAddress != 'none'): self.testCase_id = 'AUTO-KILL-UE' HTML.testCase_id=self.testCase_id self.desc = 'Automatic Termination of UE' HTML.desc='Automatic Termination of UE' self.ShowTestID() - self.TerminateUE(HTML,COTS_UE) + self.TerminateUE(HTML,COTS_UE,InfraUE,self.ue_trace) if (self.Initialize_OAI_UE_args != ''): self.testCase_id = 'AUTO-KILL-OAI-UE' HTML.testCase_id=self.testCase_id self.desc = 'Automatic Termination of OAI-UE' HTML.desc='Automatic Termination of OAI-UE' self.ShowTestID() - self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) if (RAN.Initialize_eNB_args != ''): self.testCase_id = 'AUTO-KILL-RAN' HTML.testCase_id=self.testCase_id @@ -3309,7 +3516,7 @@ class OaiCiTest(): UhdVersion = result.group('uhd_version') logging.debug('UHD Version is: ' + UhdVersion) HTML.UhdVersion[idx]=UhdVersion - SSH.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 60) + SSH.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 90) usrp_boards = re.findall('product: ([0-9A-Za-z]+)\\\\r\\\\n', SSH.getBefore()) count = 0 for board in usrp_boards: diff --git a/ci-scripts/NU_main.py b/ci-scripts/NU_main.py index 93436292f89791a2c8f9712d2444f8b4a6d73140..4282f8a5736b8f66801c0f9b650884176c7fc6a2 100755 --- a/ci-scripts/NU_main.py +++ b/ci-scripts/NU_main.py @@ -38,11 +38,13 @@ import helpreadme as HELP import constants as CONST -import NU_cls_oaicitest #main class for OAI CI test framework -import cls_physim #class PhySim for physical simulators build and test -import cls_cots_ue #class CotsUe for Airplane mode control -import cls_containerize #class Containerize for all container-based operations on RAN/UE objects - +import NU_cls_oaicitest #main class for OAI CI test framework +import cls_physim #class PhySim for physical simulators build and test +import cls_cots_ue #class CotsUe for Airplane mode control +import cls_containerize #class Containerize for all container-based operations on RAN/UE objects +import cls_static_code_analysis #class for static code analysis +import cls_ci_ueinfra #class defining the multi Ue infrastrucure +import cls_physim1 #class PhySim for physical simulators deploy and run import sshconnection import epc @@ -63,6 +65,7 @@ import xml.etree.ElementTree as ET import logging import datetime import signal +import subprocess from multiprocessing import Process, Lock, SimpleQueue logging.basicConfig( level=logging.DEBUG, @@ -150,8 +153,14 @@ def GetParametersFromXML(action): RAN.eNB_serverId[RAN.eNB_instance]=eNB_serverId elif action == 'Initialize_eNB': + RAN.eNB_Trace=test.findtext('eNB_Trace') RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') eNB_instance=test.findtext('eNB_instance') + USRPIPAddress=test.findtext('USRP_IPAddress') + if USRPIPAddress is None: + RAN.USRPIPAddress='' + else: + RAN.USRPIPAddress=USRPIPAddress if (eNB_instance is None): RAN.eNB_instance=0 else: @@ -192,13 +201,41 @@ def GetParametersFromXML(action): else : RAN.air_interface[RAN.eNB_instance] = 'ocp-enb' + elif action == 'Initialize_UE': + ue_id = test.findtext('id') + CiTestObj.ue_trace=test.findtext('UE_Trace')#temporary variable, to be passed to Module_UE in Initialize_UE call + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id + + elif action == 'Detach_UE': + ue_id = test.findtext('id') + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id + elif action == 'Attach_UE': + ue_id = test.findtext('id') + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id nbMaxUEtoAttach = test.findtext('nbMaxUEtoAttach') if (nbMaxUEtoAttach is None): CiTestObj.nbMaxUEtoAttach = -1 else: CiTestObj.nbMaxUEtoAttach = int(nbMaxUEtoAttach) + elif action == 'Terminate_UE': + ue_id = test.findtext('id') + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id + + elif action == 'CheckStatusUE': expectedNBUE = test.findtext('expectedNbOfConnectedUEs') if (expectedNBUE is None): @@ -252,9 +289,20 @@ def GetParametersFromXML(action): elif (action == 'Ping') or (action == 'Ping_CatM_module'): CiTestObj.ping_args = test.findtext('ping_args') CiTestObj.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold') + ue_id = test.findtext('id') + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id elif action == 'Iperf': CiTestObj.iperf_args = test.findtext('iperf_args') + ue_id = test.findtext('id') + if (ue_id is None): + CiTestObj.ue_id = "" + else: + CiTestObj.ue_id = ue_id + CiTestObj.iperf_direction = test.findtext('direction')#used for modules only CiTestObj.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold') CiTestObj.iperf_profile = test.findtext('iperf_profile') if (CiTestObj.iperf_profile is None): @@ -325,6 +373,41 @@ def GetParametersFromXML(action): if (string_field is not None): CONTAINERS.yamlPath[CONTAINERS.eNB_instance] = string_field + elif action == 'DeployGenObject' or action == 'UndeployGenObject': + string_field=test.findtext('yaml_path') + if (string_field is not None): + CONTAINERS.yamlPath[0] = string_field + string_field=test.findtext('services') + if (string_field is not None): + CONTAINERS.services[0] = string_field + string_field=test.findtext('nb_healthy') + if (string_field is not None): + CONTAINERS.nb_healthy[0] = int(string_field) + + elif action == 'PingFromContainer': + string_field = test.findtext('container_name') + if (string_field is not None): + CONTAINERS.pingContName = string_field + string_field = test.findtext('options') + if (string_field is not None): + CONTAINERS.pingOptions = string_field + string_field = test.findtext('loss_threshold') + if (string_field is not None): + CONTAINERS.pingLossThreshold = string_field + + elif action == 'IperfFromContainer': + string_field = test.findtext('server_container_name') + if (string_field is not None): + CONTAINERS.svrContName = string_field + string_field = test.findtext('server_options') + if (string_field is not None): + CONTAINERS.svrOptions = string_field + string_field = test.findtext('client_container_name') + if (string_field is not None): + CONTAINERS.cliContName = string_field + string_field = test.findtext('client_options') + if (string_field is not None): + CONTAINERS.cliOptions = string_field else: # ie action == 'Run_PhySim': ldpc.runargs = test.findtext('physim_run_args') @@ -369,6 +452,20 @@ with open(yaml_file,'r') as f: +#loading UE infrastructure from yaml +ue_infra_file='ci_ueinfra.yaml' +if (os.path.isfile(ue_infra_file)): + yaml_file=ue_infra_file +elif (os.path.isfile('ci-scripts/'+ue_infra_file)): + yaml_file='ci-scripts/'+ue_infra_file +else: + logging.error("UE infrastructure yaml file cannot be found") + sys.exit("UE infrastructure file cannot be found") +InfraUE=cls_ci_ueinfra.InfraUE() #initialize UE infrastructure class +InfraUE.Get_UE_Infra(yaml_file) #read the UE infra, filename is hardcoded and unique for the moment but should be passed as parameter from the test suite + + + mode = '' CiTestObj = NU_cls_oaicitest.OaiCiTest() @@ -378,17 +475,18 @@ EPC = epc.EPCManagement() RAN = NU_ran.RANManagement() HTML = html.HTMLManagement() CONTAINERS = cls_containerize.Containerize() +SCA = cls_static_code_analysis.StaticCodeAnalysis() +PHYSIM = cls_physim1.PhySim() ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build - #----------------------------------------------------------- # Parsing Command Line Arguments #----------------------------------------------------------- import args_parse -py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP) +py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM) @@ -417,6 +515,8 @@ if re.match('^TerminateeNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') + if RAN.eNBIPAddress == 'none': + sys.exit(0) RAN.eNB_instance=0 RAN.eNB_serverId[0]='0' RAN.eNBSourceCodePath='/tmp/' @@ -431,13 +531,13 @@ elif re.match('^TerminateUE$', mode, re.IGNORECASE): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) - CiTestObj.TerminateUE(HTML,COTS_UE) + CiTestObj.TerminateUE(HTML,COTS_UE,InfraUE,CiTestObj.ue_trace) elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): if CiTestObj.UEIPAddress == '' or CiTestObj.UEUserName == '' or CiTestObj.UEPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif re.match('^AnalyzeOAIUE$', mode, re.IGNORECASE): if CiTestObj.UEIPAddress == '' or CiTestObj.UEUserName == '' or CiTestObj.UEPassword == '': HELP.GenericHelp(CONST.Version) @@ -463,11 +563,18 @@ elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): if (RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '') and (CiTestObj.UEIPAddress == '' or CiTestObj.UEUserName == '' or CiTestObj.UEPassword == '' or CiTestObj.UESourceCodePath == ''): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') + if RAN.eNBIPAddress == 'none': + sys.exit(0) CiTestObj.LogCollectBuild(RAN) elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') + if RAN.eNBIPAddress == 'none': + cmd = 'zip -r enb.log.' + RAN.BuildId + '.zip cmake_targets/log' + logging.debug(cmd) + zipStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=60) + sys.exit(0) RAN.LogCollecteNB() elif re.match('^LogCollectHSS$', mode, re.IGNORECASE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '': @@ -522,7 +629,7 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): if foundCount != HTML.nbTestXMLfiles: HTML.nbTestXMLfiles=foundCount - if (CiTestObj.ADBIPAddress != 'none'): + if (CiTestObj.ADBIPAddress != 'none') and (CiTestObj.ADBIPAddress != 'modules'): terminate_ue_flag = False CiTestObj.GetAllUEDevices(terminate_ue_flag) CiTestObj.GetAllCatMDevices(terminate_ue_flag) @@ -613,6 +720,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re if (EPC.IPAddress != '') and (EPC.IPAddress != 'none'): CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC,CONTAINERS) EPC.SetMmeIPAddress() + EPC.SetAmfIPAddress() #get the list of tests to be done todo_tests=[] @@ -625,10 +733,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re signal.signal(signal.SIGUSR1, receive_signal) - if (CiTestObj.ADBIPAddress != 'none'): + if (CiTestObj.ADBIPAddress != 'none') and (CiTestObj.ADBIPAddress != 'modules'): terminate_ue_flag = False CiTestObj.GetAllUEDevices(terminate_ue_flag) CiTestObj.GetAllCatMDevices(terminate_ue_flag) + elif (CiTestObj.ADBIPAddress == 'modules'): + CiTestObj.UEDevices.append('COTS-Module') else: CiTestObj.UEDevices.append('OAI-UE') HTML.SethtmlUEConnected(len(CiTestObj.UEDevices) + len(CiTestObj.CatMDevices)) @@ -662,7 +772,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CiTestObj.ShowTestID() GetParametersFromXML(action) if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE': - if (CiTestObj.ADBIPAddress != 'none'): + if (CiTestObj.ADBIPAddress != 'none') and (CiTestObj.ADBIPAddress != 'modules'): #in these cases, having no devices is critical, GetAllUEDevices function has to manage it as a critical error, reason why terminate_ue_flag is set to True terminate_ue_flag = True # Now we stop properly the test-suite --> clean reporting @@ -682,39 +792,39 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Terminate_eNB': RAN.TerminateeNB(HTML, EPC) elif action == 'Initialize_UE': - CiTestObj.InitializeUE(HTML,COTS_UE) + CiTestObj.InitializeUE(HTML,RAN, EPC, COTS_UE, InfraUE, CiTestObj.ue_trace) elif action == 'Terminate_UE': - CiTestObj.TerminateUE(HTML,COTS_UE) + CiTestObj.TerminateUE(HTML,COTS_UE, InfraUE, CiTestObj.ue_trace) elif action == 'Attach_UE': - CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Detach_UE': - CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'DataDisable_UE': CiTestObj.DataDisableUE(HTML) elif action == 'DataEnable_UE': CiTestObj.DataEnableUE(HTML) elif action == 'CheckStatusUE': - CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Build_OAI_UE': CiTestObj.BuildOAIUE(HTML) elif action == 'Initialize_OAI_UE': - CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Terminate_OAI_UE': - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Initialize_CatM_module': CiTestObj.InitializeCatM(HTML) elif action == 'Terminate_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Attach_CatM_module': - CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC) + CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Detach_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Ping_CatM_module': - CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC) + CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE) elif action == 'Ping': - CiTestObj.Ping(HTML,RAN,EPC,COTS_UE) + CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE) elif action == 'Iperf': - CiTestObj.Iperf(HTML,RAN,EPC,COTS_UE) + CiTestObj.Iperf(HTML,RAN,EPC,COTS_UE, InfraUE) elif action == 'Reboot_UE': CiTestObj.RebootUE(HTML,RAN,EPC) elif action == 'Initialize_HSS': @@ -729,6 +839,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re EPC.InitializeSPGW(HTML) elif action == 'Terminate_SPGW': EPC.TerminateSPGW(HTML) + elif action == 'Initialize_5GCN': + EPC.Initialize5GCN(HTML) + elif action == 'Terminate_5GCN': + EPC.Terminate5GCN(HTML) elif action == 'Deploy_EPC': EPC.DeployEpc(HTML) elif action == 'Undeploy_EPC': @@ -743,7 +857,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CiTestObj.Perform_X2_Handover(HTML,RAN,EPC) elif action == 'Build_PhySim': HTML=ldpc.Build_PhySim(HTML,CONST) - if ldpc.exitStatus==1:sys.exit() + if ldpc.exitStatus==1: + RAN.prematureExit = True elif action == 'Run_PhySim': HTML=ldpc.Run_PhySim(HTML,CONST,id) elif action == 'Build_Image': @@ -752,9 +867,29 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CONTAINERS.DeployObject(HTML, EPC) elif action == 'Undeploy_Object': CONTAINERS.UndeployObject(HTML, RAN) + elif action == 'Cppcheck_Analysis': + SCA.CppCheckAnalysis(HTML) + elif action == 'Deploy_Run_PhySim': + PHYSIM.Deploy_PhySim(HTML, RAN) + elif action == 'DeployGenObject': + CONTAINERS.DeployGenObject(HTML) + if CONTAINERS.exitStatus==1: + RAN.prematureExit = True + elif action == 'UndeployGenObject': + CONTAINERS.UndeployGenObject(HTML) + if CONTAINERS.exitStatus==1: + RAN.prematureExit = True + elif action == 'PingFromContainer': + CONTAINERS.PingFromContainer(HTML) + if CONTAINERS.exitStatus==1: + RAN.prematureExit = True + elif action == 'IperfFromContainer': + CONTAINERS.IperfFromContainer(HTML) + if CONTAINERS.exitStatus==1: + RAN.prematureExit = True else: sys.exit('Invalid class (action) from xml') - if not RAN.prematureExit: + if RAN.prematureExit: if CiTestObj.testCase_id == CiTestObj.testMinStableId: logging.debug('Scenario has reached minimal stability point') CiTestObj.testStabilityPointReached = True diff --git a/ci-scripts/NU_ran.py b/ci-scripts/NU_ran.py index c4d892d9887756063fc382de62d1e8a81365569c..c5377369da5a9819d87c7d8bbc396832e344a984 100755 --- a/ci-scripts/NU_ran.py +++ b/ci-scripts/NU_ran.py @@ -38,6 +38,8 @@ import os from os import path import time from multiprocessing import Process, Lock, SimpleQueue +import yaml + #----------------------------------------------------------- # OAI Testing modules @@ -91,6 +93,9 @@ class RANManagement(): self.testCase_id = '' self.epcPcapFile = '' self.runtime_stats= '' + self.datalog_rt_stats={} + self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization + self.USRPIPAddress = '' @@ -128,12 +133,16 @@ class RANManagement(): result = re.search('--eNBocp', self.Build_eNB_args) if result is not None: self.air_interface[self.eNB_instance] = 'ocp-enb' - else: - result = re.search('--gNB', self.Build_eNB_args) + else: + result = re.search('--RU', self.Build_eNB_args) if result is not None: - self.air_interface[self.eNB_instance] = 'nr-softmodem' + self.air_interface[self.eNB_instance] = 'oairu' else: - self.air_interface[self.eNB_instance] = 'lte-softmodem' + result = re.search('--gNB', self.Build_eNB_args) + if result is not None: + self.air_interface[self.eNB_instance] = 'nr-softmodem' + else: + self.air_interface[self.eNB_instance] = 'lte-softmodem' # Worakround for some servers, we need to erase completely the workspace if self.Build_eNB_forced_workspace_cleanup: @@ -142,7 +151,7 @@ class RANManagement(): # on RedHat/CentOS .git extension is mandatory result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: - full_ran_repo_name = self.ranRepository + full_ran_repo_name = self.ranRepository.replace('git/', 'git') else: full_ran_repo_name = self.ranRepository + '.git' mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) @@ -185,7 +194,7 @@ class RANManagement(): mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) # if the commit ID is provided use it to point to it if self.ranCommitID != '': - mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier if (self.ranAllowMerge): @@ -294,7 +303,7 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5) #-qq quiet / -u update orcreate files - mySSH.command('unzip -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5) + mySSH.command('unzip -o -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5) mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5) mySSH.close() else: @@ -334,10 +343,41 @@ class RANManagement(): self.testCase_id = HTML.testCase_id mySSH = SSH.SSHConnection() + #reboot USRP if requested in xml + if self.USRPIPAddress!='': + logging.debug('USRP '+ self.USRPIPAddress +'reboot request') + mySSH.open(lIpAddr, lUserName, lPassWord) + cmd2usrp='ssh root@'+self.USRPIPAddress+' reboot' + mySSH.command2(cmd2usrp,1) + mySSH.close() + logging.debug('Waiting for USRP to be ready') + time.sleep(120) + + if (self.pStatus < 0): HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus) HTML.CreateHtmlTabFooter(False) sys.exit(1) + + #Get pcap on enb and/or gnb if enabled in the xml + if self.eNB_Trace=='yes': + if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')): + pcapfile_prefix="enb_" + else: + pcapfile_prefix="gnb_" + mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) + result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) + if result is not None: + eth_interface = result.group('eth_interface') + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap' + mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5) + mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' 2>&1 &','\$', 5) + mySSH.close() + + + # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None): @@ -382,13 +422,13 @@ class RANManagement(): # do not reset board twice in IF4.5 case result = re.search('^rru|^enb|^du.band', str(config_file)) if result is not None: - mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60) + mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) result = re.search('type: b200', mySSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware - mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60) + mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) # Make a copy and adapt to EPC / eNB IP addresses mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) localMmeIpAddr = EPC.MmeIPAddress @@ -465,7 +505,7 @@ class RANManagement(): self.prematureExit = True return else: - mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4) + mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting|Started"', '\$', 4) if rruCheck: result = re.search('wait RUs', mySSH.getBefore()) else: @@ -573,6 +613,10 @@ class RANManagement(): mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(5) mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + #stopping tshark (valid if eNB and enabled in xml, will not harm otherwise) + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + time.sleep(1) mySSH.close() # If tracer options is on, stopping tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) @@ -591,6 +635,7 @@ class RANManagement(): mySSH.close() logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('killall --signal SIGKILL record', '\$', 5) mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5) enbLogFile = self.eNBLogFiles[int(self.eNB_instance)] raw_record_file = enbLogFile.replace('.log', '_record.raw') @@ -633,6 +678,9 @@ class RANManagement(): HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) else: HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + #display rt stats for gNB only + if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g': + HTML.CreateHtmlDataLogTable(self.datalog_rt_stats) self.eNBmbmsEnables[int(self.eNB_instance)] = False self.eNBstatuses[int(self.eNB_instance)] = -1 @@ -661,9 +709,11 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): @@ -714,12 +764,32 @@ class RANManagement(): NSA_RAPROC_PUSCH_check = 0 #dlsch and ulsch statistics (dictionary) dlsch_ulsch_stats = {} - #count "L1 thread not ready" msg - L1_thread_not_ready_cnt = 0 + #real time statistics (dictionary) + real_time_stats = {} #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 + #count "removing UE" msg + removing_ue = 0 + #NSA specific log markers + nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []} + #the datalog config file has to be loaded + datalog_rt_stats_file='datalog_rt_stats.yaml' + if (os.path.isfile(datalog_rt_stats_file)): + yaml_file=datalog_rt_stats_file + elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): + yaml_file='ci-scripts/'+datalog_rt_stats_file + else: + logging.error("Datalog RT stats yaml file cannot be found") + sys.exit("Datalog RT stats yaml file cannot be found") + + with open(yaml_file,'r') as f: + datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) + rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field + + line_cnt=0 #log file line counter for line in enb_log_file.readlines(): + line_cnt+=1 # Runtime statistics result = re.search('Run time:' ,str(line)) if result is not None: @@ -877,20 +947,36 @@ class RANManagement(): #keys below are the markers we are loooking for, loop over this keys list #everytime these markers are found in the log file, the previous ones are overwritten in the dict #eventually we record and print only the last occurence - keys = {'dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'} + keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'} for k in keys: result = re.search(k, line) if result is not None: #remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) - #count "L1 thread not ready" msg - result = re.search('\[PHY\]\s+L1_thread isn\'t ready', str(line)) - if result is not None: - L1_thread_not_ready_cnt += 1 + #real time statistics for gNB + for k in rt_keys: + result = re.search(k, line) + if result is not None: + #remove 1- all useless char before relevant info 2- trailing char + line=line.replace('[0m','') + tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex + if tmp!=None: #with ULULULUULULULLLL at the head of the line, we skip it + real_time_stats[k]=tmp.group(1) + #count "problem receiving samples" msg result = re.search('\[PHY\]\s+problem receiving samples', str(line)) if result is not None: - pb_receiving_samples_cnt += 1 + pb_receiving_samples_cnt += 1 + #count "Removing UE" msg + result = re.search('\[MAC\]\s+Removing UE', str(line)) + if result is not None: + removing_ue += 1 + + #nsa markers logging + for k in nsa_markers: + result = re.search(k, line) + if result is not None: + nsa_markers[k].append(line_cnt) enb_log_file.close() logging.debug(' File analysis completed') @@ -899,7 +985,7 @@ class RANManagement(): else: nodeB_prefix = 'g' - if self.air_interface[self.eNB_instance] == 'nr-softmodem': + if nodeB_prefix == 'g': if ulschReceiveOK > 0: statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') @@ -917,17 +1003,28 @@ class RANManagement(): statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS ' htmlMsg = statMsg+'\n' else: - statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL ' + statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant' htmlMsg = statMsg+'\n' logging.debug(statMsg) htmleNBFailureMsg += htmlMsg - #L1 thread not ready log - statMsg = '[PHY] L1 thread is not ready msg count = '+str(L1_thread_not_ready_cnt) + #problem receiving samples log + statMsg = '[PHY] problem receiving samples msg count = '+str(pb_receiving_samples_cnt) htmlMsg = statMsg+'\n' logging.debug(statMsg) htmleNBFailureMsg += htmlMsg - #problem receiving samples log - statMsg = '[PHY] problem receiving samples msg count = '+str(pb_receiving_samples_cnt) + #nsa markers + statMsg = 'logfile line count = ' + str(line_cnt) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + if len(nsa_markers['SgNBReleaseRequestAcknowledge'])!=0: + statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequestAcknowledge'][0]) + else: + statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + statMsg = 'FAILURE = ' + str(len(nsa_markers['FAILURE'])) + ' occurences' htmlMsg = statMsg+'\n' logging.debug(statMsg) htmleNBFailureMsg += htmlMsg @@ -940,6 +1037,52 @@ class RANManagement(): logging.debug(dlsch_ulsch_stats[key]) htmleNBFailureMsg += statMsg + #real time statistics + datalog_rt_stats['Data']={} + if len(real_time_stats)!=0: #check if dictionary is not empty + for k in real_time_stats: + tmp=re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+\d+;\s+(?P<max>\d+\.\d+) us;',real_time_stats[k]) + if tmp is not None: + metric=tmp.group('metric') + avg=float(tmp.group('avg')) + max=float(tmp.group('max')) + datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])] + #once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward + self.datalog_rt_stats=datalog_rt_stats + #check if there is a fail => will render the test as failed + for k in datalog_rt_stats['Data']: + if float(datalog_rt_stats['Data'][k][2])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold + #setting prematureExit is ok although not the best option + self.prematureExit=True + else: + statMsg = 'No real time stats found in the log file\n' + logging.debug('No real time stats found in the log file') + htmleNBFailureMsg += statMsg + + else: + #Removing UE log + statMsg = '[MAC] Removing UE msg count = '+str(removing_ue) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + #nsa markers + statMsg = 'logfile line count = ' + str(line_cnt) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + if len(nsa_markers['SgNBReleaseRequest'])!=0: + statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequest'][0]) + else: + statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences' + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + statMsg = 'scgFailureInformationNR-r15 = ' + str(len(nsa_markers['scgFailureInformationNR-r15'])) + ' occurences' + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg + + if uciStatMsgCount > 0: statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')