diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 66d12eed33a7c5172649aff6cde5a95bd449fda6..1e4933838b907817f767386dacbe2b10c2799694 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -33,10 +33,9 @@ def buildStageStatus = true // Name of the test stage def testStageName = params.pipelineTestStageName -// Name of the phone resource +// Name of the phone/server resource def ciSmartPhonesResource1 = params.SmartPhonesResource1 def ciSmartPhonesResource2 = params.SmartPhonesResource2 -def ciSmartPhonesResource3 = params.SmartPhonesResource3 // Global Parameters. Normally they should be populated when the master job // triggers the slave job with parameters @@ -53,7 +52,7 @@ pipeline { options { disableConcurrentBuilds() ansiColor('xterm') - lock(extra: [[resource: ciSmartPhonesResource2],[resource: ciSmartPhonesResource3]], resource: ciSmartPhonesResource1) + lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1) } stages { stage("Build Init") { @@ -85,9 +84,6 @@ pipeline { if (params.SmartPhonesResource2 == null) { allParametersPresent = false } - if (params.SmartPhonesResource3== null) { - allParametersPresent = false - } // 1st eNB parameters if (params.eNB_IPAddress == null) { allParametersPresent = false diff --git a/ci-scripts/Jenkinsfile-trig-nsa b/ci-scripts/Jenkinsfile-trig-nsa index b9172225cab88a95227945430f45b8b6d6cb3c95..8f623ffb057cbec3cdcf312f7c8f9c9ae1c9b734 100644 --- a/ci-scripts/Jenkinsfile-trig-nsa +++ b/ci-scripts/Jenkinsfile-trig-nsa @@ -36,7 +36,7 @@ pipeline { stage ("Launcher") { steps { script { - //retrieve MR that are opened nd with tag READY_TO_BE_MERGED + //retrieve MR that are opened nd with tag NSA MR_LIST= sh returnStdout: true, script: 'curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100&labels=NSA" | jq ".[].iid" || true ' echo "List of selected MR:\n${MR_LIST}" def MR_ARRAY = MR_LIST.split('\n') @@ -48,7 +48,7 @@ pipeline { COMMIT_ID=COMMIT_ID.trim() echo "Testing NSA on : ${MR} ${SRC_BRANCH} ${COMMIT_ID}" //calling NSA sub job - build job: "RAN-CI-NSA-B210-N310-ModuleUE", wait : false, propagate : false, parameters: [ + build job: "RAN-NSA-Mini-Module", wait : false, propagate : false, parameters: [ string(name: 'eNB_MR', value: String.valueOf(MR)), string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), @@ -56,13 +56,13 @@ pipeline { booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) ] //calling Benetel sub job - build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [ - string(name: 'eNB_MR', value: String.valueOf(MR)), - string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), - string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)), - booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) - ] + //build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [ + // string(name: 'eNB_MR', value: String.valueOf(MR)), + // string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), + // string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), + // string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)), + // booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) + //] } } } diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index c778b1cad3cf77952b0e15083ab2f1b9c32b2876..67331fdfb27cca122cfccb39bd53ca5194fbc4f9 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -8,7 +8,7 @@ idefix: WakeupScript : ci_ctl_qtel.py /dev/ttyUSB2 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB2 detach PLMN : 22201 - UENetwork : enxb22a941dbf00 + UENetwork : wwan0 HostIPAddress : 192.168.18.188 HostUsername : oaicicd HostPassword : oaicicd diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index abfa8c1106ed68c98cf71a3abfeb6143f27e2c3e..f93cb9f0365ff463fd5d103198fd63b7490ed224 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -37,7 +37,7 @@ import time import re import subprocess - +from datetime import datetime class Module_UE: @@ -101,7 +101,7 @@ class Module_UE: def GetModuleIPAddress(self): HOST=self.HostIPAddress response= [] - tentative = 10 + tentative = 3 while (len(response)==0) and (tentative>0): COMMAND="ip a show dev " + self.UENetwork + " | grep inet | grep " + self.UENetwork logging.debug(COMMAND) @@ -126,7 +126,37 @@ class Module_UE: logging.debug('\u001B[1;37;41m Module IP Address Not Found! \u001B[0m') return -1 + def EnableTrace(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + #delete old artifacts + mySSH.command('echo ' + self.HostPassword + ' | sudo -S rm -rf ci_qlog','\$',5) + #start Trace + mySSH.command('echo $USER; nohup sudo -E QLog/QLog -s ci_qlog -f NR5G.cfg &','\$', 5) + mySSH.close() + + def DisableTrace(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal=SIGINT *QLog*', '\$',5) + mySSH.close() + def DisableCM(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal SIGKILL *'+self.Process['Name']+'*', '\$', 5) + mySSH.close() + def LogCollect(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + #archive qlog to /opt/ci_qlogs with datetime suffix + now=datetime.now() + now_string = now.strftime("%Y%m%d-%H%M") + source='ci_qlog' + destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip' + mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10) + mySSH.close() + return destination diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 7202e7c881631f597b1ceec1c84cc629022b0ef0..809af4ebec7e6337da470825b67f93c4bc3ce143 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -388,18 +388,23 @@ class OaiCiTest(): Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) is_module=Module_UE.CheckCMProcess() if is_module: + Module_UE.EnableTrace() + time.sleep(5) Module_UE.Command("wup") + logging.debug("Waiting for IP address to be assigned") + time.sleep(20) + logging.debug("Retrieve IP address") status=Module_UE.GetModuleIPAddress() if status==0: HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) else: #status==-1 failed to retrieve IP address HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return - def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE): + def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -623,7 +628,7 @@ class OaiCiTest(): HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced' HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE') logging.error('\033[91mInitialize OAI UE Failed! \033[0m') - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def checkDevTTYisUnlocked(self): SSH = sshconnection.SSHConnection() @@ -701,7 +706,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) self.checkDevTTYisUnlocked() - def AttachCatM(self,HTML,RAN,COTS_UE,EPC): + def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -774,9 +779,9 @@ class OaiCiTest(): html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>' html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) - def PingCatM(self,HTML,RAN,EPC,COTS_UE): + def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -785,7 +790,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return try: statusQueue = SimpleQueue() @@ -806,7 +811,7 @@ class OaiCiTest(): moduleIPAddr = result.group('ipaddr') else: HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_time = re.findall("-c (\d+)",str(self.ping_args)) device_id = 'catm' @@ -870,7 +875,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) except: os.kill(os.getppid(),signal.SIGUSR1) @@ -962,7 +967,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def AttachUE(self,HTML,RAN,EPC,COTS_UE): + def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -972,7 +977,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return multi_jobs = [] status_queue = SimpleQueue() @@ -991,7 +996,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: attach_status = True @@ -1020,7 +1025,7 @@ class OaiCiTest(): time.sleep(5) else: HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def DetachUE_common(self, device_id, idx,COTS_UE): try: @@ -1054,7 +1059,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return multi_jobs = [] cnt = 0 @@ -1079,7 +1084,10 @@ class OaiCiTest(): else:#if an ID is specified, it is a module from the yaml infrastructure file Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) Module_UE.Command("detach") - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + Module_UE.DisableTrace() + Module_UE.DisableCM() + archive_destination=Module_UE.LogCollect() + HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK) @@ -1329,7 +1337,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE): + def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -1376,7 +1384,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: check_status = True html_queue = SimpleQueue() @@ -1392,7 +1400,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def GetAllUEIPAddresses(self): SSH = sshconnection.SSHConnection() @@ -1612,14 +1620,14 @@ class OaiCiTest(): html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - def PingNoS1(self,HTML,RAN,EPC,COTS_UE): + def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH=sshconnection.SSHConnection() check_eNB = True check_OAI_UE = True pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args)) if ping_from_eNB is not None: @@ -1708,7 +1716,7 @@ class OaiCiTest(): def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.PingNoS1(HTML,RAN,EPC,COTS_UE) + self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -1721,7 +1729,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return if self.ue_id=="": @@ -1729,7 +1737,7 @@ class OaiCiTest(): ueIpStatus = self.GetAllUEIPAddresses() if (ueIpStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: self.UEIPAddresses=[] @@ -1756,7 +1764,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: ping_status = True html_queue = SimpleQueue() @@ -1773,7 +1781,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def Iperf_ComputeTime(self): result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args)) @@ -2467,7 +2475,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def IperfNoS1(self,HTML,RAN,EPC,COTS_UE): + def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH = sshconnection.SSHConnection() if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '': HELP.GenericHelp(CONST.Version) @@ -2477,7 +2485,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return server_on_enb = re.search('-R', str(self.iperf_args)) if server_on_enb is not None: @@ -2576,12 +2584,12 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.IperfNoS1(HTML,RAN,EPC,COTS_UE) + self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -2594,14 +2602,14 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return if self.ue_id=="":#is not a module, follow legacy code ueIpStatus = self.GetAllUEIPAddresses() if (ueIpStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: #is a module self.UEIPAddresses=[] @@ -2646,7 +2654,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfaUE) else: iperf_status = True iperf_noperf = False @@ -2668,7 +2676,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def CheckProcessExist(self, check_eNB, check_OAI_UE,RAN,EPC): multi_jobs = [] @@ -3088,7 +3096,7 @@ class OaiCiTest(): job.join() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC): + def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC, InfraUE): SSH = sshconnection.SSHConnection() SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword) SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) @@ -3129,11 +3137,11 @@ class OaiCiTest(): # Not an error then if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m') HTML.htmlUEFailureMsg='<b>' + ueAction + ' Completed</b>\n' + HTML.htmlUEFailureMsg @@ -3142,7 +3150,14 @@ class OaiCiTest(): else: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC): + def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE): + if self.ue_id!='': + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.Command("detach") + Module_UE.DisableTrace() + Module_UE.DisableCM() + archive_destination=Module_UE.LogCollect() + HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK) if (self.ADBIPAddress != 'none'): self.testCase_id = 'AUTO-KILL-UE' HTML.testCase_id=self.testCase_id @@ -3156,7 +3171,7 @@ class OaiCiTest(): self.desc = 'Automatic Termination of OAI-UE' HTML.desc='Automatic Termination of OAI-UE' self.ShowTestID() - self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) if (RAN.Initialize_eNB_args != ''): self.testCase_id = 'AUTO-KILL-RAN' HTML.testCase_id=self.testCase_id diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3f5b57e01616d53de7773ed089d84279d31b2063..cbd323366b433981a326b522c8812ce46186b537 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -152,6 +152,7 @@ def GetParametersFromXML(action): RAN.eNB_serverId[RAN.eNB_instance]=eNB_serverId elif action == 'Initialize_eNB': + RAN.eNB_Trace=test.findtext('eNB_Trace') RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') eNB_instance=test.findtext('eNB_instance') if (eNB_instance is None): @@ -474,7 +475,7 @@ elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif re.match('^TerminateHSS$', mode, re.IGNORECASE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -717,7 +718,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Terminate_UE': CiTestObj.TerminateUE(HTML,COTS_UE) elif action == 'Attach_UE': - CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Detach_UE': CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'DataDisable_UE': @@ -725,23 +726,23 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'DataEnable_UE': CiTestObj.DataEnableUE(HTML) elif action == 'CheckStatusUE': - CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Build_OAI_UE': CiTestObj.BuildOAIUE(HTML) elif action == 'Initialize_OAI_UE': - CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Terminate_OAI_UE': - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Initialize_CatM_module': CiTestObj.InitializeCatM(HTML) elif action == 'Terminate_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Attach_CatM_module': - CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC) + CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Detach_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Ping_CatM_module': - CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC) + CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE) elif action == 'Ping': CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE) elif action == 'Iperf': diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 83945b3c17c387856f7dacaf6691f5627a034ec1..470572f2ff50e1470ad735a5c6cb6348563d2bfa 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -93,6 +93,7 @@ class RANManagement(): self.epcPcapFile = '' self.runtime_stats= '' self.datalog_rt_stats={} + self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization @@ -344,6 +345,22 @@ class RANManagement(): HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus) HTML.CreateHtmlTabFooter(False) sys.exit(1) + #Get pcap on S1 and X2 eNB interface, if enabled in the xml + #will not work for gNB at this stage + if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')) and self.eNB_Trace=='yes': + mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) + result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) + if result is not None: + eth_interface = result.group('eth_interface') + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + pcapfile = 'enb_' + self.testCase_id + '_s1x2log.pcap' + mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5) + mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' 2>&1 &','\$', 5) + mySSH.close() + + + # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None): @@ -579,6 +596,10 @@ class RANManagement(): mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(5) mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + #stopping tshark (valid if eNB and enabled in xml, will not harm otherwise) + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + time.sleep(1) mySSH.close() # If tracer options is on, stopping tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) @@ -597,6 +618,7 @@ class RANManagement(): mySSH.close() logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('killall --signal SIGKILL record', '\$', 5) mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5) enbLogFile = self.eNBLogFiles[int(self.eNB_instance)] raw_record_file = enbLogFile.replace('.log', '_record.raw') @@ -650,9 +672,10 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*_s1x2log.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): @@ -707,6 +730,8 @@ class RANManagement(): real_time_stats = {} #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 + #count "removing UE" msg + removing_ue = 0 #NSA specific log markers nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []} @@ -903,6 +928,10 @@ class RANManagement(): result = re.search('\[PHY\]\s+problem receiving samples', str(line)) if result is not None: pb_receiving_samples_cnt += 1 + #count "Removing UE" msg + result = re.search('\[MAC\]\s+Removing UE', str(line)) + if result is not None: + removing_ue += 1 #nsa markers logging for k in nsa_markers: @@ -992,6 +1021,11 @@ class RANManagement(): htmleNBFailureMsg += statMsg else: + #Removing UE log + statMsg = '[MAC] Removing UE msg count = '+str(removing_ue) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg #nsa markers statMsg = 'logfile line count = ' + str(line_cnt) htmlMsg = statMsg+'\n' diff --git a/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml b/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml index 64c35171669e48444e5f67beda0690ad03103b05..bfaaceaa2d91b54925bdbd914032cbcdae5f7bed 100644 --- a/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml @@ -32,7 +32,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/enb_usrp210_band13_build.xml b/ci-scripts/xml_files/enb_usrp210_band13_build.xml index 888a1230ae23aabc95f9ec8dd25a8681eeacde03..53066573c89a6dee1aeaae1eeebd49243e608cce 100644 --- a/ci-scripts/xml_files/enb_usrp210_band13_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band13_build.xml @@ -33,7 +33,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/enb_usrp210_band40_build.xml b/ci-scripts/xml_files/enb_usrp210_band40_build.xml index 64c35171669e48444e5f67beda0690ad03103b05..bfaaceaa2d91b54925bdbd914032cbcdae5f7bed 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_build.xml @@ -32,7 +32,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/enb_usrp210_band7_build.xml b/ci-scripts/xml_files/enb_usrp210_band7_build.xml index 64c35171669e48444e5f67beda0690ad03103b05..bfaaceaa2d91b54925bdbd914032cbcdae5f7bed 100644 --- a/ci-scripts/xml_files/enb_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band7_build.xml @@ -32,7 +32,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 6c2bd6bd261ecdb1051797b46d9d756e9c9fd2a4..fbaa32aea7795404ebef80b8b8dd9bd818d42b4b 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -24,21 +24,21 @@ <htmlTabRef>TEST-NSA-FR1-TM1</htmlTabRef> <htmlTabName>NSA Ping DL UL with QUECTEL</htmlTabName> <htmlTabIcon>tasks</htmlTabIcon> - <repeatCount>3</repeatCount> + <repeatCount>1</repeatCount> <TestCaseRequestedList> 030000 040000 000002 010000 + 000001 050000 050001 000001 070000 000001 - 070000 + 070001 000001 010002 - 000001 080001 080000 </TestCaseRequestedList> @@ -65,13 +65,14 @@ <eNB_instance>0</eNB_instance> <eNB_serverId>0</eNB_serverId> <air_interface>lte</air_interface> + <eNB_Trace>yes</eNB_Trace> </testCase> <testCase id="040000"> <class>Initialize_eNB</class> <desc>Initialize gNB</desc> - <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpn310.conf -q</Initialize_eNB_args> + <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf -E -q</Initialize_eNB_args> <eNB_instance>1</eNB_instance> <eNB_serverId>1</eNB_serverId> <air_interface>nr</air_interface> diff --git a/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml b/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml index ab221c918d50ac74a7832b3b6b7f2ba9933a6d1b..d383dbb61b50e8b019ff3e4d31747c50f1485057 100644 --- a/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml +++ b/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml @@ -33,7 +33,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP -- Ethernet Fronthaul)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml b/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml index ab221c918d50ac74a7832b3b6b7f2ba9933a6d1b..d383dbb61b50e8b019ff3e4d31747c50f1485057 100644 --- a/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml @@ -33,7 +33,7 @@ <testCase id="010101"> <class>Build_eNB</class> <desc>Build eNB (USRP -- Ethernet Fronthaul)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> </testCase> </testCaseList> diff --git a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml index c5a912a33d9a112c4ea4b0a07ee5dd5c406fcdbd..0ac6bfcc0d4f0c1cc7295f19d8a4adbabeb6cbc8 100644 --- a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml +++ b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml @@ -42,7 +42,7 @@ <testCase id="010102"> <class>Build_eNB</class> <desc>Build Slave eNB (USRP)</desc> - <Build_eNB_args>-w USRP -c --eNB</Build_eNB_args> + <Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args> <eNB_instance>1</eNB_instance> <eNB_serverId>1</eNB_serverId> </testCase>