fix(ci): display state of container when failure, better tshark filtering,...

fix(ci): display state of container when failure, better tshark filtering, auto-undeploy if deploy went wrong Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

fix(ci): display state of container when failure, better tshark filtering,...
0b7be746 · Raphael Defosseux · 454aae1d · 0b7be746 · 0b7be746 · 0b7be746
Commit 0b7be746 authored 3 years ago by Raphael Defosseux
--- a/ci-scripts/cls_containerize.py
+++ b/ci-scripts/cls_containerize.py
@@ -636,7 +636,7 @@ class Containerize():
 				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
 		logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')

-	def DeployGenObject(self, HTML):
+	def DeployGenObject(self, HTML, RAN, UE):
 		self.exitStatus = 0
 		logging.info('\u001B[1m Checking Services to deploy\u001B[0m')
 		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services'
@@ -677,14 +677,17 @@ class Containerize():
 		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a'
 		count = 0
 		healthy = 0
+		containerStatus = []
 		while (count < 10):
 			count += 1
+			containerStatus = []
 			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
 			healthy = 0
 			for state in deployStatus.split('\n'):
 				res = re.search('Up \(healthy\)', state)
 				if res is not None:
 					healthy += 1
+				containerStatus.append(state)
 			if healthy == self.nb_healthy[0]:
 				count = 100
 			else:
@@ -694,27 +697,39 @@ class Containerize():
 			if self.tsharkStarted == False:
 				logging.debug('Starting tshark on public network')
 				self.CaptureOnDockerNetworks()
-				self.tsharkStarted = True
 			HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
 			logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
 		else:
-			self.exitStatus = 1
 			HTML.CreateHtmlTestRow('Could not deploy in time', 'KO', CONST.ALL_PROCESSES_OK)
+			for cState in containerStatus:
+				logging.debug(cState)
 			logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
+			HTML.testCase_id = 'AUTO-UNDEPLOY'
+			UE.testCase_id = 'AUTO-UNDEPLOY'
+			HTML.desc = 'Automatic Undeployment'
+			UE.desc = 'Automatic Undeployment'
+			UE.ShowTestID()
+			self.UndeployGenObject(HTML, RAN, UE)
+			self.exitStatus = 1

 	def CaptureOnDockerNetworks(self):
 		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
 		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
-		cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com and not port 2152"'
+		if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None:
+			cmd = 'sudo nohup tshark -f "(host 192.168.61.11 and icmp) or (not host 192.168.61.11 and not host 192.168.61.30 and not arp and not port 53 and not port 2152)"'
+		elif re.search('5g.*rfsimulator', self.yamlPath[0]) is not None:
+			cmd = 'sudo nohup tshark -f "(host 192.168.72.135 and icmp) or (not host 192.168.72.135 and not host 192.168.71.150 and not arp and not port 53 and not port 2152 and not port 2153)"'
+		else:
+			return
 		for name in networkNames.split('\n'):
-			res = re.search('rfsim', name)
-			if res is not None:
+			if re.search('rfsim', name) is not None:
 				cmd += ' -i ' + name
 		cmd += ' -w /tmp/capture_'
 		ymlPath = self.yamlPath[0].split('/')
 		cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &'
 		logging.debug(cmd)
 		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
+		self.tsharkStarted = True

 	def UndeployGenObject(self, HTML, RAN, UE):
 		self.exitStatus = 0

--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -910,7 +910,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'Deploy_Run_PhySim':
 					PHYSIM.Deploy_PhySim(HTML, RAN)
 				elif action == 'DeployGenObject':
-					CONTAINERS.DeployGenObject(HTML)
+					CONTAINERS.DeployGenObject(HTML, RAN, CiTestObj)
 					if CONTAINERS.exitStatus==1:
 						RAN.prematureExit = True
 				elif action == 'UndeployGenObject':

--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
@@ -80,7 +80,7 @@

        <testCase id="000014">
                <class>DeployGenObject</class>
-                <desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc>
+                <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
                <yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz</yaml_path>
                <services>oai_ue0</services>
                <nb_healthy>8</nb_healthy>

--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
@@ -60,7 +60,7 @@

        <testCase id="000014">
                <class>DeployGenObject</class>
-                <desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc>
+                <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
                <yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz_noS1</yaml_path>
                <services>oai_ue0</services>
                <nb_healthy>2</nb_healthy>

--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
@@ -80,7 +80,7 @@

        <testCase id="000014">
                <class>DeployGenObject</class>
-                <desc>Deploy OAI 4G NR-UE RF sim (FDD 10MHz)</desc>
+                <desc>Deploy OAI 4G LTE-UE RF sim (FDD 10MHz)</desc>
                <yaml_path>yaml_files/4g_rfsimulator_fdd_10MHz</yaml_path>
                <services>oai_ue0</services>
                <nb_healthy>8</nb_healthy>

--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
@@ -80,7 +80,7 @@

        <testCase id="000014">
                <class>DeployGenObject</class>
-                <desc>Deploy OAI 4G NR-UE RF sim (FDD 20MHz)</desc>
+                <desc>Deploy OAI 4G LTE-UE RF sim (FDD 20MHz)</desc>
                <yaml_path>yaml_files/4g_rfsimulator_fdd_20MHz</yaml_path>
                <services>oai_ue0</services>
                <nb_healthy>8</nb_healthy>

--- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
@@ -81,7 +81,7 @@

        <testCase id="000014">
                <class>DeployGenObject</class>
-                <desc>Deploy OAI 4G NR-UE RF sim (TDD 05MHz)</desc>
+                <desc>Deploy OAI 4G LTE-UE RF sim (TDD 05MHz)</desc>
                <yaml_path>yaml_files/4g_rfsimulator_tdd_05MHz</yaml_path>
                <services>oai_ue0</services>
                <nb_healthy>8</nb_healthy>

--- a/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
+++ b/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
@@ -241,12 +241,12 @@ services:
            NSSAI_SD1: 112233
            AMF_IP_ADDRESS: 192.168.71.132
            GNB_NGA_IF_NAME: eth0
-            GNB_NGA_IP_ADDRESS: 192.168.71.136
+            GNB_NGA_IP_ADDRESS: 192.168.71.140
            GNB_NGU_IF_NAME: eth0
-            GNB_NGU_IP_ADDRESS: 192.168.71.136
+            GNB_NGU_IP_ADDRESS: 192.168.71.140
            F1_IF_NAME: eth0
-            F1_CU_IP_ADDRESS: 192.168.71.136
-            F1_DU_IP_ADDRESS: 192.168.71.138
+            F1_CU_IP_ADDRESS: 192.168.71.140
+            F1_DU_IP_ADDRESS: 192.168.71.142
            F1_CU_D_PORT: 2153
            F1_DU_D_PORT: 2153
            USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -254,7 +254,7 @@ services:
            - oai-ext-dn
        networks:
            public_net:
-                ipv4_address: 192.168.71.136
+                ipv4_address: 192.168.71.140
        healthcheck:
            test: /bin/bash -c "pgrep nr-softmodem"
            interval: 10s
@@ -277,12 +277,12 @@ services:
            NSSAI_SD1: 112233
            AMF_IP_ADDRESS: 192.168.71.132
            GNB_NGA_IF_NAME: eth0
-            GNB_NGA_IP_ADDRESS: 192.168.71.138
+            GNB_NGA_IP_ADDRESS: 192.168.71.142
            GNB_NGU_IF_NAME: eth0
-            GNB_NGU_IP_ADDRESS: 192.168.71.138
+            GNB_NGU_IP_ADDRESS: 192.168.71.142
            F1_IF_NAME: eth0
-            F1_CU_IP_ADDRESS: 192.168.71.136
-            F1_DU_IP_ADDRESS: 192.168.71.138
+            F1_CU_IP_ADDRESS: 192.168.71.140
+            F1_DU_IP_ADDRESS: 192.168.71.142
            F1_CU_D_PORT: 2153
            F1_DU_D_PORT: 2153
            USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -290,7 +290,7 @@ services:
            - oai-cu
        networks:
            public_net:
-                ipv4_address: 192.168.71.138
+                ipv4_address: 192.168.71.142
        healthcheck:
            test: /bin/bash -c "pgrep nr-softmodem"
            interval: 10s
@@ -301,7 +301,7 @@ services:
        privileged: true
        container_name: rfsim5g-oai-nr-ue
        environment: 
-            RFSIMULATOR: 192.168.71.138
+            RFSIMULATOR: 192.168.71.142
            FULL_IMSI: '208990100001100'
            FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
            OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -313,7 +313,7 @@ services:
            - oai-du
        networks:
            public_net:
-                ipv4_address: 192.168.71.137
+                ipv4_address: 192.168.71.150
        healthcheck:
            test: /bin/bash -c "pgrep nr-uesoftmodem"
            interval: 10s

--- a/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
+++ b/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
@@ -241,15 +241,15 @@ services:
            NSSAI_SD1: 112233
            AMF_IP_ADDRESS: 192.168.71.132
            GNB_NGA_IF_NAME: eth0
-            GNB_NGA_IP_ADDRESS: 192.168.71.136
+            GNB_NGA_IP_ADDRESS: 192.168.71.140
            GNB_NGU_IF_NAME: eth0
-            GNB_NGU_IP_ADDRESS: 192.168.71.136
+            GNB_NGU_IP_ADDRESS: 192.168.71.140
            USE_ADDITIONAL_OPTIONS: --sa -E --rfsim --log_config.global_log_options level,nocolor,time
        depends_on:
            - oai-ext-dn
        networks:
            public_net:
-                ipv4_address: 192.168.71.136
+                ipv4_address: 192.168.71.140
        healthcheck:
            test: /bin/bash -c "pgrep nr-softmodem"
            interval: 10s
@@ -260,7 +260,7 @@ services:
        privileged: true
        container_name: rfsim5g-oai-nr-ue
        environment: 
-            RFSIMULATOR: 192.168.71.136
+            RFSIMULATOR: 192.168.71.140
            FULL_IMSI: '208990100001100'
            FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
            OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -272,7 +272,7 @@ services:
            - oai-gnb
        networks:
            public_net:
-                ipv4_address: 192.168.71.137
+                ipv4_address: 192.168.71.150
        healthcheck:
            test: /bin/bash -c "pgrep nr-uesoftmodem"
            interval: 10s