From 0b7be7466e01816a2f79cfb21f5e99faa08b61ff Mon Sep 17 00:00:00 2001
From: Raphael Defosseux <raphael.defosseux@eurecom.fr>
Date: Thu, 10 Feb 2022 15:33:08 +0100
Subject: [PATCH] fix(ci): display state of container when failure, better
tshark filtering, auto-undeploy if deploy went wrong
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
---
ci-scripts/cls_containerize.py | 27 ++++++++++++++-----
ci-scripts/main.py | 2 +-
.../container_4g_rfsim_fdd_05MHz.xml | 2 +-
.../container_4g_rfsim_fdd_05MHz_noS1.xml | 2 +-
.../container_4g_rfsim_fdd_10MHz.xml | 2 +-
.../container_4g_rfsim_fdd_20MHz.xml | 2 +-
.../container_4g_rfsim_tdd_05MHz.xml | 2 +-
.../5g_f1_rfsimulator/docker-compose.yaml | 24 ++++++++---------
.../5g_rfsimulator/docker-compose.yaml | 10 +++----
9 files changed, 44 insertions(+), 29 deletions(-)
diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py
index e6065139351..eca75617a6d 100644
--- a/ci-scripts/cls_containerize.py
+++ b/ci-scripts/cls_containerize.py
@@ -636,7 +636,7 @@ class Containerize():
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
- def DeployGenObject(self, HTML):
+ def DeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0
logging.info('\u001B[1m Checking Services to deploy\u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services'
@@ -677,14 +677,17 @@ class Containerize():
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a'
count = 0
healthy = 0
+ containerStatus = []
while (count < 10):
count += 1
+ containerStatus = []
deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
healthy = 0
for state in deployStatus.split('\n'):
res = re.search('Up \(healthy\)', state)
if res is not None:
healthy += 1
+ containerStatus.append(state)
if healthy == self.nb_healthy[0]:
count = 100
else:
@@ -694,27 +697,39 @@ class Containerize():
if self.tsharkStarted == False:
logging.debug('Starting tshark on public network')
self.CaptureOnDockerNetworks()
- self.tsharkStarted = True
HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
else:
- self.exitStatus = 1
HTML.CreateHtmlTestRow('Could not deploy in time', 'KO', CONST.ALL_PROCESSES_OK)
+ for cState in containerStatus:
+ logging.debug(cState)
logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
+ HTML.testCase_id = 'AUTO-UNDEPLOY'
+ UE.testCase_id = 'AUTO-UNDEPLOY'
+ HTML.desc = 'Automatic Undeployment'
+ UE.desc = 'Automatic Undeployment'
+ UE.ShowTestID()
+ self.UndeployGenObject(HTML, RAN, UE)
+ self.exitStatus = 1
def CaptureOnDockerNetworks(self):
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
- cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com and not port 2152"'
+ if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None:
+ cmd = 'sudo nohup tshark -f "(host 192.168.61.11 and icmp) or (not host 192.168.61.11 and not host 192.168.61.30 and not arp and not port 53 and not port 2152)"'
+ elif re.search('5g.*rfsimulator', self.yamlPath[0]) is not None:
+ cmd = 'sudo nohup tshark -f "(host 192.168.72.135 and icmp) or (not host 192.168.72.135 and not host 192.168.71.150 and not arp and not port 53 and not port 2152 and not port 2153)"'
+ else:
+ return
for name in networkNames.split('\n'):
- res = re.search('rfsim', name)
- if res is not None:
+ if re.search('rfsim', name) is not None:
cmd += ' -i ' + name
cmd += ' -w /tmp/capture_'
ymlPath = self.yamlPath[0].split('/')
cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &'
logging.debug(cmd)
networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
+ self.tsharkStarted = True
def UndeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0
diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index 43772fb655f..5357a90a3d6 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -910,7 +910,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
elif action == 'Deploy_Run_PhySim':
PHYSIM.Deploy_PhySim(HTML, RAN)
elif action == 'DeployGenObject':
- CONTAINERS.DeployGenObject(HTML)
+ CONTAINERS.DeployGenObject(HTML, RAN, CiTestObj)
if CONTAINERS.exitStatus==1:
RAN.prematureExit = True
elif action == 'UndeployGenObject':
diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
index fd254f18567..55f60b8307f 100644
--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
@@ -80,7 +80,7 @@
<testCase id="000014">
<class>DeployGenObject</class>
- <desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc>
+ <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz</yaml_path>
<services>oai_ue0</services>
<nb_healthy>8</nb_healthy>
diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
index 2ff9461b936..07ea358ad53 100644
--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
@@ -60,7 +60,7 @@
<testCase id="000014">
<class>DeployGenObject</class>
- <desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc>
+ <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz_noS1</yaml_path>
<services>oai_ue0</services>
<nb_healthy>2</nb_healthy>
diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
index f00e277c36f..d5152c77796 100644
--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
@@ -80,7 +80,7 @@
<testCase id="000014">
<class>DeployGenObject</class>
- <desc>Deploy OAI 4G NR-UE RF sim (FDD 10MHz)</desc>
+ <desc>Deploy OAI 4G LTE-UE RF sim (FDD 10MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_10MHz</yaml_path>
<services>oai_ue0</services>
<nb_healthy>8</nb_healthy>
diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
index 52e1ac7de18..0c5b9d43bdf 100644
--- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
@@ -80,7 +80,7 @@
<testCase id="000014">
<class>DeployGenObject</class>
- <desc>Deploy OAI 4G NR-UE RF sim (FDD 20MHz)</desc>
+ <desc>Deploy OAI 4G LTE-UE RF sim (FDD 20MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_20MHz</yaml_path>
<services>oai_ue0</services>
<nb_healthy>8</nb_healthy>
diff --git a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
index cdaee430efc..aa70db1fed4 100644
--- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
+++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
@@ -81,7 +81,7 @@
<testCase id="000014">
<class>DeployGenObject</class>
- <desc>Deploy OAI 4G NR-UE RF sim (TDD 05MHz)</desc>
+ <desc>Deploy OAI 4G LTE-UE RF sim (TDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_tdd_05MHz</yaml_path>
<services>oai_ue0</services>
<nb_healthy>8</nb_healthy>
diff --git a/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml b/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
index 6a39f2147ea..74a23905f0a 100644
--- a/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
+++ b/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
@@ -241,12 +241,12 @@ services:
NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0
- GNB_NGA_IP_ADDRESS: 192.168.71.136
+ GNB_NGA_IP_ADDRESS: 192.168.71.140
GNB_NGU_IF_NAME: eth0
- GNB_NGU_IP_ADDRESS: 192.168.71.136
+ GNB_NGU_IP_ADDRESS: 192.168.71.140
F1_IF_NAME: eth0
- F1_CU_IP_ADDRESS: 192.168.71.136
- F1_DU_IP_ADDRESS: 192.168.71.138
+ F1_CU_IP_ADDRESS: 192.168.71.140
+ F1_DU_IP_ADDRESS: 192.168.71.142
F1_CU_D_PORT: 2153
F1_DU_D_PORT: 2153
USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -254,7 +254,7 @@ services:
- oai-ext-dn
networks:
public_net:
- ipv4_address: 192.168.71.136
+ ipv4_address: 192.168.71.140
healthcheck:
test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s
@@ -277,12 +277,12 @@ services:
NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0
- GNB_NGA_IP_ADDRESS: 192.168.71.138
+ GNB_NGA_IP_ADDRESS: 192.168.71.142
GNB_NGU_IF_NAME: eth0
- GNB_NGU_IP_ADDRESS: 192.168.71.138
+ GNB_NGU_IP_ADDRESS: 192.168.71.142
F1_IF_NAME: eth0
- F1_CU_IP_ADDRESS: 192.168.71.136
- F1_DU_IP_ADDRESS: 192.168.71.138
+ F1_CU_IP_ADDRESS: 192.168.71.140
+ F1_DU_IP_ADDRESS: 192.168.71.142
F1_CU_D_PORT: 2153
F1_DU_D_PORT: 2153
USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -290,7 +290,7 @@ services:
- oai-cu
networks:
public_net:
- ipv4_address: 192.168.71.138
+ ipv4_address: 192.168.71.142
healthcheck:
test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s
@@ -301,7 +301,7 @@ services:
privileged: true
container_name: rfsim5g-oai-nr-ue
environment:
- RFSIMULATOR: 192.168.71.138
+ RFSIMULATOR: 192.168.71.142
FULL_IMSI: '208990100001100'
FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -313,7 +313,7 @@ services:
- oai-du
networks:
public_net:
- ipv4_address: 192.168.71.137
+ ipv4_address: 192.168.71.150
healthcheck:
test: /bin/bash -c "pgrep nr-uesoftmodem"
interval: 10s
diff --git a/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
index f0476b0a1ef..808be7d83d5 100644
--- a/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
+++ b/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
@@ -241,15 +241,15 @@ services:
NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0
- GNB_NGA_IP_ADDRESS: 192.168.71.136
+ GNB_NGA_IP_ADDRESS: 192.168.71.140
GNB_NGU_IF_NAME: eth0
- GNB_NGU_IP_ADDRESS: 192.168.71.136
+ GNB_NGU_IP_ADDRESS: 192.168.71.140
USE_ADDITIONAL_OPTIONS: --sa -E --rfsim --log_config.global_log_options level,nocolor,time
depends_on:
- oai-ext-dn
networks:
public_net:
- ipv4_address: 192.168.71.136
+ ipv4_address: 192.168.71.140
healthcheck:
test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s
@@ -260,7 +260,7 @@ services:
privileged: true
container_name: rfsim5g-oai-nr-ue
environment:
- RFSIMULATOR: 192.168.71.136
+ RFSIMULATOR: 192.168.71.140
FULL_IMSI: '208990100001100'
FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -272,7 +272,7 @@ services:
- oai-gnb
networks:
public_net:
- ipv4_address: 192.168.71.137
+ ipv4_address: 192.168.71.150
healthcheck:
test: /bin/bash -c "pgrep nr-uesoftmodem"
interval: 10s
--
GitLab