diff --git a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf index 900566d29cafa418af312ec97344ffaa703801ec..aac4f5d2206a06c534c9d71612a551b8c75a5b4a 100644 --- a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf +++ b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf @@ -47,7 +47,7 @@ gNBs = dl_carrierBandwidth = 106; #initialDownlinkBWP #genericParameters - # this is RBstart=84,L=13 (275*(L-1))+RBstart + # this is RBstart=43,L=24 (275*(L-1))+RBstart initialDLBWPlocationAndBandwidth = 6368; # subcarrierSpacing # 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 @@ -234,7 +234,7 @@ L1s = ( num_cc = 1; tr_n_preference = "local_mac"; pusch_proc_threads = 8; - prach_dtx_threshold = 100; + prach_dtx_threshold = 120; } ); diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 6d8d00610ca4e8b596e2152a0b409d74c1f2e6b7..d29936d1bb9117020b40d1afdee6eac73d8eaea4 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -39,7 +39,6 @@ import time from multiprocessing import Process, Lock, SimpleQueue import yaml - #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- @@ -482,13 +481,14 @@ class RANManagement(): mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) - #tentative + #stats monitoring during runtime time.sleep(20) + monitor_file='stats_monitor.py' if self.eNB_Stats=='yes': - monitor_file='stats_monitor.py' - #mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) - #mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) - mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' 2>&1 > stats_monitor_execution.log &', '\$', 5) + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' enb 2>&1 > enb_stats_monitor_execution.log &', '\$', 5) + else: + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' gnb 2>&1 > gnb_stats_monitor_execution.log &', '\$', 5) @@ -686,6 +686,11 @@ class RANManagement(): fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)] self.eNBLogFiles[int(self.eNB_instance)] = '' if analyzeFile: + #*stats.log files + pickle + png + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.') + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.') + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.') + # copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.') if (copyin_res == -1): logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m') @@ -694,6 +699,14 @@ class RANManagement(): self.eNBmbmsEnables[int(self.eNB_instance)] = False return if self.eNB_serverId[self.eNB_instance] != '0': + #*stats.log files + pickle + png + + #debug / tentative + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.pickle.pickle', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.png', self.eNBSourceCodePath + '/cmake_targets/') + # mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/') logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze) logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML) @@ -719,11 +732,9 @@ class RANManagement(): mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.pickle .','\$',20) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.png .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor.png', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *.pickle *.png', '\$', 5) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): diff --git a/ci-scripts/stats_monitor.py b/ci-scripts/stats_monitor.py index 7ce00dab4643e00983fb61561d57144ac514420b..ae9b39bfa632c5b85d24d494464b2ae001040cb3 100755 --- a/ci-scripts/stats_monitor.py +++ b/ci-scripts/stats_monitor.py @@ -8,8 +8,11 @@ import pickle import numpy as np import os -def collect(d): - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' +def collect(d, node_type): + if node_type=='enb': + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + else: #'gnb' + cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) output = process.stdout.readlines() for l in output: @@ -22,7 +25,7 @@ def collect(d): d['mcs'].append(int(result.group(4))) -def graph(d): +def graph(d, node_type): figure, axis = plt.subplots(4, 1,figsize=(10, 10)) @@ -61,11 +64,12 @@ def graph(d): plt.tight_layout() # Combine all the operations and display - plt.savefig('/tmp/stats_monitor.png') + plt.savefig(node_type+'_stats_monitor.png') plt.show() if __name__ == "__main__": + node_type = sys.argv[1]#enb or gnb d={} d['PHR']=[] @@ -74,17 +78,17 @@ if __name__ == "__main__": d['mcs']=[] - cmd='ps aux | grep mode | grep -v grep' + cmd='ps aux | grep modem | grep -v grep' process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) output = process.stdout.readlines() while len(output)!=0 : - collect(d) + collect(d, node_type) process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) output = process.stdout.readlines() time.sleep(1) print('process stopped') - with open('/tmp/stats_monitor.pickle', 'wb') as handle: + with open(node_type+'_stats_monitor.pickle', 'wb') as handle: pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL) - graph(d) + graph(d, node_type) diff --git a/ci-scripts/stats_monitor_conf.yaml b/ci-scripts/stats_monitor_conf.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3da7c1e8ba369189917909e04af9f608ccbddd17 --- /dev/null +++ b/ci-scripts/stats_monitor_conf.yaml @@ -0,0 +1,11 @@ +enb : + PHR: + bler: + mcsoff: + mcs: + +gnb : + PHR: + bler: + mcsoff: + mcs: \ No newline at end of file diff --git a/ci-scripts/stats_monitor_dev.py b/ci-scripts/stats_monitor_dev.py new file mode 100755 index 0000000000000000000000000000000000000000..cbf0d20880bccd484007b16d8d1db00d00cfb858 --- /dev/null +++ b/ci-scripts/stats_monitor_dev.py @@ -0,0 +1,100 @@ +import subprocess +import time +import shlex +import re +import sys +import matplotlib.pyplot as plt +import pickle +import numpy as np +import os +import yaml + + +class Stat_Monitor(): + def __init__(self,): + with open('stats_monitor_conf.yaml','r') as f: + self.d = yaml.load(f) + for node in self.d: + for metric in self.d[node]: + self.d[node][metric]=[] + + + def process_enb (self,node_type,output): + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + self.d[node_type]['PHR'].append(int(result.group(1))) + self.d[node_type]['bler'].append(float(result.group(2))) + self.d[node_type]['mcsoff'].append(int(result.group(3))) + self.d[node_type]['mcs'].append(int(result.group(4))) + + + + def process_gnb (self,node_type,output): + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + self.d[node_type]['PHR'].append(int(result.group(1))) + self.d[node_type]['bler'].append(float(result.group(2))) + self.d[node_type]['mcsoff'].append(int(result.group(3))) + self.d[node_type]['mcs'].append(int(result.group(4))) + + + + + + def collect(self,node_type): + if node_type=='enb': + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + else: #'gnb' + cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' + process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readlines() + if node_type=='enb': + self.process_enb(node_type,output) + else: #'gnb' + self.process_gnb(node_type,output) + + def graph(self,node_type): + + col = 1 + figure, axis = plt.subplots(len(self.d[node_type]), col ,figsize=(10, 10)) + i=0 + for metric in self.d[node_type]: + major_ticks = np.arange(0, len(self.d[node_type][metric])+1, 1) + axis[i].set_xticks(major_ticks) + axis[i].set_xticklabels([]) + axis[i].plot(self.d[node_type][metric],marker='o') + axis[i].set_xlabel('time') + axis[i].set_ylabel(metric) + axis[i].set_title(metric) + i+=1 + + plt.tight_layout() + # Combine all the operations and display + plt.savefig(node_type+'_stats_monitor.png') + plt.show() + + +if __name__ == "__main__": + + node_type = sys.argv[1]#enb or gnb + mon=Stat_Monitor() + + #collecting stats when modem process is stopped + cmd='ps aux | grep mode | grep -v grep' + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + while len(output)!=0 : + mon.collect(node_type) + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + time.sleep(1) + print('Process stopped') + with open(node_type+'_stats_monitor.pickle', 'wb') as handle: + pickle.dump(mon.d, handle, protocol=pickle.HIGHEST_PROTOCOL) + mon.graph(node_type) + + diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml index 9e1c7afde37a32372ef700894c59a37ac4755753..1bc7e918f569fac63e6d09b94233ce9214a6d520 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml @@ -65,6 +65,7 @@ <eNB_serverId>0</eNB_serverId> <air_interface>lte</air_interface> <eNB_Trace>yes</eNB_Trace> + <eNB_Stats>yes</eNB_Stats> <USRP_IPAddress>192.168.18.241</USRP_IPAddress> </testCase> @@ -76,6 +77,7 @@ <eNB_instance>1</eNB_instance> <eNB_serverId>1</eNB_serverId> <air_interface>nr</air_interface> + <eNB_Stats>yes</eNB_Stats> <USRP_IPAddress>192.168.18.240</USRP_IPAddress> </testCase> @@ -111,7 +113,7 @@ <testCase id="070000"> <class>Iperf</class> <desc>iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile)</desc> - <iperf_args>-u -b 20M -t 60</iperf_args> + <iperf_args>-u -b 20M -t 10</iperf_args> <direction>DL</direction> <id>nrmodule2_quectel</id> <iperf_packetloss_threshold>5</iperf_packetloss_threshold> @@ -120,8 +122,8 @@ <testCase id="070001"> <class>Iperf</class> - <desc>iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile)</desc> - <iperf_args>-u -b 3M -t 60</iperf_args> + <desc>iperf (UL/1Mbps/UDP)(60 sec)(single-ue profile)</desc> + <iperf_args>-u -b 1M -t 10</iperf_args> <direction>UL</direction> <id>nrmodule2_quectel</id> <iperf_packetloss_threshold>5</iperf_packetloss_threshold> diff --git a/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml index 31eee2ca3d4de4208bacefe8535852052670e459..0005ce8844ad3c5e5aa696aadf63ed4a43b76515 100644 --- a/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml +++ b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml @@ -143,6 +143,7 @@ services: TAC: 1 GW_ID: 1 REALM: openairinterface.org + UE_MTU_IPV4: 1500 healthcheck: test: /bin/bash -c "pgrep oai_spgwc" interval: 10s