diff --git a/ci-scripts/main.py b/ci-scripts/main.py index aaa7ede62a7c9892380c1d486894cece855e62e8..35574512f123aa34412bc1d843be1940dfbb306e 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -128,6 +128,7 @@ class SSHConnection(): self.Build_eNB_args = '' self.backgroundBuild = False self.backgroundBuildTestId = ['', '', ''] + self.Build_eNB_forced_workspace_cleanup = False self.Initialize_eNB_args = '' self.air_interface = 'lte' self.eNB_instance = '' @@ -370,9 +371,16 @@ class SSHConnection(): self.air_interface = 'nr' else: self.air_interface = 'lte' + if self.Build_eNB_forced_workspace_cleanup: + self.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) + result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + if result is not None: + full_ran_repo_name = self.ranRepository + else: + full_ran_repo_name = self.ranRepository + '.git' self.command('mkdir -p ' + lSourcePath, '\$', 5) self.command('cd ' + lSourcePath, '\$', 5) - self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) # Raphael: here add a check if git clone or git fetch went smoothly self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) self.command('git config user.name "OAI Jenkins"', '\$', 5) @@ -4158,6 +4166,14 @@ def CheckClassValidity(action,id): def GetParametersFromXML(action): if action == 'Build_eNB': SSH.Build_eNB_args = test.findtext('Build_eNB_args') + forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') + if (forced_workspace_cleanup is None): + SSH.Build_eNB_forced_workspace_cleanup = False + else: + if re.match('true', forced_workspace_cleanup, re.IGNORECASE): + SSH.Build_eNB_forced_workspace_cleanup = True + else: + SSH.Build_eNB_forced_workspace_cleanup = False SSH.eNB_instance = test.findtext('eNB_instance') if (SSH.eNB_instance is None): SSH.eNB_instance = '0' diff --git a/ci-scripts/oai-ci-vm-tool b/ci-scripts/oai-ci-vm-tool index fb9dc9ad7638f796a8f79e2cd205de2722022570..485c22902ea3021fce964f4ad7486436a337091d 100755 --- a/ci-scripts/oai-ci-vm-tool +++ b/ci-scripts/oai-ci-vm-tool @@ -197,11 +197,15 @@ function variant__v4__cppcheck { } function variant__v5__gnb_usrp { + VM_MEMORY=8192 + VM_CPU=8 NB_PATTERN_FILES=9 BUILD_OPTIONS="--gNB -w USRP" } function variant__v6__nr_ue_usrp { + VM_MEMORY=4096 + VM_CPU=4 NB_PATTERN_FILES=9 BUILD_OPTIONS="--nrUE -w USRP" } @@ -234,8 +238,8 @@ function variant__v20__l1_sim { } function variant__v21__rf_sim { + ARCHIVES_LOC=rf_sim RUN_OPTIONS="complex" - EXPERIMENTAL="true" } function variant__v22__l2_sim { diff --git a/ci-scripts/reportTestLocally.sh b/ci-scripts/reportTestLocally.sh index dd0e6b819c95ddaae2891f53827ca5ead3100eb4..e087cc04e675ccbc1b8a83a44defb048ad0c765e 100755 --- a/ci-scripts/reportTestLocally.sh +++ b/ci-scripts/reportTestLocally.sh @@ -97,14 +97,20 @@ function analyzeIperfFiles { FILE_COMPLETE=`egrep -c "Server Report" $IPERF_CASE` if [ $FILE_COMPLETE -eq 0 ] then - echo " <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html SERVER_FILE=`echo $IPERF_CASE | sed -e "s#client#server#"` FLOAT_EFF_BITRATE=`grep --color=never sec $SERVER_FILE | sed -e "s#^.*Bytes *##" -e "s#sec *.*#sec#" | awk 'BEGIN{s=0;n=0}{n++;if ($2 ~/Mbits/){a = $1 * 1000000};if ($2 ~/Kbits/){a = $1 * 1000};s=s+a}END{br=s/n; printf "%.0f", br}'` - EFFECTIVE_BITRATE=`grep --color=never sec $SERVER_FILE | sed -e "s#^.*Bytes *##" -e "s#sec *.*#sec#" | awk 'BEGIN{s=0;n=0}{n++;if ($2 ~/Mbits/){a = $1 * 1000000};if ($2 ~/Kbits/){a = $1 * 1000};s=s+a}END{br=s/n; if(br>1000000){printf "%.2f MBits/sec", br/1000000}}'` + EFFECTIVE_BITRATE=`grep --color=never sec $SERVER_FILE | sed -e "s#^.*Bytes *##" -e "s#sec *.*#sec#" | awk 'BEGIN{s=0;n=0}{n++;if ($2 ~/Mbits/){a = $1 * 1000000};if ($2 ~/Kbits/){a = $1 * 1000};s=s+a}END{br=s/n; if(br>1000000){printf "%.2f MBits/sec", br/1000000}else{printf "%.2f KBits/sec", br/1000}}'` PERF=`echo "100 * $FLOAT_EFF_BITRATE / $FLOAT_REQ_BITRATE" | bc -l | awk '{printf "%.2f", $0}'` + PERF_INT=`echo "100 * $FLOAT_EFF_BITRATE / $FLOAT_REQ_BITRATE" | bc -l | awk '{printf "%.0f", $0}'` JITTER=`grep --color=never sec $SERVER_FILE | sed -e "s#^.*/sec *##" -e "s# *ms.*##" | awk 'BEGIN{s=0;n=0}{n++;s+=$1}END{jitter=s/n; printf "%.3f ms", jitter}'` PACKETLOSS_NOSIGN=`grep --color=never sec $SERVER_FILE | sed -e "s#^.*(##" -e "s#%.*##" | awk 'BEGIN{s=0;n=0}{n++;s+=$1}END{per=s/n; printf "%.1f", per}'` PACKETLOSS=`echo "${PACKETLOSS_NOSIGN}%"` + if [[ $PERF_INT -lt 80 ]] + then + echo " <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html + else + echo " <td bgcolor = \"green\" >OK</td>" >> ./test_simulator_results.html + fi else EFFECTIVE_BITRATE=`tail -n3 $IPERF_CASE | egrep "Mbits/sec" | sed -e "s#^.*MBytes *##" -e "s#sec.*#sec#"` if [[ $EFFECTIVE_BITRATE =~ .*Kbits/sec.* ]] @@ -394,6 +400,126 @@ function report_test { echo " </div>" >> ./test_simulator_results.html fi + ARCHIVES_LOC=archives/rf_sim/test + if [ -d $ARCHIVES_LOC ] + then + echo " <h3>5G NR RF Simulator Check</h3>" >> ./test_simulator_results.html + + if [ -f $ARCHIVES_LOC/test_final_status.log ] + then + if [ `grep -c TEST_OK $ARCHIVES_LOC/test_final_status.log` -eq 1 ] + then + echo " <div class=\"alert alert-success\">" >> ./test_simulator_results.html + echo " <strong>TEST was SUCCESSFUL <span class=\"glyphicon glyphicon-ok-circle\"></span></strong>" >> ./test_simulator_results.html + echo " </div>" >> ./test_simulator_results.html + else + echo " <div class=\"alert alert-danger\">" >> ./test_simulator_results.html + echo " <strong>TEST was a FAILURE! <span class=\"glyphicon glyphicon-ban-circle\"></span></strong>" >> ./test_simulator_results.html + echo " </div>" >> ./test_simulator_results.html + fi + else + echo " <div class=\"alert alert-danger\">" >> ./test_simulator_results.html + echo " <strong>COULD NOT DETERMINE TEST FINAL STATUS! <span class=\"glyphicon glyphicon-ban-circle\"></span></strong>" >> ./test_simulator_results.html + echo " </div>" >> ./test_simulator_results.html + fi + + echo " <button data-toggle=\"collapse\" data-target=\"#oai-rf-sim-test-details\">More details on 5G RF Simulator test results</button>" >> ./test_simulator_results.html + echo " <div id=\"oai-rf-sim-test-details\" class=\"collapse\">" >> ./test_simulator_results.html + echo " <table border = \"1\">" >> ./test_simulator_results.html + echo " <tr bgcolor = \"#33CCFF\" >" >> ./test_simulator_results.html + echo " <th>Log File Name</th>" >> ./test_simulator_results.html + echo " <th>Command</th>" >> ./test_simulator_results.html + echo " <th>Status</th>" >> ./test_simulator_results.html + echo " <th>Statistics</th>" >> ./test_simulator_results.html + echo " </tr>" >> ./test_simulator_results.html + + EPC_CONFIGS=("noS1") + TRANS_MODES=("tdd") + BW_CASES=(106) + for CN_CONFIG in ${EPC_CONFIGS[@]} + do + for TMODE in ${TRANS_MODES[@]} + do + for BW in ${BW_CASES[@]} + do + echo " <tr bgcolor = \"#8FBC8F\" >" >> ./test_simulator_results.html + if [[ $CN_CONFIG =~ .*wS1.* ]] + then + echo " <td align = \"center\" colspan = 4 >Test with EPC (aka withS1): ${TMODE} -- ${BW}PRB </td>" >> ./test_simulator_results.html + else + echo " <td align = \"center\" colspan = 4 >Test without EPC (aka noS1): ${TMODE} -- ${BW}PRB </td>" >> ./test_simulator_results.html + fi + echo " </tr>" >> ./test_simulator_results.html + ENB_LOG=$ARCHIVES_LOC/${TMODE}_${BW}prb_${CN_CONFIG}_gnb.log + UE_LOG=`echo $ENB_LOG | sed -e "s#gnb#ue#"` + if [ -f $ENB_LOG ] && [ -f $UE_LOG ] + then + NAME_ENB=`echo $ENB_LOG | sed -e "s#$ARCHIVES_LOC/##"` + NAME_UE=`echo $UE_LOG | sed -e "s#$ARCHIVES_LOC/##"` + echo " <tr>" >> ./test_simulator_results.html + echo " <td>$NAME_ENB --- $NAME_UE</td>" >> ./test_simulator_results.html + echo " <td>N/A</td>" >> ./test_simulator_results.html + NB_ENB_GOT_SYNC=`egrep -c "got sync" $ENB_LOG` + NB_ENB_TUNNEL_UP=`egrep -c "Interface oaitun_enb1 successfuly configured" $ENB_LOG` + NB_UE_GOT_SYNC=`egrep -c "rfsimulator: Success" $UE_LOG` + NB_ENB_SYNCED_WITH_UE=`egrep -c "Initial sync: starting PBCH detection" $UE_LOG` + NB_UE_TUNNEL_UP=`egrep -c "Interface oaitun_ue1 successfuly configured" $UE_LOG` + if [ $NB_ENB_GOT_SYNC -gt 0 ] && [ $NB_UE_GOT_SYNC -gt 0 ] && [ $NB_ENB_SYNCED_WITH_UE -gt 0 ] + then + echo " <td bgcolor = \"green\" >OK</td>" >> ./test_simulator_results.html + else + echo " <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html + fi + echo " <td><pre>" >> ./test_simulator_results.html + if [ $NB_ENB_GOT_SYNC -gt 0 ] + then + echo "<font color = \"blue\">- gNB --> got sync</font>" >> ./test_simulator_results.html + else + echo "<font color = \"red\"><b>- gNB NEVER got sync</b></font>" >> ./test_simulator_results.html + fi + if [ $NB_ENB_TUNNEL_UP -gt 0 ] + then + echo "<font color = \"blue\">- gNB mounted oaitun_enb1 interface</font>" >> ./test_simulator_results.html + else + echo "<font color = \"red\"><b>- gNB NEVER mounted oaitun_enb1 interface</b></font>" >> ./test_simulator_results.html + fi + if [ $NB_UE_GOT_SYNC -gt 0 ] + then + echo "<font color = \"blue\">- NR UE --> got sync</font>" >> ./test_simulator_results.html + else + echo "<font color = \"red\"><b>- NR UE NEVER got sync</b></font>" >> ./test_simulator_results.html + fi + if [ $NB_ENB_SYNCED_WITH_UE -gt 0 ] + then + echo "<font color = \"blue\">- NR UE attached to gNB</font>" >> ./test_simulator_results.html + else + echo "<font color = \"red\"><b>- NR UE NEVER attached to eNB</b></font>" >> ./test_simulator_results.html + fi + if [ $NB_UE_TUNNEL_UP -gt 0 ] + then + echo "<font color = \"blue\">- NR UE mounted oaitun_ue1 interface</font>" >> ./test_simulator_results.html + else + echo "<font color = \"red\"><b>- NR UE NEVER mounted oaitun_ue1 interface</b></font>" >> ./test_simulator_results.html + fi + echo " </pre></td>" >> ./test_simulator_results.html + echo " </tr>" >> ./test_simulator_results.html + fi + #PING_LOGS=`ls $ARCHIVES_LOC/${TMODE}_${BW}MHz_${UES}users_${CN_CONFIG}_ping*.log 2> /dev/null` + #analyzePingFiles + + IPERF_TESTS=`ls $ARCHIVES_LOC/${TMODE}_${BW}prb_${CN_CONFIG}_iperf_dl*client*txt 2> /dev/null` + analyzeIperfFiles + + #IPERF_TESTS=`ls $ARCHIVES_LOC/${TMODE}_${BW}MHz_${UES}users_${CN_CONFIG}_iperf_ul*client*txt 2> /dev/null` + #analyzeIperfFiles + done + done + done + + echo " </table>" >> ./test_simulator_results.html + echo " </div>" >> ./test_simulator_results.html + fi + ARCHIVES_LOC=archives/l2_sim/test if [ -d $ARCHIVES_LOC ] then diff --git a/ci-scripts/runTestOnVM.sh b/ci-scripts/runTestOnVM.sh index ded67443bc8089a0d08cd31ef3c85903df190a9b..32a16527839ef6da00bc69fc9ae142f0677ad74b 100755 --- a/ci-scripts/runTestOnVM.sh +++ b/ci-scripts/runTestOnVM.sh @@ -274,21 +274,30 @@ function generic_iperf { local LOC_BASE_LOG_FILE=$8 local LOC_PORT_ID=$[$9+5001] local LOC_FG_OR_BG=${10} + # By default the requested bandwidth is in Mbits/sec + if [[ $LOC_REQ_BANDWIDTH =~ .*K.* ]] + then + local IPERF_FORMAT="-fk" + local FORMATTED_REQ_BW=$LOC_REQ_BANDWIDTH + else + local IPERF_FORMAT="-fm" + local FORMATTED_REQ_BW=${LOC_REQ_BANDWIDTH}"M" + fi # Starting Iperf Server - echo "iperf -B ${LOC_ISERVER_BOND_IP} -u -s -i 1 -fm -p ${LOC_PORT_ID}" - echo "nohup iperf -B ${LOC_ISERVER_BOND_IP} -u -s -i 1 -fm -p ${LOC_PORT_ID} > ${LOC_BASE_LOG_FILE}_server.txt 2>&1 &" > ${LOC_ISERVER_CMD} + echo "iperf -B ${LOC_ISERVER_BOND_IP} -u -s -i 1 ${IPERF_FORMAT} -p ${LOC_PORT_ID}" + echo "nohup iperf -B ${LOC_ISERVER_BOND_IP} -u -s -i 1 ${IPERF_FORMAT} -p ${LOC_PORT_ID} > ${LOC_BASE_LOG_FILE}_server.txt 2>&1 &" > ${LOC_ISERVER_CMD} ssh -T -o StrictHostKeyChecking=no ubuntu@${LOC_ISERVER_IP} < ${LOC_ISERVER_CMD} rm ${LOC_ISERVER_CMD} # Starting Iperf Client - echo "iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${LOC_REQ_BANDWIDTH}M -i 1 -fm -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID}" - echo "echo \"COMMAND IS: iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${LOC_REQ_BANDWIDTH}M -i 1 -fm -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID}\" > ${LOC_BASE_LOG_FILE}_client.txt" > ${LOC_ICLIENT_CMD} + echo "iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${FORMATTED_REQ_BW} -i 1 ${IPERF_FORMAT} -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID}" + echo "echo \"COMMAND IS: iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${FORMATTED_REQ_BW} -i 1 ${IPERF_FORMAT} -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID}\" > ${LOC_BASE_LOG_FILE}_client.txt" > ${LOC_ICLIENT_CMD} if [ $LOC_FG_OR_BG -eq 0 ] then - echo "iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${LOC_REQ_BANDWIDTH}M -i 1 -fm -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID} >> ${LOC_BASE_LOG_FILE}_client.txt 2>&1" >> ${LOC_ICLIENT_CMD} + echo "iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${FORMATTED_REQ_BW} -i 1 ${IPERF_FORMAT} -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID} >> ${LOC_BASE_LOG_FILE}_client.txt 2>&1" >> ${LOC_ICLIENT_CMD} echo "tail -3 ${LOC_BASE_LOG_FILE}_client.txt | grep -v datagram" >> ${LOC_ICLIENT_CMD} else - echo "nohup iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${LOC_REQ_BANDWIDTH}M -i 1 -fm -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID} >> ${LOC_BASE_LOG_FILE}_client.txt 2>&1 &" >> ${LOC_ICLIENT_CMD} + echo "nohup iperf -c ${LOC_ISERVER_BOND_IP} -u -t 30 -b ${FORMATTED_REQ_BW} -i 1 ${IPERF_FORMAT} -B ${LOC_ICLIENT_BOND_IP} -p ${LOC_PORT_ID} >> ${LOC_BASE_LOG_FILE}_client.txt 2>&1 &" >> ${LOC_ICLIENT_CMD} fi ssh -T -o StrictHostKeyChecking=no ubuntu@${LOC_ICLIENT_IP} < ${LOC_ICLIENT_CMD} rm -f ${LOC_ICLIENT_CMD} @@ -305,19 +314,46 @@ function generic_iperf { function check_iperf { local LOC_BASE_LOG=$1 - local LOC_REQ_BW=$2 + local LOC_REQ_BW=`echo $2 | sed -e "s#K##"` local LOC_REQ_BW_MINUS_ONE=`echo "$LOC_REQ_BW - 1" | bc -l` local LOC_REQ_BW_MINUS_TWO=`echo "$LOC_REQ_BW - 2" | bc -l` local LOC_REQ_BW_MINUS_THREE=`echo "$LOC_REQ_BW - 3" | bc -l` local LOC_IS_DL=`echo $LOC_BASE_LOG | grep -c _dl` local LOC_IS_BASIC_SIM=`echo $LOC_BASE_LOG | grep -c basic_sim` + local LOC_IS_RF_SIM=`echo $LOC_BASE_LOG | grep -c rf_sim` + local LOC_IS_NR=`echo $LOC_BASE_LOG | grep -c tdd_106prb` if [ -f ${LOC_BASE_LOG}_client.txt ] then local FILE_COMPLETE=`egrep -c "Server Report" ${LOC_BASE_LOG}_client.txt` if [ $FILE_COMPLETE -eq 0 ] then - IPERF_STATUS=-1 - echo "File Report not found" + if [[ $LOC_IS_RF_SIM -eq 1 ]] && [[ $LOC_IS_NR -eq 1 ]] + then + echo "no UL integration right now --> normal to have no server report" + if [ -f ${LOC_BASE_LOG}_server.txt ] + then + local EFFECTIVE_BANDWIDTH=`tail -n1 ${LOC_BASE_LOG}_server.txt | sed -e "s#^.*MBytes *##" -e "s#^.*KBytes *##" -e "s#sec.*#sec#"` + if [[ $2 =~ .*K.* ]] + then + local BW_SUFFIX="K" + else + local BW_SUFFIX="M" + fi + if [[ $EFFECTIVE_BANDWIDTH =~ .*${LOC_REQ_BW}.*${BW_SUFFIX}bits.* ]] || [[ $EFFECTIVE_BANDWIDTH =~ .*${LOC_REQ_BW_MINUS_ONE}.*${BW_SUFFIX}bits.* ]] + then + echo "got requested DL bandwidth: $EFFECTIVE_BANDWIDTH" + else + echo "got LESS than requested DL bandwidth: $EFFECTIVE_BANDWIDTH" + IPERF_STATUS=-1 + fi + else + IPERF_STATUS=-1 + echo "Server File Report not found" + fi + else + IPERF_STATUS=-1 + echo "File Report not found" + fi else local EFFECTIVE_BANDWIDTH=`tail -n3 ${LOC_BASE_LOG}_client.txt | egrep "Mbits/sec" | sed -e "s#^.*MBytes *##" -e "s#sec.*#sec#"` if [ $LOC_IS_DL -eq 1 ] && [ $LOC_IS_BASIC_SIM -eq 1 ] @@ -351,40 +387,26 @@ function check_iperf { } function terminate_enb_ue_basic_sim { - # mode = 0 : eNB + UE - # mode = 1 : eNB - # mode = 2 : UE + # mode = 0 : eNB + UE or gNB and NR-UE + # mode = 1 : eNB or gNB + # mode = 2 : UE or NR-UE local LOC_MODE=$3 echo "NB_OAI_PROCESSES=\`ps -aux | grep modem | grep -v grep | grep -c softmodem\`" > $1 if [ $LOC_MODE -eq 0 ] || [ $LOC_MODE -eq 1 ] then - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo daemon --name=enb_daemon --stop\"; fi" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo daemon --name=enb_daemon --stop; fi" >> $1 - fi - if [ $LOC_MODE -eq 0 ] || [ $LOC_MODE -eq 2 ] - then - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo daemon --name=ue_daemon --stop\"; fi" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo daemon --name=ue_daemon --stop; fi" >> $1 - fi - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sleep 5; fi" >> $1 - echo "echo \"ps -aux | grep softmodem\"" >> $1 - echo "ps -aux | grep softmodem | grep -v grep" >> $1 - echo "NB_OAI_PROCESSES=\`ps -aux | grep modem | grep -v grep | grep -c softmodem\`" >> $1 - if [ $LOC_MODE -eq 0 ] || [ $LOC_MODE -eq 1 ] - then - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall --signal SIGINT lte-softmodem\"; fi" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall --signal SIGINT lte-softmodem; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall -r --signal SIGINT .*-softmodem\"; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall -r --signal SIGINT .*-softmodem; fi" >> $1 echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sleep 5; fi" >> $1 echo "echo \"ps -aux | grep softmodem\"" >> $1 echo "ps -aux | grep softmodem | grep -v grep" >> $1 echo "NB_OAI_PROCESSES=\`ps -aux | grep modem | grep -v grep | grep -c softmodem\`" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall --signal SIGKILL lte-softmodem\"; fi" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall --signal SIGKILL lte-softmodem; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall -r --signal SIGKILL .*-softmodem\"; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall -r --signal SIGKILL .*-softmodem; fi" >> $1 fi if [ $LOC_MODE -eq 0 ] || [ $LOC_MODE -eq 2 ] then - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall --signal SIGKILL lte-uesoftmodem\"; fi" >> $1 - echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall --signal SIGKILL lte-uesoftmodem; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then echo \"sudo killall -r --signal SIGKILL .*-uesoftmodem\"; fi" >> $1 + echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sudo killall -r --signal SIGKILL .*-uesoftmodem; fi" >> $1 echo "if [ \$NB_OAI_PROCESSES -ne 0 ]; then sleep 5; fi" >> $1 fi echo "echo \"ps -aux | grep softmodem\"" >> $1 @@ -1075,6 +1097,162 @@ function start_rf_sim_ue { } +function start_rf_sim_gnb { + local LOC_GNB_VM_IP_ADDR=$2 + local LOC_LOG_FILE=$3 + local LOC_NB_RBS=$4 + local LOC_CONF_FILE=$5 + # 1 is with S1 and 0 without S1 aka noS1 + local LOC_S1_CONFIGURATION=$6 + echo "cd /home/ubuntu/tmp" > $1 + echo "echo \"sudo apt-get --yes --quiet install daemon \"" >> $1 + echo "sudo apt-get --yes install daemon >> /home/ubuntu/tmp/cmake_targets/log/daemon-install.txt 2>&1" >> $1 + echo "echo \"source oaienv\"" >> $1 + echo "source oaienv" >> $1 + echo "cd ci-scripts/conf_files/" >> $1 + echo "cp $LOC_CONF_FILE ci-$LOC_CONF_FILE" >> $1 + #echo "sed -i -e 's#N_RB_DL.*=.*;#N_RB_DL = $LOC_NB_RBS;#' -e 's#CI_MME_IP_ADDR#$LOC_EPC_IP_ADDR#' -e 's#CI_ENB_IP_ADDR#$LOC_ENB_VM_IP_ADDR#' -e 's#CI_UE_IP_ADDR#$LOC_UE_VM_IP_ADDR#' ci-$LOC_CONF_FILE" >> $1 + #echo "echo \"grep N_RB_DL ci-$LOC_CONF_FILE\"" >> $1 + #echo "grep N_RB_DL ci-$LOC_CONF_FILE | sed -e 's#N_RB_DL.*=#N_RB_DL =#'" >> $1 + echo "echo \"cd /home/ubuntu/tmp/cmake_targets/ran_build/build/\"" >> $1 + echo "sudo chmod 777 /home/ubuntu/tmp/cmake_targets/ran_build/build/" >> $1 + echo "cd /home/ubuntu/tmp/cmake_targets/ran_build/build/" >> $1 + if [ $LOC_S1_CONFIGURATION -eq 0 ] + then + echo "echo \"RFSIMULATOR=server ./nr-softmodem -O /home/ubuntu/tmp/ci-scripts/conf_files/ci-$LOC_CONF_FILE --log_config.global_log_options level,nocolor --parallel-config PARALLEL_SINGLE_THREAD --noS1 --nokrnmod 1 --rfsim\" > ./my-nr-softmodem-run.sh " >> $1 + fi + echo "chmod 775 ./my-nr-softmodem-run.sh" >> $1 + echo "cat ./my-nr-softmodem-run.sh" >> $1 + echo "if [ -e /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ]; then sudo sudo rm -f /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE; fi" >> $1 + echo "sudo -E daemon --inherit --unsafe --name=gnb_daemon --chdir=/home/ubuntu/tmp/cmake_targets/ran_build/build/ -o /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ./my-nr-softmodem-run.sh" >> $1 + + ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_GNB_VM_IP_ADDR < $1 + rm $1 + + local i="0" + echo "egrep -c \"got sync\" /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE" > $1 + while [ $i -lt 10 ] + do + sleep 5 + CONNECTED=`ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_GNB_VM_IP_ADDR < $1` + if [ $CONNECTED -ne 0 ] + then + i="100" + else + i=$[$i+1] + fi + done + rm $1 + if [ $i -lt 50 ] + then + GNB_SYNC=0 + echo "RF-SIM gNB is NOT sync'ed: process still alive?" + else + GNB_SYNC=1 + echo "RF-SIM gNB is sync'ed: waiting for UE(s) to connect" + fi + if [ $LOC_S1_CONFIGURATION -eq 0 ] + then + echo "ifconfig oaitun_enb1 | egrep -c \"inet addr\"" > $1 + # Checking oaitun_enb1 interface has now an IP address + i="0" + while [ $i -lt 10 ] + do + CONNECTED=`ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_GNB_VM_IP_ADDR < $1` + if [ $CONNECTED -eq 1 ] + then + i="100" + else + i=$[$i+1] + sleep 5 + fi + done + rm $1 + if [ $i -lt 50 ] + then + GNB_SYNC=0 + echo "RF-SIM gNB oaitun_enb1 is DOWN or NOT CONFIGURED" + else + echo "RF-SIM gNB oaitun_enb1 is UP and CONFIGURED" + fi + fi + sleep 10 +} + +function start_rf_sim_nr_ue { + local LOC_NR_UE_VM_IP_ADDR=$2 + local LOC_GNB_VM_IP_ADDR=$3 + local LOC_LOG_FILE=$4 + local LOC_PRB=$5 + local LOC_FREQUENCY=$6 + # 1 is with S1 and 0 without S1 aka noS1 + local LOC_S1_CONFIGURATION=$7 + echo "echo \"sudo apt-get --yes --quiet install daemon \"" > $1 + echo "sudo apt-get --yes install daemon >> /home/ubuntu/tmp/cmake_targets/log/daemon-install.txt 2>&1" >> $1 + echo "echo \"cd /home/ubuntu/tmp/cmake_targets/ran_build/build/\"" >> $1 + echo "sudo chmod 777 /home/ubuntu/tmp/cmake_targets/ran_build/build/" >> $1 + echo "cd /home/ubuntu/tmp/cmake_targets/ran_build/build/" >> $1 + if [ $LOC_S1_CONFIGURATION -eq 0 ] + then + echo "echo \"RFSIMULATOR=${LOC_GNB_VM_IP_ADDR} ./nr-uesoftmodem --numerology 1 -C ${LOC_FREQUENCY}000000 -r $LOC_PRB --nokrnmod 1 --rfsim --log_config.global_log_options level,nocolor --noS1\" > ./my-nr-softmodem-run.sh " >> $1 + fi + echo "chmod 775 ./my-nr-softmodem-run.sh" >> $1 + echo "cat ./my-nr-softmodem-run.sh" >> $1 + echo "if [ -e /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ]; then sudo sudo rm -f /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE; fi" >> $1 + echo "sudo -E daemon --inherit --unsafe --name=nr_ue_daemon --chdir=/home/ubuntu/tmp/cmake_targets/ran_build/build/ -o /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ./my-nr-softmodem-run.sh" >> $1 + + ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_NR_UE_VM_IP_ADDR < $1 + rm $1 + + local i="0" + echo "egrep -c \"rfsimulator: Success\" /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE" > $1 + while [ $i -lt 10 ] + do + sleep 5 + CONNECTED=`ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_NR_UE_VM_IP_ADDR < $1` + if [ $CONNECTED -ne 0 ] + then + i="100" + else + i=$[$i+1] + fi + done + NR_UE_SYNC=1 + rm $1 + if [ $i -lt 50 ] + then + NR_UE_SYNC=0 + echo "RF-SIM NR-UE is NOT sync'ed w/ gNB" + return + else + echo "RF-SIM NR-UE is sync'ed w/ gNB" + fi + # Checking oaitun_ue1 interface has now an IP address + i="0" + echo "ifconfig oaitun_ue1 | egrep -c \"inet addr\"" > $1 + while [ $i -lt 10 ] + do + sleep 5 + CONNECTED=`ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_NR_UE_VM_IP_ADDR < $1` + if [ $CONNECTED -eq 1 ] + then + i="100" + else + i=$[$i+1] + fi + done + rm $1 + if [ $i -lt 50 ] + then + NR_UE_SYNC=0 + echo "RF-SIM NR-UE oaitun_ue1 is DOWN or NOT CONFIGURED" + else + echo "RF-SIM NR-UE oaitun_ue1 is UP and CONFIGURED" + fi + sleep 10 +} + + function run_test_on_vm { echo "############################################################" echo "OAI CI VM script" @@ -1089,6 +1267,14 @@ function run_test_on_vm { UE_VM_CMDS=${UE_VM_NAME}_cmds.txt echo "UE_VM_NAME = $UE_VM_NAME" echo "UE_VM_CMD_FILE = $UE_VM_CMDS" + GNB_VM_NAME=`echo $VM_NAME | sed -e "s#l2-sim#gnb-usrp#" -e "s#rf-sim#gnb-usrp#"` + GNB_VM_CMDS=${GNB_VM_NAME}_cmds.txt + echo "GNB_VM_NAME = $GNB_VM_NAME" + echo "GNB_VM_CMD_FILE = $GNB_VM_CMDS" + NR_UE_VM_NAME=`echo $VM_NAME | sed -e "s#l2-sim#nr-ue-usrp#" -e "s#rf-sim#nr-ue-usrp#"` + NR_UE_VM_CMDS=${UE_VM_NAME}_cmds.txt + echo "NR_UE_VM_NAME = $NR_UE_VM_NAME" + echo "NR_UE_VM_CMD_FILE = $NR_UE_VM_CMDS" else echo "VM_NAME = $VM_NAME" echo "VM_CMD_FILE = $VM_CMDS" @@ -1113,6 +1299,22 @@ function run_test_on_vm { UE_VM_IP_ADDR=`uvt-kvm ip $UE_VM_NAME` echo "$UE_VM_NAME has for IP addr = $UE_VM_IP_ADDR" + + echo "############################################################" + echo "Waiting for GNB VM to be started" + echo "############################################################" + uvt-kvm wait $GNB_VM_NAME --insecure + + GNB_VM_IP_ADDR=`uvt-kvm ip $GNB_VM_NAME` + echo "$GNB_VM_NAME has for IP addr = $GNB_VM_IP_ADDR" + + echo "############################################################" + echo "Waiting for NR-UE VM to be started" + echo "############################################################" + uvt-kvm wait $NR_UE_VM_NAME --insecure + + NR_UE_VM_IP_ADDR=`uvt-kvm ip $NR_UE_VM_NAME` + echo "$NR_UE_VM_NAME has for IP addr = $NR_UE_VM_IP_ADDR" else echo "############################################################" echo "Waiting for VM to be started" @@ -1466,7 +1668,7 @@ function run_test_on_vm { fi fi - if [[ "$RUN_OPTIONS" == "complex" ]] && [[ $VM_NAME =~ .*-rf-sim.* ]] + if [[ "$RUN_OPTIONS" == "complex" ]] && [[ $VM_NAME =~ .*-ru-sim.* ]] then PING_STATUS=0 IPERF_STATUS=0 @@ -1666,6 +1868,84 @@ function run_test_on_vm { fi fi + if [[ "$RUN_OPTIONS" == "complex" ]] && [[ $VM_NAME =~ .*-rf-sim.* ]] + then + PING_STATUS=0 + IPERF_STATUS=0 + if [ -d $ARCHIVES_LOC ] + then + rm -Rf $ARCHIVES_LOC + fi + mkdir --parents $ARCHIVES_LOC + + CN_CONFIG="noS1" + CONF_FILE=gnb.band78.tm1.106PRB.usrpn300.conf + S1_NOS1_CFG=0 + PRB=106 + FREQUENCY=3510 + + ######### start of loop + + echo "############################################################" + echo "${CN_CONFIG} : Starting the gNB" + echo "############################################################" + CURRENT_GNB_LOG_FILE=tdd_${PRB}prb_${CN_CONFIG}_gnb.log + start_rf_sim_gnb $GNB_VM_CMDS "$GNB_VM_IP_ADDR" $CURRENT_GNB_LOG_FILE $PRB $CONF_FILE $S1_NOS1_CFG + + echo "############################################################" + echo "${CN_CONFIG} : Starting the NR-UE" + echo "############################################################" + CURRENT_NR_UE_LOG_FILE=tdd_${PRB}prb_${CN_CONFIG}_ue.log + start_rf_sim_nr_ue $NR_UE_VM_CMDS $NR_UE_VM_IP_ADDR $GNB_VM_IP_ADDR $CURRENT_NR_UE_LOG_FILE $PRB $FREQUENCY $S1_NOS1_CFG + if [ $NR_UE_SYNC -eq 0 ] + then + echo "Problem w/ gNB and NR-UE not syncing" + terminate_enb_ue_basic_sim $NR_UE_VM_CMDS $NR_UE_VM_IP_ADDR 2 + terminate_enb_ue_basic_sim $GNB_VM_CMDS $GNB_VM_IP_ADDR 1 + scp -o StrictHostKeyChecking=no ubuntu@$GNB_VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_GNB_LOG_FILE $ARCHIVES_LOC + scp -o StrictHostKeyChecking=no ubuntu@$NR_UE_VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_NR_UE_LOG_FILE $ARCHIVES_LOC + echo "TEST_KO" > $ARCHIVES_LOC/test_final_status.log + STATUS=-1 + return + fi + + echo "############################################################" + echo "${CN_CONFIG} : iperf DL -- UE is server and eNB is client" + echo "############################################################" + THROUGHPUT="30K" + CURR_IPERF_LOG_BASE=tdd_${PRB}prb_${CN_CONFIG}_iperf_dl + get_enb_noS1_ip_addr $GNB_VM_CMDS $GNB_VM_IP_ADDR + get_ue_ip_addr $NR_UE_VM_CMDS $NR_UE_VM_IP_ADDR 1 + generic_iperf $NR_UE_VM_CMDS $NR_UE_VM_IP_ADDR $UE_IP_ADDR $GNB_VM_CMDS $GNB_VM_IP_ADDR $ENB_IP_ADDR $THROUGHPUT $CURR_IPERF_LOG_BASE 1 0 + scp -o StrictHostKeyChecking=no ubuntu@$GNB_VM_IP_ADDR:/home/ubuntu/${CURR_IPERF_LOG_BASE}_client.txt $ARCHIVES_LOC + scp -o StrictHostKeyChecking=no ubuntu@$NR_UE_VM_IP_ADDR:/home/ubuntu/${CURR_IPERF_LOG_BASE}_server.txt $ARCHIVES_LOC + check_iperf $ARCHIVES_LOC/$CURR_IPERF_LOG_BASE $THROUGHPUT + + echo "############################################################" + echo "${CN_CONFIG} : Terminate gNB/NR-UE simulators" + echo "############################################################" + terminate_enb_ue_basic_sim $NR_UE_VM_CMDS $NR_UE_VM_IP_ADDR 2 + terminate_enb_ue_basic_sim $GNB_VM_CMDS $GNB_VM_IP_ADDR 1 + scp -o StrictHostKeyChecking=no ubuntu@$GNB_VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_GNB_LOG_FILE $ARCHIVES_LOC + scp -o StrictHostKeyChecking=no ubuntu@$NR_UE_VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_NR_UE_LOG_FILE $ARCHIVES_LOC + + ######### end of loop + full_l2_sim_destroy + + echo "############################################################" + echo "Checking run status" + echo "############################################################" + + if [ $PING_STATUS -ne 0 ]; then STATUS=-1; fi + if [ $IPERF_STATUS -ne 0 ]; then STATUS=-1; fi + if [ $STATUS -eq 0 ] + then + echo "TEST_OK" > $ARCHIVES_LOC/test_final_status.log + else + echo "TEST_KO" > $ARCHIVES_LOC/test_final_status.log + fi + fi + if [[ "$RUN_OPTIONS" == "complex" ]] && [[ $VM_NAME =~ .*-l2-sim.* ]] then PING_STATUS=0 diff --git a/ci-scripts/xml_files/gnb_usrp_build.xml b/ci-scripts/xml_files/gnb_usrp_build.xml index 1b57c077bad00e3ffa4c33d1a007ce17cb3cce9f..bba92930c51eed145afe6df831defea38d3ef5c5 100644 --- a/ci-scripts/xml_files/gnb_usrp_build.xml +++ b/ci-scripts/xml_files/gnb_usrp_build.xml @@ -34,6 +34,7 @@ <class>Build_eNB</class> <desc>Build gNB (USRP)</desc> <Build_eNB_args>--gNB -w USRP</Build_eNB_args> + <forced_workspace_cleanup>True</forced_workspace_cleanup> </testCase> </testCaseList>