main.py 106 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Version
#-----------------------------------------------------------
Version = '0.1'

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
#-----------------------------------------------------------
# Constants
#-----------------------------------------------------------
ALL_PROCESSES_OK = 0
ENB_PROCESS_FAILED = -1
ENB_PROCESS_OK = +1
ENB_PROCESS_SEG_FAULT = -11
ENB_PROCESS_ASSERTION = -12
ENB_PROCESS_REALTIME_ISSUE = -13
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
MME_PROCESS_OK = +3
SPGW_PROCESS_FAILED = -4
SPGW_PROCESS_OK = +4
UE_IP_ADDRESS_ISSUE = -5

53 54 55 56 57 58 59 60 61 62 63 64
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect		# pexpect
import time		# sleep
import os
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
Raphael Defosseux's avatar
Raphael Defosseux committed
65
from multiprocessing import Process, Lock, SimpleQueue
66
logging.basicConfig(
67 68
	level=logging.DEBUG,
	format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
69 70 71 72 73 74 75 76 77 78
)

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
	def __init__(self):
		self.eNBIPAddress = ''
		self.eNBRepository = ''
		self.eNBBranch = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
79
		self.eNB_AllowMerge = False
80
		self.eNBCommitID = ''
81
		self.eNBTargetBranch = ''
82 83 84 85 86 87 88 89 90 91 92 93
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.EPCIPAddress = ''
		self.EPCUserName = ''
		self.EPCPassword = ''
		self.EPCSourceCodePath = ''
		self.EPCType = ''
		self.ADBIPAddress = ''
		self.ADBUserName = ''
		self.ADBPassword = ''
		self.testCase_id = ''
94
		self.testXMLfile = ''
95 96 97
		self.desc = ''
		self.Build_eNB_args = ''
		self.Initialize_eNB_args = ''
98
		self.eNBLogFile = ''
99
		self.eNB_instance = ''
100 101 102 103
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
104
		self.iperf_profile = ''
105
		self.nbMaxUEtoAttach = -1
106
		self.UEDevices = []
107
		self.CatMDevices = []
108
		self.UEIPAddresses = []
Raphael Defosseux's avatar
Raphael Defosseux committed
109 110 111 112
		self.htmlFile = ''
		self.htmlHeaderCreated = False
		self.htmlFooterCreated = False
		self.htmlUEConnected = 0
113
		self.htmleNBFailureMsg = ''
114
		self.picocom_closure = False
115

116
	def open(self, ipaddress, username, password):
117 118 119 120 121
		count = 0
		connect_status = False
		while count < 4:
			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
122
			if self.sshresponse == 0:
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
				self.ssh.sendline('yes')
				self.ssh.expect('password:')
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 1:
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 2:
				# Checking if we are really on the remote client defined by its IP address
				self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:"', '\$', 5)
				result = re.search(str(ipaddress), str(self.ssh.before))
				if result is None:
					self.close()
				else:
					count = 10
					connect_status = True
149
			else:
150 151
				# debug output
				logging.debug(str(self.ssh.before))
152
				logging.debug('self.sshresponse = ' + str(self.sshresponse))
Raphael Defosseux's avatar
Raphael Defosseux committed
153 154 155
			# adding a tempo when failure
			if not connect_status:
				time.sleep(1)
156 157
			count += 1
		if connect_status:
158 159
			pass
		else:
160
			sys.exit('SSH Connection Failed')
161

162 163 164 165 166 167
	def command(self, commandline, expectedline, timeout):
		logging.debug(commandline)
		self.ssh.timeout = timeout
		self.ssh.sendline(commandline)
		self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
168
			return 0
169 170 171 172 173 174 175
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
		elif self.sshresponse == 2:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
176
			result = re.search('ping |iperf |picocom', str(commandline))
Raphael Defosseux's avatar
Raphael Defosseux committed
177
			if result is None:
178
				logging.debug(str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
179 180 181
				sys.exit(self.sshresponse)
			else:
				return -1
182 183 184 185
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
186

187 188 189 190 191 192 193
	def close(self):
		self.ssh.timeout = 5
		self.ssh.sendline('exit')
		self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
			pass
		elif self.sshresponse == 1:
194 195
			if not self.picocom_closure:
				logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m')
196
		else:
197
			logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m')
198

199
	def copyin(self, ipaddress, username, password, source, destination):
Raphael Defosseux's avatar
Raphael Defosseux committed
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
		logging.debug('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
		scp_spawn = pexpect.spawn('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 5)
		scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
		if scp_response == 0:
			scp_spawn.sendline('yes')
			scp_spawn.expect('password:')
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0:
				pass
			else:
				logging.debug('1 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 1:
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0 or scp_response == 3:
				pass
			else:
				logging.debug('2 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 2:
			pass
		else:
			logging.debug('3 - scp_response = ' + str(scp_response))
			sys.exit('SCP failed')

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
	def copyout(self, ipaddress, username, password, source, destination):
		logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
		scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 5)
		scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
		if scp_response == 0:
			scp_spawn.sendline('yes')
			scp_spawn.expect('password:')
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0:
				pass
			else:
				logging.debug('1 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 1:
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0 or scp_response == 3:
				pass
			else:
				logging.debug('2 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 2:
			pass
		else:
			logging.debug('3 - scp_response = ' + str(scp_response))
			sys.exit('SCP failed')

255 256 257 258 259 260 261
	def BuildeNB(self):
		if self.eNBIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
262 263
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
264 265
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
266
		self.command('echo ' + self.eNBPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
267 268 269 270 271
		# if the commit ID is provided use it to point to it
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
Raphael Defosseux's avatar
Raphael Defosseux committed
272
		if (self.eNB_AllowMerge):
273 274 275 276 277 278
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
279 280
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
281 282
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
283
		# no need to remove in log (git clean did the trick)
284
		self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_enb.log', 'Bypassing the Tests', 600)
285
		self.command('mkdir -p build_log_' + SSH.testCase_id, '\$', 5)
286 287 288 289
		self.command('mv log/* ' + 'build_log_' + SSH.testCase_id, '\$', 5)
		self.command('mv compile_oai_enb.log ' + 'build_log_' + SSH.testCase_id, '\$', 5)
		# Workaround to run with develop-nr
		self.command('if [ -e ran_build ]; then cp -rf ran_build lte_build_oai; fi', '\$', 30)
290
		self.close()
291
		self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
292

293 294 295 296 297 298 299
	def InitializeHSS(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			logging.debug('Using the OAI EPC HSS')
300
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
301 302
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
303
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + SSH.testCase_id + '.log &', 'Core state: 2 -> 3', 35)
304 305
		else:
			logging.debug('Using the ltebox simulated HSS')
306 307 308 309
			self.command('if [ -d ' + self.EPCSourceCodePath + '/scripts ]; then echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('mkdir -p ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			self.command('cd /opt/hss_sim0609', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f hss.log daemon.log', '\$', 5)
310 311
			self.command('echo ' + self.EPCPassword + ' | sudo -S echo "Starting sudo session" && sudo daemon --unsafe --name=simulated_hss --chdir=/opt/hss_sim0609 ./starthss_real  ', '\$', 5)
		self.close()
312
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
313

314 315 316 317 318 319
	def InitializeMME(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
320
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
321 322
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
323
			self.command('stdbuf -o0 hostname', '\$', 5)
324 325 326 327 328 329 330
			result = re.search('hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m')
				sys.exit(1)
			host_name = result.group('host_name')
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + SSH.testCase_id + '.log &', 'MME app initialization complete', 100)
		else:
331
			self.command('cd /opt/ltebox/tools', '\$', 5)
332 333
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_mme', '\$', 5)
		self.close()
334
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
335 336 337

	def InitializeSPGW(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
338 339
			Usage()
			sys.exit('Insufficient Parameter')
340 341
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
342
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
343 344 345 346
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + SSH.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30)
		else:
347
			self.command('cd /opt/ltebox/tools', '\$', 5)
348 349
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_xGw', '\$', 5)
		self.close()
350
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
351 352 353

	def InitializeeNB(self):
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
354 355 356
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = True
Raphael Defosseux's avatar
Raphael Defosseux committed
357 358 359
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
360
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
361
			sys.exit(1)
362 363 364 365 366 367 368 369 370 371 372 373
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', str(self.ssh.before))
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
				self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/enb_' + SSH.testCase_id + '_s1log.pcap', '\$', 5)
				self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/enb_' + SSH.testCase_id + '_s1log.pcap > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5)
			self.close()
374 375
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
376 377
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
378 379 380
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
381 382 383 384 385 386 387 388
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				self.command('cd common/utils/T/tracer', '\$', 5)
				self.command('make', '\$', 10)
				self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5)
				self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
389 390 391 392
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
393
		ci_full_config_file = config_path + '/ci-' + config_file
394 395 396 397
		rruCheck = False
		result = re.search('rru', str(config_file))
		if result is not None:
			rruCheck = True
398 399
		# Make a copy and adapt to EPC / eNB IP addresses
		self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
400 401
		self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
402
		# Launch eNB with the modified config file
403 404
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
405 406
		self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh ', '\$', 5)
		self.command('chmod 775 ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh ', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
407
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + SSH.testCase_id + '.log', '\$', 5)
408 409 410
		self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(SSH.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '.log ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh', '\$', 5)
		if not rruCheck:
			self.eNBLogFile = 'enb_' + SSH.testCase_id + '.log'
411 412 413 414 415 416
		time.sleep(6)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
417 418 419 420 421
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.command('killall --signal SIGKILL record', '\$', 5)
				self.close()
422
				doLoop = False
423
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
424
				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
425
				self.CreateHtmlFooter(False)
426 427 428 429 430 431 432 433 434 435 436
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
					self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					self.close()
					time.sleep(1)
					pcap_log_file = 'enb_' + SSH.testCase_id + '_s1log.pcap'
					self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.')
					self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.')
437
				sys.exit(1)
438
			else:
439
				self.command('stdbuf -o0 cat enb_' + SSH.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
440 441 442 443
				if rruCheck:
					result = re.search('wait RUs', str(self.ssh.before))
				else:
					result = re.search('got sync', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
444 445 446 447
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
448
					self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
449
					logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')
450

451 452
		self.close()

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
	def InitializeUE_common(self, device_id):
		logging.debug('send adb commands')
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# The following commands are deprecated since we no longer work on Android 7+
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell settings put global airplane_mode_on 1', '\$', 10)
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', '\$', 60)
			# a dedicated script has to be installed inside the UE
			# airplane mode on means call /data/local/tmp/off
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			#airplane mode off means call /data/local/tmp/on
			logging.debug('\u001B[1mUE (' + device_id + ') Initialize Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def InitializeUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.InitializeUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
481
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
482

483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
	def checkDevTTYisUnlocked(self):
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		count = 0
		while count < 5:
			self.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep ttyUSB0', '\$', 10)
			result = re.search('picocom', str(self.ssh.before))
			if result is None:
				count = 10
			else:
				time.sleep(5)
				count = count + 1
		self.close()

	def InitializeCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		# Checking if auto-attach is enabled
		self.command('AT^AUTOATT?', 'OK', 5)
		result = re.search('AUTOATT: (?P<state>[0-9\-]+)', str(self.ssh.before))
		if result is not None:
			if result.group('state') is not None:
				autoAttachState = int(result.group('state'))
				if autoAttachState is not None:
					if autoAttachState == 0:
						self.command('AT^AUTOATT=1', 'OK', 5)
					logging.debug('\u001B[1m Auto-Attach enabled\u001B[0m')
		else:
			logging.debug('\u001B[1;37;41m Could not check Auto-Attach! \u001B[0m')
		# Force closure of picocom but device might still be locked
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def TerminateCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def AttachCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Enabling the Radio
		self.command('AT+CFUN=1', 'SIMSTORE,READY', 5)
		logging.debug('\u001B[1m Cellular Functionality enabled\u001B[0m')
		time.sleep(4)
		# We should check if we register
		count = 0
		while count < 3:
			self.command('AT+CEREG?', 'OK', 5)
			result = re.search('CEREG: 2,(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is not None:
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState is not None:
					logging.debug('+CEREG: 2,' + str(mDataConnectionState))
			else:
				logging.debug(str(self.ssh.before))
			count = count + 1
			time.sleep(1)
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

Raphael Defosseux's avatar
Raphael Defosseux committed
586
	def AttachUE_common(self, device_id, statusQueue, lock):
587 588 589 590
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
			time.sleep(2)
Raphael Defosseux's avatar
Raphael Defosseux committed
591 592
			max_count = 45
			count = max_count
593 594 595 596 597
			while count > 0:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
598 599 600 601 602 603
					lock.acquire()
					statusQueue.put(-1)
					statusQueue.put(device_id)
					statusQueue.put('mDataConnectionState Not Found!')
					lock.release()
					break
604 605 606
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState == 2:
					logging.debug('\u001B[1mUE (' + device_id + ') Attach Completed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
607 608 609 610 611
					lock.acquire()
					statusQueue.put(max_count - count)
					statusQueue.put(device_id)
					statusQueue.put('Attach Completed')
					lock.release()
612 613 614
					break
				count = count - 1
				if count == 15 or count == 30:
Raphael Defosseux's avatar
Raphael Defosseux committed
615
					logging.debug('\u001B[1;30;43m Retry UE (' + device_id + ') Flight Mode Off \u001B[0m')
616 617 618 619
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
					time.sleep(0.5)
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
					time.sleep(0.5)
Raphael Defosseux's avatar
Raphael Defosseux committed
620
				logging.debug('\u001B[1mWait UE (' + device_id + ') a second until mDataConnectionState=2 (' + str(max_count-count) + ' times)\u001B[0m')
621 622 623
				time.sleep(1)
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Attach Failed \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
624 625 626 627 628
				lock.acquire()
				statusQueue.put(-1)
				statusQueue.put(device_id)
				statusQueue.put('Attach Failed')
				lock.release()
629 630 631 632 633 634 635 636 637
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def AttachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
638 639 640
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
641
			self.AutoTerminateUEandeNB()
642
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
643
			sys.exit(1)
644
		multi_jobs = []
Raphael Defosseux's avatar
Raphael Defosseux committed
645 646
		status_queue = SimpleQueue()
		lock = Lock()
647
		nb_ue_to_connect = 0
648
		for device_id in self.UEDevices:
649 650 651 652 653 654
			if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach):
				p = Process(target = SSH.AttachUE_common, args = (device_id, status_queue, lock,))
				p.daemon = True
				p.start()
				multi_jobs.append(p)
			nb_ue_to_connect = nb_ue_to_connect + 1
655 656
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
Raphael Defosseux committed
657 658

		if (status_queue.empty()):
659
			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
660
			self.CreateHtmlFooter(False)
661
			self.AutoTerminateUEandeNB()
Raphael Defosseux's avatar
Raphael Defosseux committed
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
			sys.exit(1)
		else:
			attach_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					attach_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				if (count < 0):
					html_cell = "<pre>UE (" + device_id + ")\n" + message + "</pre>"
				else:
					html_cell = "<pre>UE (" + device_id + ")\n" + message + ' in ' + str(count + 2) + ' seconds</pre>'
				html_queue.put(html_cell)
			if (attach_status):
				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
679 680 681 682
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					logging.debug('Waiting 5 seconds to fill up record file')
					time.sleep(5)
Raphael Defosseux's avatar
Raphael Defosseux committed
683 684
			else:
				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
685
				self.AutoTerminateUEandeNB()
686
				self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
687
				sys.exit(1)
688 689 690 691

	def DetachUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
692
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
693 694 695 696 697 698 699 700 701 702
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DetachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
703 704 705
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
706
			self.AutoTerminateUEandeNB()
707
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
708
			sys.exit(1)
709 710 711 712 713 714 715 716
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.DetachUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
717
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
718 719 720 721
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			logging.debug('Waiting 5 seconds to fill up record file')
			time.sleep(5)
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766

	def RebootUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			previousmDataConnectionStates = []
			# Save mDataConnectionState
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
				sys.exit(1)
			previousmDataConnectionStates.append(int(result.group('state')))
			# Reboot UE
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell reboot', '\$', 10)
			time.sleep(60)
			previousmDataConnectionState = previousmDataConnectionStates.pop(0)
			count = 180
			while count > 0:
				count = count - 1
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					mDataConnectionState = None
				else:
					mDataConnectionState = int(result.group('state'))
					logging.debug('mDataConnectionState = ' + result.group('state'))
				if mDataConnectionState is None or (previousmDataConnectionState == 2 and mDataConnectionState != 2):
					logging.debug('\u001B[1mWait UE (' + device_id + ') a second until reboot completion (' + str(180-count) + ' times)\u001B[0m')
					time.sleep(1)
				else:
					logging.debug('\u001B[1mUE (' + device_id + ') Reboot Completed\u001B[0m')
					break
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Reboot Failed \u001B[0m')
				sys.exit(1)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def RebootUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
767 768 769
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
770
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
771
			sys.exit(1)
772 773 774 775 776 777 778 779
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.RebootUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
780
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
781 782 783 784 785 786 787 788 789 790 791 792

	def GetAllUEDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('adb devices', '\$', 15)
		self.UEDevices = re.findall("\\\\r\\\\n([A-Za-z0-9]+)\\\\tdevice",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.UEDevices) == 0:
				logging.debug('\u001B[1;37;41m UE Not Found! \u001B[0m')
				sys.exit(1)
793 794
		self.close()

795 796 797 798 799 800 801 802 803 804 805 806 807
	def GetAllCatMDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
		self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.CatMDevices) == 0:
				logging.debug('\u001B[1;37;41m CAT-M UE Not Found! \u001B[0m')
				sys.exit(1)
		self.close()

808 809 810 811
	def GetAllUEIPAddresses(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
812
		ue_ip_status = 0
813 814 815
		self.UEIPAddresses = []
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		for device_id in self.UEDevices:
816 817 818 819 820 821
			count = 0
			while count < 4:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ip addr show | grep rmnet', '\$', 15)
				result = re.search('inet (?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+[0-9a-zA-Z\.\s]+', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
822
					time.sleep(1)
823 824 825 826 827 828
					count += 1
				else:
					count = 10
			if count < 9:
				ue_ip_status -= 1
				continue
829 830 831 832 833
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + device_id + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			for ueipaddress in self.UEIPAddresses:
				if ueipaddress == UE_IPAddress:
					logging.debug('\u001B[1mUE (' + device_id + ') IP Address ' + UE_IPAddress + 'has been existed!' + '\u001B[0m')
834 835
					ue_ip_status -= 1
					continue
836
			self.UEIPAddresses.append(UE_IPAddress)
837
		self.close()
838
		return ue_ip_status
839

Raphael Defosseux's avatar
Raphael Defosseux committed
840 841 842 843 844 845 846 847 848
	def ping_iperf_wrong_exit(self, lock, UE_IPAddress, device_id, statusQueue, message):
		lock.acquire()
		statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
		statusQueue.put(message)
		lock.release()

	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue):
849 850 851 852 853
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
854 855 856 857 858 859 860
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee -a ping_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
				return
861 862
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
Raphael Defosseux committed
863 864
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
865
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
866
				return
867 868
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
Raphael Defosseux's avatar
Raphael Defosseux committed
869 870
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
871
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
872
				return
873 874
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
Raphael Defosseux committed
875 876
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
877
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
878
				return
879 880 881
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
Raphael Defosseux's avatar
Raphael Defosseux committed
882 883 884 885
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
886 887
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
888 889 890 891 892 893
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
894 895
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
Raphael Defosseux's avatar
Raphael Defosseux committed
896
					qMsg += '\nPacket Loss too high'
897
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
898
					packetLossOK = False
899
				elif float(packetloss) > 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
900
					qMsg += '\nPacket Loss is not 0%'
Raphael Defosseux's avatar
Raphael Defosseux committed
901
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
902 903 904 905 906 907 908 909
			if (packetLossOK):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(qMsg)
			lock.release()
910 911 912 913 914 915 916 917 918
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def Ping(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
919 920 921
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
922
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
923
			sys.exit(1)
924 925 926
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
927
			self.CreateHtmlFooter(False)
928
			sys.exit(1)
929 930 931
		multi_jobs = []
		i = 0
		lock = Lock()
Raphael Defosseux's avatar
Raphael Defosseux committed
932
		status_queue = SimpleQueue()
933 934
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]
Raphael Defosseux's avatar
Raphael Defosseux committed
935
			p = Process(target = SSH.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,))
936 937 938 939 940 941
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			i = i + 1
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
Raphael Defosseux committed
942 943

		if (status_queue.empty()):
944
			self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
945
			self.AutoTerminateUEandeNB()
946
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
			sys.exit(1)
		else:
			ping_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					ping_status = False
				device_id = status_queue.get()
				ip_addr = status_queue.get()
				message = status_queue.get()
				html_cell = "<pre>UE (" + device_id + ")\nIP Address  : " + ip_addr + "\n" + message + "</pre>"
				html_queue.put(html_cell)
			if (ping_status):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
964
				self.AutoTerminateUEandeNB()
965
				self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
966
				sys.exit(1)
967

968 969 970 971 972 973 974
	def Iperf_ComputeTime(self):
		result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf time Not Found! \u001B[0m')
			sys.exit(1)
		return result.group('iperf_time')

975
	def Iperf_ComputeModifiedBW(self, idx, ue_num):
976 977 978 979 980
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf bandwidth Not Found! \u001B[0m')
			sys.exit(1)
		iperf_bandwidth = result.group('iperf_bandwidth')
981 982 983 984 985 986 987 988 989 990 991
		if SSH.iperf_profile == 'balanced':
			iperf_bandwidth_new = float(iperf_bandwidth)/ue_num
		if SSH.iperf_profile == 'single-ue':
			iperf_bandwidth_new = float(iperf_bandwidth)
		if SSH.iperf_profile == 'unbalanced':
			# residual is 2% of max bw
			residualBW = float(iperf_bandwidth) / 50
			if idx == 0:
				iperf_bandwidth_new = float(iperf_bandwidth) - ((ue_num - 1) * residualBW)
			else:
				iperf_bandwidth_new = residualBW
992
		iperf_bandwidth_str = '-b ' + iperf_bandwidth
993
		iperf_bandwidth_str_new = '-b ' + ('%.2f' % iperf_bandwidth_new)
994 995 996 997 998 999
		result = re.sub(iperf_bandwidth_str, iperf_bandwidth_str_new, str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Calculate Iperf bandwidth Failed! \u001B[0m')
			sys.exit(1)
		return result

1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
	def Iperf_analyzeV2TCPOutput(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
		self.command('awk -f /tmp/tcp_iperf_stats.awk /tmp/CI-eNB/scripts/iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
		result = re.search('Avg Bitrate : (?P<average>[0-9\.]+ Mbits\/sec) Max Bitrate : (?P<maximum>[0-9\.]+ Mbits\/sec) Min Bitrate : (?P<minimum>[0-9\.]+ Mbits\/sec)', str(self.ssh.before))
		if result is not None:
			avgbitrate = result.group('average')
			maxbitrate = result.group('maximum')
			minbitrate = result.group('minimum')
			lock.acquire()
			logging.debug('\u001B[1;37;44m TCP iperf result (' + UE_IPAddress + ') \u001B[0m')
			msg = 'TCP Stats   :\n'
			if avgbitrate is not None:
				logging.debug('\u001B[1;34m    Avg Bitrate : ' + avgbitrate + '\u001B[0m')
				msg += 'Avg Bitrate : ' + avgbitrate + '\n'
			if maxbitrate is not None:
				logging.debug('\u001B[1;34m    Max Bitrate : ' + maxbitrate + '\u001B[0m')
				msg += 'Max Bitrate : ' + maxbitrate + '\n'
			if minbitrate is not None:
				logging.debug('\u001B[1;34m    Min Bitrate : ' + minbitrate + '\u001B[0m')
				msg += 'Min Bitrate : ' + minbitrate + '\n'
			statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
			lock.release()
		return 0

1026
	def Iperf_analyzeV2Output(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
1027 1028 1029 1030
		result = re.search('-u', str(iperf_real_options))
		if result is None:
			return self.Iperf_analyzeV2TCPOutput(lock, UE_IPAddress, device_id, statusQueue, iperf_real_options)

1031 1032 1033 1034 1035 1036 1037
		result = re.search('Server Report:', str(self.ssh.before))
		if result is None:
			result = re.search('read failed: Connection refused', str(self.ssh.before))
			if result is not None:
				logging.debug('\u001B[1;37;41m Could not connect to iperf server! \u001B[0m')
			else:
				logging.debug('\u001B[1;37;41m Server Report and Connection refused Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1038
			return -1
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is not None:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

1057
		result = re.search('Server Report:\\\\r\\\\n(?:|\[ *\d+\].*) (?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(\d+\/..\d+) (\((?P<packetloss>[0-9\.]+)%\))', str(self.ssh.before))
1058 1059 1060 1061 1062 1063
		if result is not None:
			bitrate = result.group('bitrate')
			packetloss = result.group('packetloss')
			jitter = result.group('jitter')
			lock.acquire()
			logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1064
			iperfStatus = True
1065 1066
			msg = 'Req Bitrate : ' + req_bandwidth + '\n'
			logging.debug('\u001B[1;34m    Req Bitrate : ' + req_bandwidth + '\u001B[0m')
1067
			if bitrate is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1068
				msg += 'Bitrate     : ' + bitrate + '\n'
1069
				logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
				result = re.search('(?P<real_bw>[0-9\.]+) [KMG]bits/sec', str(bitrate))
				if result is not None:
					actual_bw = float(str(result.group('real_bw')))
					result = re.search('[0-9\.]+ K', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000
					result = re.search('[0-9\.]+ M', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000
					result = re.search('[0-9\.]+ G', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000000
					br_loss = 100 * actual_bw / req_bw
					bitperf = '%.2f ' % br_loss
					msg += 'Bitrate Perf: ' + bitperf + '%\n'
					logging.debug('\u001B[1;34m    Bitrate Perf: ' + bitperf + '%\u001B[0m')
1086
			if packetloss is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1087
				msg += 'Packet Loss : ' + packetloss + '%\n'
1088 1089
				logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
				if float(packetloss) > float(self.iperf_packetloss_threshold):
Raphael Defosseux's avatar
Raphael Defosseux committed
1090
					msg += 'Packet Loss too high!\n'
1091
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1092
					iperfStatus = False
1093
			if jitter is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1094
				msg += 'Jitter      : ' + jitter + '\n'
1095
				logging.debug('\u001B[1;34m    Jitter      : ' + jitter + '\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1096 1097 1098 1099 1100 1101 1102
			if (iperfStatus):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
1103
			lock.release()
Raphael Defosseux's avatar
Raphael Defosseux committed
1104 1105 1106 1107
			return 0

	def Iperf_analyzeV2Server(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
		if (not os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
Raphael Defosseux's avatar
Raphael Defosseux committed
1108
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
1109 1110 1111 1112 1113
			return
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is None:
			logging.debug('Iperf bandwidth Not Found!')
Raphael Defosseux's avatar
Raphael Defosseux committed
1114
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not compute Iperf bandwidth!')
Raphael Defosseux's avatar
Raphael Defosseux committed
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
			return
		else:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

		server_file = open('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', 'r')
		br_sum = 0.0
		ji_sum = 0.0
		pl_sum = 0
		ps_sum = 0
		row_idx = 0
		for line in server_file.readlines():
			result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line))
			if result is not None:
				bitrate = result.group('bitrate')
				jitter = result.group('jitter')
				packetlost = result.group('lostPack')
				packetsent = result.group('sentPack')
				br = bitrate.split(' ')
				ji = jitter.split(' ')
				row_idx = row_idx + 1
				curr_br = float(br[0])
				pl_sum = pl_sum + int(packetlost)
				ps_sum = ps_sum + int(packetsent)
				if (br[1] == 'Kbits/sec'):
					curr_br = curr_br * 1000
				if (br[1] == 'Mbits/sec'):
					curr_br = curr_br * 1000 * 1000
				br_sum = curr_br + br_sum
				ji_sum = float(ji[0]) + ji_sum
		if (row_idx > 0):
			br_sum = br_sum / row_idx
			ji_sum = ji_sum / row_idx
			br_loss = 100 * br_sum / req_bw
			if (br_sum > 1000):
				br_sum = br_sum / 1000
				if (br_sum > 1000):
					br_sum = br_sum / 1000
					bitrate = '%.2f Mbits/sec' % br_sum
				else:
					bitrate = '%.2f Kbits/sec' % br_sum
			else:
				bitrate = '%.2f bits/sec' % br_sum
			bitperf = '%.2f ' % br_loss
			bitperf += '%'
			jitter = '%.2f ms' % (ji_sum)
			if (ps_sum > 0):
				pl = float(100 * pl_sum / ps_sum)
				packetloss = '%2.1f ' % (pl)
				packetloss += '%'
			else:
				packetloss = 'unknown'
			lock.acquire()
			if (br_loss < 90):
				statusQueue.put(1)
			else:
				statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			req_msg = 'Req Bitrate : ' + req_bandwidth
			bir_msg = 'Bitrate     : ' + bitrate
			brl_msg = 'Bitrate Perf: ' + bitperf
			jit_msg = 'Jitter      : ' + jitter
			pal_msg = 'Packet Loss : ' + packetloss
			statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n')
			logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
			logging.debug('\u001B[1;35m    ' + req_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + bir_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + brl_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + jit_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + pal_msg + '\u001B[0m')
			lock.release()
		else:
Raphael Defosseux's avatar
Raphael Defosseux committed
1200
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
1201 1202 1203

		server_file.close()

1204

Raphael Defosseux's avatar
Raphael Defosseux committed
1205
	def Iperf_analyzeV3Output(self, lock, UE_IPAddress, device_id, statusQueue):
1206 1207 1208
		result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?:|[0-9\.]+ ms +\d+\/\d+ \((?P<packetloss>[0-9\.]+)%\)) +(?:|receiver)\\\\r\\\\n(?:|\[ *\d+\] Sent \d+ datagrams)\\\\r\\\\niperf Done\.', str(self.ssh.before))
		if result is None:
			result = re.search('(?P<error>iperf: error - [a-zA-Z0-9 :]+)', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
1209 1210 1211 1212
			lock.acquire()
			statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
1213 1214
			if result is not None:
				logging.debug('\u001B[1;37;41m ' + result.group('error') + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1215
				statusQueue.put(result.group('error'))
1216 1217
			else:
				logging.debug('\u001B[1;37;41m Bitrate and/or Packet Loss Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1218 1219 1220
				statusQueue.put('Bitrate and/or Packet Loss Not Found!')
			lock.release()

1221 1222 1223 1224 1225
		bitrate = result.group('bitrate')
		packetloss = result.group('packetloss')
		lock.acquire()
		logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
		logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1226 1227
		msg = 'Bitrate     : ' + bitrate + '\n'
		iperfStatus = True
1228 1229
		if packetloss is not None:
			logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1230
			msg += 'Packet Loss : ' + packetloss + '%\n'
1231 1232
			if float(packetloss) > float(self.iperf_packetloss_threshold):
				logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1233 1234 1235 1236 1237 1238 1239 1240
				msg += 'Packet Loss too high!\n'
				iperfStatus = False
		if (iperfStatus):
			statusQueue.put(0)
		else:
			statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
Raphael Defosseux's avatar
Raphael Defosseux committed
1241
		statusQueue.put(msg)
1242 1243
		lock.release()

Raphael Defosseux's avatar
Raphael Defosseux committed
1244
	def Iperf_UL_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
1245 1246 1247 1248
		udpIperf = True
		result = re.search('-u', str(self.iperf_args))
		if result is None:
			udpIperf = False
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
		ipnumbers = UE_IPAddress.split('.')
		if (len(ipnumbers) == 4):
			ipnumbers[3] = '1'
		EPC_Iperf_UE_IPAddress = ipnumbers[0] + '.' + ipnumbers[1] + '.' + ipnumbers[2] + '.' + ipnumbers[3]

		# Launch iperf server on EPC side
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
		self.command('rm -f iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
		port = 5001 + idx
1259 1260 1261 1262
		if udpIperf:
			self.command('echo $USER; nohup iperf -u -s -i 1 -p ' + str(port) + ' > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
		else:
			self.command('echo $USER; nohup iperf -s -i 1 -p ' + str(port) + ' > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
1263 1264 1265 1266 1267 1268 1269 1270 1271
		time.sleep(0.5)
		self.close()

		# Launch iperf client on UE
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
		iperf_time = self.Iperf_ComputeTime()
		time.sleep(0.5)

1272 1273 1274 1275
		if udpIperf:
			modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
		else:
			modified_options = str(self.iperf_args)
Raphael Defosseux's avatar
Raphael Defosseux committed
1276
		modified_options = modified_options.replace('-R','')
1277 1278 1279
		time.sleep(0.5)

		self.command('rm -f iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1280 1281 1282 1283 1284 1285 1286 1287
		iperf_status = self.command('stdbuf -o0 adb -s ' + device_id + ' shell "/data/local/tmp/iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + '" 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
		# TIMEOUT Case
		if iperf_status < 0:
			self.close()
			message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
			logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
			return
1288
		clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
Raphael Defosseux's avatar
Raphael Defosseux committed
1289
		self.close()
1290

Raphael Defosseux's avatar
Raphael Defosseux committed
1291
		# Kill iperf server on EPC side
1292 1293 1294
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('killall --signal SIGKILL iperf', self.EPCUserName, 5)
		self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1295
		# in case of failure, retrieve server log
Raphael Defosseux's avatar
Raphael Defosseux committed
1296 1297 1298 1299
		if (clientStatus == -1):
			time.sleep(1)
			if (os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
				os.remove('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')
1300
			self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
1301
			self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1302

Raphael Defosseux's avatar
Raphael Defosseux committed
1303
	def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
1304
		try:
1305 1306 1307
			# Single-UE profile -- iperf only on one UE
			if SSH.iperf_profile == 'single-ue' and idx != 0:
				return
1308
			useIperf3 = False
1309
			udpIperf = True
1310
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
Raphael Defosseux's avatar
Raphael Defosseux committed
1311
			# if by chance ADB server and EPC are on the same remote host, at least log collection will take care of it
Raphael Defosseux's avatar
Raphael Defosseux committed
1312 1313
			self.command('if [ ! -d ' + self.EPCSourceCodePath + '/scripts ]; then mkdir -p ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
1314 1315 1316 1317 1318 1319
			# Checking if iperf / iperf3 are installed
			self.command('adb -s ' + device_id + ' shell "ls /data/local/tmp"', '\$', 5)
			result = re.search('iperf3', str(self.ssh.before))
			if result is None:
				result = re.search('iperf', str(self.ssh.before))
				if result is None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1320 1321
					message = 'Neither iperf nor iperf3 installed on UE!'
					logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1322
					self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
1323
					return
1324 1325
			else:
				useIperf3 = True
1326 1327 1328 1329 1330
			# in case of iperf, UL has its own function
			if (not useIperf3):
				result = re.search('-R', str(self.iperf_args))
				if result is not None:
					self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1331
					self.Iperf_UL_common(lock, UE_IPAddress, device_id, idx, ue_num, statusQueue)
1332 1333
					return

1334 1335 1336
			if (useIperf3):
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/iperf3 -s &', '\$', 5)
			else:
1337
				self.command('rm -f iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
1338 1339 1340 1341 1342 1343
				result = re.search('-u', str(self.iperf_args))
				if result is None:
					self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -s -i 1" > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
					udpIperf = False
				else:
					self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -u -s -i 1" > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
1344 1345 1346 1347
			time.sleep(0.5)
			self.close()

			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
1348 1349
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			iperf_time = self.Iperf_ComputeTime()
1350 1351
			time.sleep(0.5)

1352 1353 1354 1355
			if udpIperf:
				modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
			else:
				modified_options = str(self.iperf_args)
1356 1357
			time.sleep(0.5)

1358 1359
			self.command('rm -f iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
			if (useIperf3):
1360
				self.command('stdbuf -o0 iperf3 -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
1361

Raphael Defosseux's avatar
Raphael Defosseux committed
1362
				clientStatus = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
1363
				self.Iperf_analyzeV3Output(lock, UE_IPAddress, device_id, statusQueue)
1364
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
1365 1366 1367 1368 1369 1370 1371
				iperf_status = self.command('stdbuf -o0 iperf -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
				if iperf_status < 0:
					self.close()
					message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
					logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
					self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
					return
1372
				clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1373 1374 1375
			self.close()

			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1376
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell ps | grep --color=never iperf | grep -v grep', '\$', 5)
1377 1378 1379 1380 1381
			result = re.search('shell +(?P<pid>\d+)', str(self.ssh.before))
			if result is not None:
				pid_iperf = result.group('pid')
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell kill -KILL ' + pid_iperf, '\$', 5)
			self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1382 1383 1384 1385
			if (clientStatus == -1):
				time.sleep(1)
				if (os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
					os.remove('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')
1386
				self.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
1387
				self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1388 1389 1390 1391 1392 1393 1394 1395
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def Iperf(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1396 1397 1398
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
1399
			self.AutoTerminateUEandeNB()
1400
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1401
			sys.exit(1)
1402 1403
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
1404
			self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
1405
			self.AutoTerminateUEandeNB()
1406
			self.CreateHtmlFooter(False)
1407
			sys.exit(1)
1408 1409 1410 1411
		multi_jobs = []
		i = 0
		ue_num = len(self.UEIPAddresses)
		lock = Lock()
Raphael Defosseux's avatar
Raphael Defosseux committed
1412
		status_queue = SimpleQueue()
1413 1414
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]