main.py 166 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Version
#-----------------------------------------------------------
Version = '0.1'

35 36 37 38 39 40 41 42 43
#-----------------------------------------------------------
# Constants
#-----------------------------------------------------------
ALL_PROCESSES_OK = 0
ENB_PROCESS_FAILED = -1
ENB_PROCESS_OK = +1
ENB_PROCESS_SEG_FAULT = -11
ENB_PROCESS_ASSERTION = -12
ENB_PROCESS_REALTIME_ISSUE = -13
44
ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
45 46 47 48 49 50 51
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
MME_PROCESS_OK = +3
SPGW_PROCESS_FAILED = -4
SPGW_PROCESS_OK = +4
UE_IP_ADDRESS_ISSUE = -5
52 53 54 55 56
OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20
OAI_UE_PROCESS_COULD_NOT_SYNC = -21
OAI_UE_PROCESS_ASSERTION = -22
OAI_UE_PROCESS_FAILED = -6
OAI_UE_PROCESS_OK = +6
57

58 59 60 61 62 63 64 65
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect		# pexpect
import time		# sleep
import os
66
import subprocess
67 68 69 70
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
Raphael Defosseux's avatar
Raphael Defosseux committed
71
from multiprocessing import Process, Lock, SimpleQueue
72
logging.basicConfig(
73 74
	level=logging.DEBUG,
	format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
75 76 77 78 79 80 81 82 83 84
)

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
	def __init__(self):
		self.eNBIPAddress = ''
		self.eNBRepository = ''
		self.eNBBranch = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
85
		self.eNB_AllowMerge = False
86
		self.eNBCommitID = ''
87
		self.eNBTargetBranch = ''
88 89 90 91 92 93 94 95
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.EPCIPAddress = ''
		self.EPCUserName = ''
		self.EPCPassword = ''
		self.EPCSourceCodePath = ''
		self.EPCType = ''
96
		self.EPC_PcapFileName = ''
97 98 99 100
		self.ADBIPAddress = ''
		self.ADBUserName = ''
		self.ADBPassword = ''
		self.testCase_id = ''
101 102
		self.testXMLfiles = []
		self.nbTestXMLfiles = 0
103 104 105
		self.desc = ''
		self.Build_eNB_args = ''
		self.Initialize_eNB_args = ''
106
		self.eNBLogFile = ''
107
		self.eNB_instance = ''
108 109
		self.eNBOptions = ''
		self.rruOptions = ''
110
		self.rruLogFile = ''
111 112 113 114
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
115
		self.iperf_profile = ''
116
		self.nbMaxUEtoAttach = -1
117
		self.UEDevices = []
118
		self.CatMDevices = []
119
		self.UEIPAddresses = []
Raphael Defosseux's avatar
Raphael Defosseux committed
120 121 122
		self.htmlFile = ''
		self.htmlHeaderCreated = False
		self.htmlFooterCreated = False
123
		self.htmlUEConnected = -1
124
		self.htmleNBFailureMsg = ''
125
		self.htmlUEFailureMsg = ''
126
		self.picocom_closure = False
127 128 129 130 131
		self.idle_sleep_time = 0
		self.htmlTabRefs = []
		self.htmlTabNames = []
		self.htmlTabIcons = []
		self.finalStatus = False
132 133 134
		self.OsVersion = ''
		self.KernelVersion = ''
		self.UhdVersion = ''
135
		self.UsrpBoard = ''
136 137 138
		self.CpuNb = ''
		self.CpuModel = ''
		self.CpuMHz = ''
139 140 141
		self.UEIPAddress = ''
		self.UEUserName = ''
		self.UEPassword = ''
Boris Djalal's avatar
Boris Djalal committed
142
		self.UE_instance = ''
143
		self.UESourceCodePath = ''
144
		self.UELogFile = ''
Boris Djalal's avatar
Boris Djalal committed
145 146
		self.Build_OAI_UE_args = ''
		self.Initialize_OAI_UE_args = ''
147

148
	def open(self, ipaddress, username, password):
149 150 151 152 153
		count = 0
		connect_status = False
		while count < 4:
			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
154
			if self.sshresponse == 0:
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
				self.ssh.sendline('yes')
				self.ssh.expect('password:')
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 1:
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 2:
				# Checking if we are really on the remote client defined by its IP address
				self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:"', '\$', 5)
				result = re.search(str(ipaddress), str(self.ssh.before))
				if result is None:
					self.close()
				else:
					count = 10
					connect_status = True
181
			else:
182 183
				# debug output
				logging.debug(str(self.ssh.before))
184
				logging.debug('self.sshresponse = ' + str(self.sshresponse))
Raphael Defosseux's avatar
Raphael Defosseux committed
185 186 187
			# adding a tempo when failure
			if not connect_status:
				time.sleep(1)
188 189
			count += 1
		if connect_status:
190 191
			pass
		else:
192
			sys.exit('SSH Connection Failed')
193

194 195 196 197 198 199
	def command(self, commandline, expectedline, timeout):
		logging.debug(commandline)
		self.ssh.timeout = timeout
		self.ssh.sendline(commandline)
		self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
200
			return 0
201 202 203 204 205 206 207
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
		elif self.sshresponse == 2:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
208
			result = re.search('ping |iperf |picocom', str(commandline))
Raphael Defosseux's avatar
Raphael Defosseux committed
209
			if result is None:
210
				logging.debug(str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
211 212 213
				sys.exit(self.sshresponse)
			else:
				return -1
214 215 216 217
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
218

219 220 221 222 223 224 225
	def close(self):
		self.ssh.timeout = 5
		self.ssh.sendline('exit')
		self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
			pass
		elif self.sshresponse == 1:
226 227
			if not self.picocom_closure:
				logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m')
228
		else:
229
			logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m')
230

231
	def copyin(self, ipaddress, username, password, source, destination):
232 233
		count = 0
		copy_status = False
Raphael Defosseux's avatar
Raphael Defosseux committed
234
		logging.debug('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
235
		while count < 10:
236
			scp_spawn = pexpect.spawn('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 100)
237
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
Raphael Defosseux's avatar
Raphael Defosseux committed
238
			if scp_response == 0:
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
Raphael Defosseux's avatar
Raphael Defosseux committed
259
			else:
260 261 262 263 264 265
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
266
			return 0
Raphael Defosseux's avatar
Raphael Defosseux committed
267
		else:
268
			return -1
Raphael Defosseux's avatar
Raphael Defosseux committed
269

270
	def copyout(self, ipaddress, username, password, source, destination):
271 272
		count = 0
		copy_status = False
273
		logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
274
		while count < 4:
275
			scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 100)
276
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
277
			if scp_response == 0:
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
298
			else:
299 300 301 302 303 304
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
305 306 307 308
			pass
		else:
			sys.exit('SCP failed')

309 310 311 312 313 314 315
	def BuildeNB(self):
		if self.eNBIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
316 317
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
318 319
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
320
		self.command('echo ' + self.eNBPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
321 322 323 324 325
		# if the commit ID is provided use it to point to it
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
Raphael Defosseux's avatar
Raphael Defosseux committed
326
		if (self.eNB_AllowMerge):
327 328 329 330 331 332
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
333 334
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
335 336
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
337
		# no need to remove in log (git clean did the trick)
338
		self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests', 600)
339 340 341
		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.command('mv compile_oai_enb.log ' + 'build_log_' + self.testCase_id, '\$', 5)
342
		self.close()
343
		self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
344

345
	def BuildOAIUE(self):
Boris Djalal's avatar
Boris Djalal committed
346
		if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
347 348 349
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
350 351
		self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5)
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
352
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
353 354 355 356 357
		# here add a check if git clone or git fetch went smoothly
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
		self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
Boris Djalal's avatar
Boris Djalal committed
358 359
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
360 361
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
Boris Djalal's avatar
Boris Djalal committed
362 363 364
		if (self.eNB_AllowMerge):
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
365 366
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
Boris Djalal's avatar
Boris Djalal committed
367 368
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
369 370 371 372 373
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
374
		self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee compile_oai_ue.log', 'Bypassing the Tests', 600)
375 376 377 378
		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.close()
379
		self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
380

381

382 383 384 385 386 387 388
	def InitializeHSS(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			logging.debug('Using the OAI EPC HSS')
389
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
390 391
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
392
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + self.testCase_id + '.log &', 'Core state: 2 -> 3', 35)
393 394
		else:
			logging.debug('Using the ltebox simulated HSS')
395 396 397 398
			self.command('if [ -d ' + self.EPCSourceCodePath + '/scripts ]; then echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('mkdir -p ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			self.command('cd /opt/hss_sim0609', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f hss.log daemon.log', '\$', 5)
399 400
			self.command('echo ' + self.EPCPassword + ' | sudo -S echo "Starting sudo session" && sudo daemon --unsafe --name=simulated_hss --chdir=/opt/hss_sim0609 ./starthss_real  ', '\$', 5)
		self.close()
401
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
402

403 404 405 406 407 408
	def InitializeMME(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
409
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
410 411
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
412
			self.command('stdbuf -o0 hostname', '\$', 5)
413 414 415 416 417
			result = re.search('hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m')
				sys.exit(1)
			host_name = result.group('host_name')
418
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + self.testCase_id + '.log &', 'MME app initialization complete', 100)
419
		else:
420
			self.command('cd /opt/ltebox/tools', '\$', 5)
421 422
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_mme', '\$', 5)
		self.close()
423
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
424 425 426

	def InitializeSPGW(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
427 428
			Usage()
			sys.exit('Insufficient Parameter')
429 430
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
431
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
432 433
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
434
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + self.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30)
435
		else:
436
			self.command('cd /opt/ltebox/tools', '\$', 5)
437 438
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_xGw', '\$', 5)
		self.close()
439
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
440 441 442

	def InitializeeNB(self):
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
443 444
			Usage()
			sys.exit('Insufficient Parameter')
445 446 447 448 449 450 451
		check_eNB = False
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
452 453 454 455 456 457 458 459 460
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', str(self.ssh.before))
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
461
				self.EPC_PcapFileName = 'enb_' + self.testCase_id + '_s1log.pcap'
462 463
				self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/' + self.EPC_PcapFileName, '\$', 5)
				self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/' + self.EPC_PcapFileName + ' > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5)
464
			self.close()
465 466
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
467 468
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
469
		extra_options = ''
470 471 472
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
473 474 475 476 477 478
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				self.command('cd common/utils/T/tracer', '\$', 5)
				self.command('make', '\$', 10)
479
				self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5)
480
				self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
481 482 483 484
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
485
		ci_full_config_file = config_path + '/ci-' + config_file
486
		rruCheck = False
487
		result = re.search('rru|du', str(config_file))
488 489
		if result is not None:
			rruCheck = True
Raphael Defosseux's avatar
Raphael Defosseux committed
490
		# do not reset board twice in IF4.5 case
491
		result = re.search('rru|enb|du', str(config_file))
Raphael Defosseux's avatar
Raphael Defosseux committed
492
		if result is not None:
493
			self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 10)
Raphael Defosseux's avatar
Raphael Defosseux committed
494 495 496
			result = re.search('type: b200', str(self.ssh.before))
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
497
				self.command('echo ' + self.eNBPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
Raphael Defosseux's avatar
Raphael Defosseux committed
498
				# Reloading FGPA bin firmware
499
				self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 15)
500 501
		# Make a copy and adapt to EPC / eNB IP addresses
		self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
502 503
		self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
504
		# Launch eNB with the modified config file
505 506
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
507 508
		self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
509 510
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
		self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
511
		if not rruCheck:
512
			self.eNBLogFile = 'enb_' + self.testCase_id + '.log'
513 514
			if extra_options != '':
				self.eNBOptions = extra_options
515 516 517
		result = re.search('rru|du', str(config_file))
		if result is not None:
			self.rruLogFile = 'enb_' + self.testCase_id + '.log'
518 519 520 521 522 523
		time.sleep(6)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
524 525 526 527 528
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.command('killall --signal SIGKILL record', '\$', 5)
				self.close()
529
				doLoop = False
530
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
531
				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
532
				self.CreateHtmlTabFooter(False)
533 534 535 536 537 538
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
					self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
539
					if self.EPC_PcapFileName != '':
540
						time.sleep(0.5)
541
						self.command('echo ' + self.EPCPassword + ' | sudo -S chmod 666 /tmp/' + self.EPC_PcapFileName, '\$', 5)
542 543
					self.close()
					time.sleep(1)
544 545 546 547
					if self.EPC_PcapFileName != '':
						copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
						if (copyin_res == 0):
							self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.EPC_PcapFileName, self.eNBSourceCodePath + '/cmake_targets/.')
548
				sys.exit(1)
549
			else:
550
				self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
551 552 553
				if rruCheck:
					result = re.search('wait RUs', str(self.ssh.before))
				else:
554
					result = re.search('got sync|Starting F1AP at CU', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
555 556 557 558
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
559 560
					if rruCheck and extra_options != '':
						self.rruOptions = extra_options
561
					self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
562
					logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')
563
					time.sleep(10)
564

565 566
		self.close()

567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588
	def InitializeUE_common(self, device_id):
		logging.debug('send adb commands')
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# The following commands are deprecated since we no longer work on Android 7+
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell settings put global airplane_mode_on 1', '\$', 10)
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', '\$', 60)
			# a dedicated script has to be installed inside the UE
			# airplane mode on means call /data/local/tmp/off
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			#airplane mode off means call /data/local/tmp/on
			logging.debug('\u001B[1mUE (' + device_id + ') Initialize Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def InitializeUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
589
			p = Process(target = self.InitializeUE_common, args = (device_id,))
590 591 592 593 594
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
595
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
596

597 598 599 600
	def InitializeOAIUE(self):
		if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
601 602 603 604 605 606 607 608 609
		result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
		if result is None:
			check_eNB = True
			check_OAI_UE = False
			pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
			if (pStatus < 0):
				self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus)
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
610
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
611
		# b2xx_fx3_utils reset procedure
612
		self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 10)
613 614 615
		result = re.search('type: b200', str(self.ssh.before))
		if result is not None:
			logging.debug('Found a B2xx device --> resetting it')
616
			self.command('echo ' + self.UEPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
617
			# Reloading FGPA bin firmware
618
			self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 15)
619
		else:
620
			logging.debug('Did not find any B2xx device')
621
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
622
		self.command('source oaienv', '\$', 5)
Boris Djalal's avatar
Boris Djalal committed
623
		self.command('cd cmake_targets/lte_build_oai/build', '\$', 5)
624 625 626 627 628 629 630
		result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
		# We may have to regenerate the .u* files
		if result is None:
			self.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf .u*', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S ../../../targets/bin/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5)
		# Launch UE with the modified config file
Boris Djalal's avatar
Boris Djalal committed
631 632 633
		self.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.UELogFile = 'ue_' + self.testCase_id + '.log'
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657

		# We are now looping several times to hope we really sync w/ an eNB
		doOutterLoop = True
		outterLoopCounter = 5
		gotSyncStatus = True
		fullSyncStatus = True
		while (doOutterLoop):
			self.command('cd ' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5)
			self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
			time.sleep(6)
			self.command('cd ../..', '\$', 5)
			doLoop = True
			loopCounter = 10
			gotSyncStatus = True
			# the 'got sync' message is for the UE threads synchronization
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					# Here should never occur
					logging.error('"got sync" message never showed!')
					gotSyncStatus = False
					doLoop = False
					continue
Boris Djalal's avatar
Boris Djalal committed
658 659
				self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
				result = re.search('got sync', str(self.ssh.before))
660 661 662 663
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
					logging.debug('Found "got sync" message!')
			if gotSyncStatus == False:
				# we certainly need to stop the lte-uesoftmodem process if it is still running!
				self.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
				result = re.search('lte-uesoftmodem', str(self.ssh.before))
				if result is not None:
					self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
					time.sleep(3)
			# We are now checking if sync w/ eNB DOES NOT OCCUR
			# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
			doLoop = True
			loopCounter = 10
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					# Here we do have a great chance that the UE did cell-sync w/ eNB
					doLoop = False
					doOutterLoop = False
					fullSyncStatus = True
					continue
				self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
				result = re.search('No cell synchronization found', str(self.ssh.before))
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					fullSyncStatus = False
					logging.debug('Found: "No cell synchronization" message! --> try again')
					time.sleep(6)
					self.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
					result = re.search('lte-uesoftmodem', str(self.ssh.before))
					if result is not None:
						self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
			outterLoopCounter = outterLoopCounter - 1
			if (outterLoopCounter == 0):
				doOutterLoop = False

		if fullSyncStatus and gotSyncStatus:
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			if result is None:
				self.command('ifconfig oaitun_ue1', '\$', 4)
				result = re.search('inet addr', str(self.ssh.before))
				if result is not None:
					logging.debug('\u001B[1m oaitun_ue1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_ue1 interface is either NOT mounted or NOT configured\u001B[0m')

711
		self.close()
712 713 714
		# For the moment we are always OK!!!
		self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
		logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m')
715

716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
	def checkDevTTYisUnlocked(self):
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		count = 0
		while count < 5:
			self.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep ttyUSB0', '\$', 10)
			result = re.search('picocom', str(self.ssh.before))
			if result is None:
				count = 10
			else:
				time.sleep(5)
				count = count + 1
		self.close()

	def InitializeCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		# Checking if auto-attach is enabled
		self.command('AT^AUTOATT?', 'OK', 5)
		result = re.search('AUTOATT: (?P<state>[0-9\-]+)', str(self.ssh.before))
		if result is not None:
			if result.group('state') is not None:
				autoAttachState = int(result.group('state'))
				if autoAttachState is not None:
					if autoAttachState == 0:
						self.command('AT^AUTOATT=1', 'OK', 5)
					logging.debug('\u001B[1m Auto-Attach enabled\u001B[0m')
		else:
			logging.debug('\u001B[1;37;41m Could not check Auto-Attach! \u001B[0m')
		# Force closure of picocom but device might still be locked
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def TerminateCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def AttachCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Enabling the Radio
		self.command('AT+CFUN=1', 'SIMSTORE,READY', 5)
		logging.debug('\u001B[1m Cellular Functionality enabled\u001B[0m')
		time.sleep(4)
		# We should check if we register
		count = 0
803 804 805
		attach_cnt = 0
		attach_status = False
		while count < 5:
806
			self.command('AT+CEREG?', 'OK', 5)
807
			result = re.search('CEREG: 2,(?P<state>[0-9\-]+),', str(self.ssh.before))
808 809 810
			if result is not None:
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState is not None:
811 812 813 814 815 816 817 818 819 820 821 822 823
					if mDataConnectionState == 1:
						count = 10
						attach_status = True
						result = re.search('CEREG: 2,1,"(?P<networky>[0-9A-Z]+)","(?P<networkz>[0-9A-Z]+)"', str(self.ssh.before))
						if result is not None:
							networky = result.group('networky')
							networkz = result.group('networkz')
							logging.debug('\u001B[1m CAT-M module attached to eNB (' + str(networky) + '/' + str(networkz) + ')\u001B[0m')
						else:
							logging.debug('\u001B[1m CAT-M module attached to eNB\u001B[0m')
					else:
						logging.debug('+CEREG: 2,' + str(mDataConnectionState))
						attach_cnt = attach_cnt + 1
824 825
			else:
				logging.debug(str(self.ssh.before))
826
				attach_cnt = attach_cnt + 1
827 828
			count = count + 1
			time.sleep(1)
829 830 831 832 833 834 835 836 837
		if attach_status:
			self.command('AT+CESQ', 'OK', 5)
			result = re.search('CESQ: 99,99,255,255,(?P<rsrq>[0-9]+),(?P<rsrp>[0-9]+)', str(self.ssh.before))
			if result is not None:
				nRSRQ = int(result.group('rsrq'))
				nRSRP = int(result.group('rsrp'))
				if (nRSRQ is not None) and (nRSRP is not None):
					logging.debug('    RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB')
					logging.debug('    RSRP = ' + str(-140+nRSRP) + ' dBm')
838 839
		self.close()
		self.picocom_closure = False
840
		html_queue = SimpleQueue()
841
		self.checkDevTTYisUnlocked()
842
		if attach_status:
843
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Completed in ' + str(attach_cnt+4) + ' seconds'
844 845 846 847 848 849 850 851
			if (nRSRQ is not None) and (nRSRP is not None):
				html_cell += '\n   RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB'
				html_cell += '\n   RSRP = ' + str(-140+nRSRP) + ' dBm</pre>'
			else:
				html_cell += '</pre>'
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'OK', 1, html_queue)
		else:
852
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Failed</pre>'
853 854 855 856 857 858 859
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)

	def PingCatM(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
860 861 862
		check_eNB = True
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
		try:
			statusQueue = SimpleQueue()
			lock = Lock()
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				logging.debug('Using the OAI EPC HSS: not implemented yet')
				self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
			else:
				self.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5)
				result = re.search('Allocated ipv4 addr: (?P<ipaddr>[0-9\.]+) from Pool', str(self.ssh.before))
				if result is not None:
					moduleIPAddr = result.group('ipaddr')
				else:
					return
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
			device_id = 'catm'
887
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + str(moduleIPAddr) + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
888 889 890 891
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(moduleIPAddr) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
892
				self.close()
893 894 895 896 897 898
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
899
				self.close()
900 901 902 903 904 905
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
906
				self.close()
907 908 909 910 911 912
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
913
				self.close()
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + moduleIPAddr + ') \u001B[0m')
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
					qMsg += '\nPacket Loss too high'
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
					packetLossOK = False
				elif float(packetloss) > 0:
					qMsg += '\nPacket Loss is not 0%'
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
			lock.release()
			self.close()
			html_cell = '<pre style="background-color:white">CAT-M module\nIP Address  : ' + moduleIPAddr + '\n' + qMsg + '</pre>'
			statusQueue.put(html_cell)
			if (packetLossOK):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
				self.AutoTerminateUEandeNB()
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
952

Raphael Defosseux's avatar
Raphael Defosseux committed
953
	def AttachUE_common(self, device_id, statusQueue, lock):
954 955 956 957
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
			time.sleep(2)
Raphael Defosseux's avatar
Raphael Defosseux committed
958 959
			max_count = 45
			count = max_count
960 961 962 963 964
			while count > 0:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
965 966 967 968 969 970
					lock.acquire()
					statusQueue.put(-1)
					statusQueue.put(device_id)
					statusQueue.put('mDataConnectionState Not Found!')
					lock.release()
					break
971 972 973
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState == 2:
					logging.debug('\u001B[1mUE (' + device_id + ') Attach Completed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
974 975 976 977 978
					lock.acquire()
					statusQueue.put(max_count - count)
					statusQueue.put(device_id)
					statusQueue.put('Attach Completed')
					lock.release()
979 980 981
					break
				count = count - 1
				if count == 15 or count == 30:
Raphael Defosseux's avatar
Raphael Defosseux committed
982
					logging.debug('\u001B[1;30;43m Retry UE (' + device_id + ') Flight Mode Off \u001B[0m')
983 984 985 986
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
					time.sleep(0.5)
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
					time.sleep(0.5)
Raphael Defosseux's avatar
Raphael Defosseux committed
987
				logging.debug('\u001B[1mWait UE (' + device_id + ') a second until mDataConnectionState=2 (' + str(max_count-count) + ' times)\u001B[0m')
988 989 990
				time.sleep(1)
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Attach Failed \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
991 992 993 994 995
				lock.acquire()
				statusQueue.put(-1)
				statusQueue.put(device_id)
				statusQueue.put('Attach Failed')
				lock.release()
996 997 998 999 1000 1001 1002 1003
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def AttachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1004 1005 1006
		check_eNB = True
		check_OAI_UE = False
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1007 1008
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1009
			self.AutoTerminateUEandeNB()
1010
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1011
			sys.exit(1)
1012
		multi_jobs = []
Raphael Defosseux's avatar
Raphael Defosseux committed
1013 1014
		status_queue = SimpleQueue()
		lock = Lock()
1015
		nb_ue_to_connect = 0
1016
		for device_id in self.UEDevices:
1017
			if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach):
1018
				p = Process(target = self.AttachUE_common, args = (device_id, status_queue, lock,))
1019 1020 1021 1022
				p.daemon = True
				p.start()
				multi_jobs.append(p)
			nb_ue_to_connect = nb_ue_to_connect + 1
1023 1024
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
Raphael Defosseux committed
1025 1026

		if (status_queue.empty()):
1027
			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
1028
			self.CreateHtmlTabFooter(False)
1029
			self.AutoTerminateUEandeNB()
Raphael Defosseux's avatar
Raphael Defosseux committed
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
			sys.exit(1)
		else:
			attach_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					attach_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				if (count < 0):
1041
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>'
Raphael Defosseux's avatar
Raphael Defosseux committed
1042
				else:
1043
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + ' in ' + str(count + 2) + ' seconds</pre>'
Raphael Defosseux's avatar
Raphael Defosseux committed
1044 1045 1046
				html_queue.put(html_cell)
			if (attach_status):
				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
1047 1048 1049 1050
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					logging.debug('Waiting 5 seconds to fill up record file')
					time.sleep(5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1051 1052
			else:
				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
1053
				self.AutoTerminateUEandeNB()
1054
				self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1055
				sys.exit(1)
1056 1057 1058 1059

	def DetachUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1060
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
1061 1062 1063 1064 1065 1066 1067 1068 1069
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DetachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1070
		check_eNB = True
Raphael Defosseux's avatar
Raphael Defosseux committed
1071
		check_OAI_UE = False
1072
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1073 1074
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1075
			self.AutoTerminateUEandeNB()
1076
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1077
			sys.exit(1)
1078 1079
		multi_jobs = []
		for device_id in self.UEDevices:
1080
			p = Process(target = self.DetachUE_common, args = (device_id,))
1081 1082 1083 1084 1085
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1086
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1087 1088 1089 1090
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			logging.debug('Waiting 5 seconds to fill up record file')
			time.sleep(5)
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134

	def RebootUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			previousmDataConnectionStates = []
			# Save mDataConnectionState
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
				sys.exit(1)
			previousmDataConnectionStates.append(int(result.group('state')))
			# Reboot UE
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell reboot', '\$', 10)
			time.sleep(60)
			previousmDataConnectionState = previousmDataConnectionStates.pop(0)
			count = 180
			while count > 0:
				count = count - 1
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					mDataConnectionState = None
				else:
					mDataConnectionState = int(result.group('state'))
					logging.debug('mDataConnectionState = ' + result.group('state'))
				if mDataConnectionState is None or (previousmDataConnectionState == 2 and mDataConnectionState != 2):
					logging.debug('\u001B[1mWait UE (' + device_id + ') a second until reboot completion (' + str(180-count) + ' times)\u001B[0m')
					time.sleep(1)
				else:
					logging.debug('\u001B[1mUE (' + device_id + ') Reboot Completed\u001B[0m')
					break
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Reboot Failed \u001B[0m')
				sys.exit(1)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def RebootUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1135
		check_eNB = True
Raphael Defosseux's avatar
Raphael Defosseux committed
1136
		check_OAI_UE = False
1137
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
Raphael Defosseux's avatar
Raphael Defosseux committed
1138 1139
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1140
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1141
			sys.exit(1)
1142 1143
		multi_jobs = []
		for device_id in self.UEDevices:
1144
			p = Process(target = self.RebootUE_common, args = (device_id,))
1145 1146 1147 1148 1149
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1150
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162

	def GetAllUEDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('adb devices', '\$', 15)
		self.UEDevices = re.findall("\\\\r\\\\n([A-Za-z0-9]+)\\\\tdevice",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.UEDevices) == 0:
				logging.debug('\u001B[1;37;41m UE Not Found! \u001B[0m')
				sys.exit(1)
1163 1164
		self.close()

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	def GetAllCatMDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
		self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.CatMDevices) == 0:
				logging.debug('\u001B[1;37;41m CAT-M UE Not Found! \u001B[0m')
				sys.exit(1)
		self.close()

1178 1179 1180 1181
	def GetAllUEIPAddresses(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1182
		ue_ip_status = 0
1183
		self.UEIPAddresses = []
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
		if (len(self.UEDevices) == 1) and (self.UEDevices[0] == 'OAI-UE'):
			if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
				Usage()
				sys.exit('Insufficient Parameter')
			self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
			self.command('ifconfig oaitun_ue1', '\$', 4)
			result = re.search('inet addr:(?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', str(self.ssh.before))
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + self.UEDevices[0] + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			self.UEIPAddresses.append(UE_IPAddress)
			self.close()
			return ue_ip_status
1196 1197
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		for device_id in self.UEDevices:
1198 1199 1200 1201 1202 1203
			count = 0
			while count < 4:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ip addr show | grep rmnet', '\$', 15)
				result = re.search('inet (?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+[0-9a-zA-Z\.\s]+', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
1204
					time.sleep(1)
1205 1206 1207 1208 1209 1210
					count += 1
				else:
					count = 10
			if count < 9:
				ue_ip_status -= 1
				continue
1211 1212 1213 1214
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + device_id + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			for ueipaddress in self.UEIPAddresses:
				if ueipaddress == UE_IPAddress:
1215
					logging.debug('\u001B[1mUE (' + device_id + ') IP Address ' + UE_IPAddress + ': has already been allocated to another device !' + '\u001B[0m')
1216 1217
					ue_ip_status -= 1
					continue
1218
			self.UEIPAddresses.append(UE_IPAddress)
1219
		self.close()
1220
		return ue_ip_status
1221

Raphael Defosseux's avatar
Raphael Defosseux committed
1222 1223 1224 1225 1226 1227 1228 1229 1230
	def ping_iperf_wrong_exit(self, lock, UE_IPAddress, device_id, statusQueue, message):
		lock.acquire()
		statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
		statusQueue.put(message)
		lock.release()

	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue):
1231 1232 1233 1234 1235
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
1236
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1237 1238 1239 1240
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1241
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1242 1243
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
				return
1244 1245
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1246 1247
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1248
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1249
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
1250
				return
1251 1252
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
Raphael Defosseux's avatar
Raphael Defosseux committed
1253 1254
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1255
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1256
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
1257
				return
1258 1259
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
Raphael Defosseux committed
1260 1261
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
1262
				self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1263
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
Raphael Defosseux committed
1264
				return
1265 1266 1267
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
Raphael Defosseux's avatar
Raphael Defosseux committed
1268 1269 1270 1271
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
1272 1273
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1274 1275 1276 1277 1278 1279
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
1280 1281
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
Raphael Defosseux's avatar
Raphael Defosseux committed
1282
					qMsg += '\nPacket Loss too high'
1283
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1284
					packetLossOK = False
1285
				elif float(packetloss) > 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
1286
					qMsg += '\nPacket Loss is not 0%'
Raphael Defosseux's avatar
Raphael Defosseux committed
1287
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1288 1289 1290 1291 1292 1293 1294 1295
			if (packetLossOK):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(qMsg)
			lock.release()
1296 1297 1298 1299
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

1300
	def PingNoS1(self):
1301 1302 1303 1304 1305 1306 1307 1308
		check_eNB = True
		check_OAI_UE = True
		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
			self.AutoTerminateUEandeNB()
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
		ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
		if ping_from_eNB is not None:
			if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '':
				Usage()
				sys.exit('Insufficient Parameter')
		else:
			if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
				Usage()
				sys.exit('Insufficient Parameter')
		try:
			if ping_from_eNB is not None:
				self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets/', '\$', 5)
			else:
				self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
				self.command('cd ' + self.UESourceCodePath + '/cmake_targets/', '\$', 5)
			self.command('cd cmake_targets', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
1327