main.py 155 KB
Newer Older
1
# dummy commit
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Version
#-----------------------------------------------------------
Version = '0.1'

37
38
39
40
41
42
43
44
45
#-----------------------------------------------------------
# Constants
#-----------------------------------------------------------
ALL_PROCESSES_OK = 0
ENB_PROCESS_FAILED = -1
ENB_PROCESS_OK = +1
ENB_PROCESS_SEG_FAULT = -11
ENB_PROCESS_ASSERTION = -12
ENB_PROCESS_REALTIME_ISSUE = -13
46
ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
47
48
49
UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20
UE_PROCESS_COULD_NOT_SYNC = -21
UE_PROCESS_ASSERTION = -22
50
51
52
53
54
55
56
57
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
MME_PROCESS_OK = +3
SPGW_PROCESS_FAILED = -4
SPGW_PROCESS_OK = +4
UE_IP_ADDRESS_ISSUE = -5

58
59
60
61
62
63
64
65
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect		# pexpect
import time		# sleep
import os
66
import subprocess
67
68
69
70
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
71
#import platform
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
72
from multiprocessing import Process, Lock, SimpleQueue
73
logging.basicConfig(
74
75
	level=logging.DEBUG,
	format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
76
77
78
79
80
81
82
83
84
85
)

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
	def __init__(self):
		self.eNBIPAddress = ''
		self.eNBRepository = ''
		self.eNBBranch = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
86
		self.eNB_AllowMerge = False
87
		self.eNBCommitID = ''
88
		self.eNBTargetBranch = ''
89
90
91
92
93
94
95
96
97
98
99
100
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.EPCIPAddress = ''
		self.EPCUserName = ''
		self.EPCPassword = ''
		self.EPCSourceCodePath = ''
		self.EPCType = ''
		self.ADBIPAddress = ''
		self.ADBUserName = ''
		self.ADBPassword = ''
		self.testCase_id = ''
101
102
		self.testXMLfiles = []
		self.nbTestXMLfiles = 0
103
104
105
		self.desc = ''
		self.Build_eNB_args = ''
		self.Initialize_eNB_args = ''
106
		self.air_interface = 'lte'
107
		self.eNBLogFile = ''
108
		self.eNB_instance = ''
109
110
		self.eNBOptions = ''
		self.rruOptions = ''
111
112
113
114
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
115
		self.iperf_profile = ''
116
		self.nbMaxUEtoAttach = -1
117
		self.UEDevices = []
118
		self.CatMDevices = []
119
		self.UEIPAddresses = []
Raphael Defosseux's avatar
Raphael Defosseux committed
120
121
122
		self.htmlFile = ''
		self.htmlHeaderCreated = False
		self.htmlFooterCreated = False
123
		self.htmlUEConnected = -1
124
		self.htmleNBFailureMsg = ''
125
126
127
128
129
130
		self.picocom_closure = False
		self.idle_sleep_time = 0
		self.htmlTabRefs = []
		self.htmlTabNames = []
		self.htmlTabIcons = []
		self.finalStatus = False
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
		self.OsVersion = ''
		self.KernelVersion = ''
		self.UhdVersion = ''
		self.UsrpBoard = ''
		self.CpuNb = ''
		self.CpuModel = ''
		self.CpuMHz = ''
		self.UEIPAddress = ''
		self.UEUserName = ''
		self.UEPassword = ''
		self.UE_instance = ''
		self.UELogFile = ''
		self.UESourceCodePath = ''
		self.Build_OAI_UE_args = ''
		self.Initialize_OAI_UE_args = ''
		self.Initialize_OAI_eNB_args = ''
		self.clean_repository = True
148
		self.eNBOsVersion = ''
149
		
150
	def open(self, ipaddress, username, password):
151
152
153
154
155
		count = 0
		connect_status = False
		while count < 4:
			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
156
			if self.sshresponse == 0:
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
				self.ssh.sendline('yes')
				self.ssh.expect('password:')
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 1:
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 2:
				# Checking if we are really on the remote client defined by its IP address
				self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:"', '\$', 5)
				result = re.search(str(ipaddress), str(self.ssh.before))
				if result is None:
					self.close()
				else:
					count = 10
					connect_status = True
183
			else:
184
185
				# debug output
				logging.debug(str(self.ssh.before))
186
				logging.debug('self.sshresponse = ' + str(self.sshresponse))
Raphael Defosseux's avatar
Raphael Defosseux committed
187
188
189
			# adding a tempo when failure
			if not connect_status:
				time.sleep(1)
190
191
			count += 1
		if connect_status:
192
193
			pass
		else:
194
			sys.exit('SSH Connection Failed')
195

196
197
198
199
200
201
	def command(self, commandline, expectedline, timeout):
		logging.debug(commandline)
		self.ssh.timeout = timeout
		self.ssh.sendline(commandline)
		self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
202
			return 0
203
204
205
206
207
208
209
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
		elif self.sshresponse == 2:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
210
			result = re.search('ping |iperf |picocom', str(commandline))
Raphael Defosseux's avatar
Raphael Defosseux committed
211
			if result is None:
212
				logging.debug(str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
213
214
215
				sys.exit(self.sshresponse)
			else:
				return -1
216
217
218
219
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
220

221
222
223
224
225
226
227
	def close(self):
		self.ssh.timeout = 5
		self.ssh.sendline('exit')
		self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
			pass
		elif self.sshresponse == 1:
228
229
			if not self.picocom_closure:
				logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m')
230
		else:
231
			logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m')
232

233
	def copyin(self, ipaddress, username, password, source, destination):
234
235
		count = 0
		copy_status = False
Raphael Defosseux's avatar
Raphael Defosseux committed
236
		logging.debug('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
237
238
239
		while count < 10:
			scp_spawn = pexpect.spawn('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 100)
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
Raphael Defosseux's avatar
Raphael Defosseux committed
240
			if scp_response == 0:
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
Raphael Defosseux's avatar
Raphael Defosseux committed
261
			else:
262
263
264
265
266
267
268
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
			return 0
Raphael Defosseux's avatar
Raphael Defosseux committed
269
		else:
270
			return -1
Raphael Defosseux's avatar
Raphael Defosseux committed
271

272
	def copyout(self, ipaddress, username, password, source, destination):
273
274
		count = 0
		copy_status = False
275
		logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
276
277
278
		while count < 4:
			scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 100)
			scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
279
			if scp_response == 0:
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
				scp_spawn.sendline('yes')
				scp_spawn.expect('password:')
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0:
					count = 10
					copy_status = True
				else:
					logging.debug('1 - scp_response = ' + str(scp_response))
			elif scp_response == 1:
				scp_spawn.sendline(password)
				scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if scp_response == 0 or scp_response == 3:
					count = 10
					copy_status = True
				else:
					logging.debug('2 - scp_response = ' + str(scp_response))
			elif scp_response == 2:
				count = 10
				copy_status = True
300
			else:
301
302
303
304
305
306
				logging.debug('3 - scp_response = ' + str(scp_response))
			# adding a tempo when failure
			if not copy_status:
				time.sleep(1)
			count += 1
		if copy_status:
307
308
309
310
			pass
		else:
			sys.exit('SCP failed')

311
312
313
314
315
316
317
	def BuildeNB(self):
		if self.eNBIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
318
319
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
320
321
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
322
		self.command('echo ' + self.eNBPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
323
324
325
326
327
		# if the commit ID is provided use it to point to it
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
Raphael Defosseux's avatar
Raphael Defosseux committed
328
		if (self.eNB_AllowMerge):
329
330
331
332
333
334
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
335
336
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
337
338
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
339
		# no need to remove in log (git clean did the trick)
340
		self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_enb.log', 'Bypassing the Tests', 600)
341
342
343
		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.command('mv compile_oai_enb.log ' + 'build_log_' + self.testCase_id, '\$', 5)
344
		self.close()
345
		self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
346

347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
	def BuildOAIUE(self):
		if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
		self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5)
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
		# here add a check if git clone or git fetch went smoothly
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
		if self.clean_repository:
			pass
			self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.eNB_AllowMerge):
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600)
		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5)
		self.close()
		self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')


386
387
388
389
390
391
392
	def InitializeHSS(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			logging.debug('Using the OAI EPC HSS')
393
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
394
395
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
396
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + self.testCase_id + '.log &', 'Core state: 2 -> 3', 35)
397
398
		else:
			logging.debug('Using the ltebox simulated HSS')
399
400
401
402
			self.command('if [ -d ' + self.EPCSourceCodePath + '/scripts ]; then echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('mkdir -p ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			self.command('cd /opt/hss_sim0609', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f hss.log daemon.log', '\$', 5)
403
404
			self.command('echo ' + self.EPCPassword + ' | sudo -S echo "Starting sudo session" && sudo daemon --unsafe --name=simulated_hss --chdir=/opt/hss_sim0609 ./starthss_real  ', '\$', 5)
		self.close()
405
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
406

407
408
409
410
411
412
	def InitializeMME(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
413
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
414
415
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
416
			self.command('stdbuf -o0 hostname', '\$', 5)
417
418
419
420
421
			result = re.search('hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m')
				sys.exit(1)
			host_name = result.group('host_name')
422
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + self.testCase_id + '.log &', 'MME app initialization complete', 100)
423
		else:
424
			self.command('cd /opt/ltebox/tools', '\$', 5)
425
426
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_mme', '\$', 5)
		self.close()
427
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
428
429
430

	def InitializeSPGW(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
431
432
			Usage()
			sys.exit('Insufficient Parameter')
433
434
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
435
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
436
437
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
438
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + self.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30)
439
		else:
440
			self.command('cd /opt/ltebox/tools', '\$', 5)
441
442
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_xGw', '\$', 5)
		self.close()
443
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
444
445
446

	def InitializeeNB(self):
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
447
448
449
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = True
Raphael Defosseux's avatar
Raphael Defosseux committed
450
451
452
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
453
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
454
			sys.exit(1)
455
456
457
458
459
460
461
462
463
464
465
466
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', str(self.ssh.before))
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
				self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/enb_' + self.testCase_id + '_s1log.pcap', '\$', 5)
				self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/enb_' + self.testCase_id + '_s1log.pcap > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5)
			self.close()
467
468
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
469
470
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
471
		extra_options = ''
472
473
474
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
475
476
477
478
479
480
481
482
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				self.command('cd common/utils/T/tracer', '\$', 5)
				self.command('make', '\$', 10)
				self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5)
				self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
483
484
485
486
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
487
		ci_full_config_file = config_path + '/ci-' + config_file
488
489
490
491
		rruCheck = False
		result = re.search('rru', str(config_file))
		if result is not None:
			rruCheck = True
492
493
494
495
496
497
498
499
500
501
		# do not reset board twice in IF4.5 case
		result = re.search('rru|enb', str(config_file))
		if result is not None:
			self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 30)
			result = re.search('type: b200', str(self.ssh.before))
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				self.command('echo ' + self.eNBPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5)
				# Reloading FGPA bin firmware
				self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 30)
502
503
		# Make a copy and adapt to EPC / eNB IP addresses
		self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
504
505
		self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
506
		# Launch eNB with the modified config file
507
508
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
509
		self.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface + '-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5)
510
		self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5)
511
512
513
514
515
516
517
518
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)				
		#to use daemon on CentOS we need to source the function
		#linux_distro = platform.linux_distribution()[0]
		#if re.match('(.*)CentOS(.*)', linux_distro, re.IGNORECASE):
		#self.command('source /etc/init.d/functions', '\$', 5)
		#use nohup instead of daemon
		#self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.command('echo $USER; nohup sudo ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh' + ' > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log' + ' 2>&1 &', self.eNBUserName, 5)		
519
		if not rruCheck:
520
			self.eNBLogFile = 'enb_' + self.testCase_id + '.log'
521
522
			if extra_options != '':
				self.eNBOptions = extra_options
523
524
525
526
527
528
		time.sleep(6)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
529
530
531
532
533
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.command('killall --signal SIGKILL record', '\$', 5)
				self.close()
534
				doLoop = False
Raphael Defosseux's avatar
Raphael Defosseux committed
535
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
536
				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
537
538
539
540
541
542
543
544
545
546
547
548
549
				self.CreateHtmlTabFooter(False)
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
					self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					self.close()
					time.sleep(1)
					pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap'
					copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.')
					if (copyin_res == 0):
						self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.')
550
				sys.exit(1)
551
			else:
552
				self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
553
554
555
556
				if rruCheck:
					result = re.search('wait RUs', str(self.ssh.before))
				else:
					result = re.search('got sync', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
557
558
559
560
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
561
562
					if rruCheck and extra_options != '':
						self.rruOptions = extra_options
563
					self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
564
					logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')
565

566
567
		self.close()

568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
	def InitializeUE_common(self, device_id):
		logging.debug('send adb commands')
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# The following commands are deprecated since we no longer work on Android 7+
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell settings put global airplane_mode_on 1', '\$', 10)
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', '\$', 60)
			# a dedicated script has to be installed inside the UE
			# airplane mode on means call /data/local/tmp/off
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			#airplane mode off means call /data/local/tmp/on
			logging.debug('\u001B[1mUE (' + device_id + ') Initialize Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def InitializeUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
590
			p = Process(target = self.InitializeUE_common, args = (device_id,))
591
592
593
594
595
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
596
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
597

598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
	def InitializeOAIUE(self):
		if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		#initialize_OAI_UE_flag = True
		#pStatus = self.CheckOAIUEProcessExist(initialize_OAI_UE_flag)
		#if (pStatus < 0):
		#	self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus)
		#	self.CreateHtmlTabFooter(False)
		#	sys.exit(1)
		self.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
		# b2xx_fx3_utils reset procedure
		self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 30)
		result = re.search('type: n3xx', str(self.ssh.before))
		if result is not None:
			pass
			logging.debug('Found a B2xx device --> resetting it')
			self.command('echo ' + self.UEPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5)
			# Reloading FGPA bin firmware
			self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 30)
		else:
			logging.debug('Did not find any B2xx device')
		self.command('cd ' + self.UESourceCodePath, '\$', 5)
		# Initialize_OAI_UE_args usually start with -C and followed by the location in repository
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets/ran_build/build', '\$', 5)
		self.command('echo "ulimit -c unlimited && ./'+ self.air_interface +'-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5)
		#to use daemon on CentOS we need to source the function
		#linux_distro = platform.linux_distribution()[0]		
		#if re.match('(.*)CentOS(.*)', linux_distro, re.IGNORECASE):
			#self.command('source /etc/init.d/functions', '\$', 5)
		#use nohup instead of daemon
		self.command('echo $USER; nohup sudo ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh' + ' > ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ' + ' 2>&1 &', self.UEUserName, 5)
		#self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/ran_build/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
		self.UELogFile = 'ue_' + self.testCase_id + '.log'
		time.sleep(6)
		self.command('cd ../..', '\$', 5)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				self.close()
				doLoop = False
				logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m')
				self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE')
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
			else:
				self.command('stdbuf -o0 cat ' + self.UELogFile + ' | egrep --text --color=never -i "wait|sync"', '\$', 4)
				if self.air_interface == 'nr':
					result = re.search('Starting sync detection', str(self.ssh.before))
				else:
					result = re.search('got sync', str(self.ssh.before))
				if result is None:
					time.sleep(10)
				else:
					doLoop = False
					self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE')
					logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m')
		self.close()

	def InitializeOAIeNB(self):
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		#initialize_OAI_eNB_flag = True
		#pStatus = self.CheckOAIeNBProcessExist(initialize_OAI_eNB_flag)
		#if (pStatus < 0):
		#	self.CreateHtmlTestRow(self.Initialize_OAI_eNB_args, 'KO', pStatus)
		#	self.CreateHtmlTabFooter(False)
		#	sys.exit(1)
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		# Initialize_OAI_eNB_args usually start with -C and followed by the location in repository
		#full_config_file = self.Initialize_OAI_eNB_args.replace('-O ','')
		#extIdx = full_config_file.find('.conf')
		#if (extIdx > 0):
		#	extra_options = full_config_file[extIdx + 5:]
		#	# if tracer options is on, compiling and running T Tracer
		#	result = re.search('T_stdout', str(extra_options))
		##	if result is not None:
		#		logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
		#		self.command('cd common/utils/T/tracer', '\$', 5)
		#		self.command('make', '\$', 10)
		#		self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5)
		#		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		#	full_config_file = full_config_file[:extIdx + 5]
		#	config_path, config_file = os.path.split(full_config_file)
		#ci_full_config_file = config_path + '/ci-' + config_file
		#rruCheck = False
		#result = re.search('rru', str(config_file))
		#if result is not None:
		#	rruCheck = True
		## Make a copy and adapt to EPC / eNB IP addresses
		#self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
		#self.command('sed -i -e \'s/CI_eNB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		# Launch eNB with the modified config file
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets/ran_build/build', '\$', 5)
		#self.command('echo "ulimit -c unlimited && ./' + self.air_interface + '-softmodem ' + self.Initialize_OAI_eNB_args + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.eNBLogFile = 'enb_' + self.testCase_id + '.log'
		self.command('echo "ulimit -c unlimited && ./' + self.air_interface + '-softmodem ' + self.Initialize_OAI_eNB_args + '|& tee ' + self.eNBSourceCodePath + '/cmake_targets/' + self.eNBLogFile + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log', '\$', 5)
		#to use daemon on CentOS we need to source the function
		#linux_distro = platform.linux_distribution()[0]		
		#if re.match('(.*)CentOS(.*)', linux_distro, re.IGNORECASE):
			#self.command('source /etc/init.d/functions', '\$', 5)
		#use nohup instead of daemon
		self.command('echo $USER; nohup sudo ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh' + ' > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log' + ' 2>&1 &', self.eNBUserName, 5)
		#self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		time.sleep(6)
		self.command('cd ../..', '\$', 5)
		doLoop = True
		loopCounter = 10
		print('gNB log file: ' + self.eNBLogFile)
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				#result = re.search('T_stdout', str(self.Initialize_OAI_eNB_args))
				#if result is not None:
				#	self.command('killall --signal SIGKILL record', '\$', 5)
				self.close()
				doLoop = False
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
				self.CreateHtmlTestRow(self.Initialize_OAI_eNB_args, 'KO', ALL_PROCESSES_OK, 'OAI eNB')
				self.CreateHtmlTabFooter(False)
				## In case of T tracer recording, we need to kill tshark on EPC side
				#result = re.search('T_stdout', str(self.Initialize_OAI_eNB_args))
				#if result is not None:
				#	self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
				#	logging.debug('\u001B[1m Stopping tshark \u001B[0m')
				#	self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
				#	self.close()
				#	time.sleep(1)
				#	pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap'
				#	copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.')
				#	if (copyin_res == 0):
				#		self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.')
				sys.exit(1)
			else:
				#print('current directory: ' + os.getcwd())
				#self.command('pwd', '\$', 4)
				#print('self.command pwd: ' + str(self.ssh.before))
				self.command('stdbuf -o0 cat ' + self.eNBLogFile + ' | egrep --text --color=never -i "wait|sync"', '\$', 30)
				#print(self.ssh.before)
				result = re.search('got sync', str(self.ssh.before))
				if result is None:
					time.sleep(11)
				else:
					doLoop = False
					self.CreateHtmlTestRow(self.Initialize_OAI_eNB_args, 'OK', ALL_PROCESSES_OK, 'OAI eNB')
					logging.debug('\u001B[1m Initialize OAI eNB Completed\u001B[0m')
		self.close()

757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
	def checkDevTTYisUnlocked(self):
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		count = 0
		while count < 5:
			self.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep ttyUSB0', '\$', 10)
			result = re.search('picocom', str(self.ssh.before))
			if result is None:
				count = 10
			else:
				time.sleep(5)
				count = count + 1
		self.close()

	def InitializeCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		# Checking if auto-attach is enabled
		self.command('AT^AUTOATT?', 'OK', 5)
		result = re.search('AUTOATT: (?P<state>[0-9\-]+)', str(self.ssh.before))
		if result is not None:
			if result.group('state') is not None:
				autoAttachState = int(result.group('state'))
				if autoAttachState is not None:
					if autoAttachState == 0:
						self.command('AT^AUTOATT=1', 'OK', 5)
					logging.debug('\u001B[1m Auto-Attach enabled\u001B[0m')
		else:
			logging.debug('\u001B[1;37;41m Could not check Auto-Attach! \u001B[0m')
		# Force closure of picocom but device might still be locked
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def TerminateCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Disabling the Radio
		self.command('AT+CFUN=0', 'OK', 5)
		logging.debug('\u001B[1m Cellular Functionality disabled\u001B[0m')
		self.close()
		self.picocom_closure = False
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
		self.checkDevTTYisUnlocked()

	def AttachCatM(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.picocom_closure = True
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		# dummy call to start a sudo session. The picocom command does NOT handle well the `sudo -S`
		self.command('echo ' + self.ADBPassword + ' | sudo -S ls', '\$', 10)
		self.command('sudo picocom --baud 921600 --flow n --databits 8 /dev/ttyUSB0', 'Terminal ready', 10)
		time.sleep(1)
		# Calling twice AT to clear all buffers
		self.command('AT', 'OK|ERROR', 5)
		self.command('AT', 'OK', 5)
		# Enabling the Radio
		self.command('AT+CFUN=1', 'SIMSTORE,READY', 5)
		logging.debug('\u001B[1m Cellular Functionality enabled\u001B[0m')
		time.sleep(4)
		# We should check if we register
		count = 0
844
845
846
		attach_cnt = 0
		attach_status = False
		while count < 5:
847
			self.command('AT+CEREG?', 'OK', 5)
848
			result = re.search('CEREG: 2,(?P<state>[0-9\-]+),', str(self.ssh.before))
849
850
851
			if result is not None:
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState is not None:
852
853
854
855
856
857
858
859
860
861
862
863
864
					if mDataConnectionState == 1:
						count = 10
						attach_status = True
						result = re.search('CEREG: 2,1,"(?P<networky>[0-9A-Z]+)","(?P<networkz>[0-9A-Z]+)"', str(self.ssh.before))
						if result is not None:
							networky = result.group('networky')
							networkz = result.group('networkz')
							logging.debug('\u001B[1m CAT-M module attached to eNB (' + str(networky) + '/' + str(networkz) + ')\u001B[0m')
						else:
							logging.debug('\u001B[1m CAT-M module attached to eNB\u001B[0m')
					else:
						logging.debug('+CEREG: 2,' + str(mDataConnectionState))
						attach_cnt = attach_cnt + 1
865
866
			else:
				logging.debug(str(self.ssh.before))
867
				attach_cnt = attach_cnt + 1
868
869
			count = count + 1
			time.sleep(1)
870
871
872
873
874
875
876
877
878
		if attach_status:
			self.command('AT+CESQ', 'OK', 5)
			result = re.search('CESQ: 99,99,255,255,(?P<rsrq>[0-9]+),(?P<rsrp>[0-9]+)', str(self.ssh.before))
			if result is not None:
				nRSRQ = int(result.group('rsrq'))
				nRSRP = int(result.group('rsrp'))
				if (nRSRQ is not None) and (nRSRP is not None):
					logging.debug('    RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB')
					logging.debug('    RSRP = ' + str(-140+nRSRP) + ' dBm')
879
880
		self.close()
		self.picocom_closure = False
881
		html_queue = SimpleQueue()
882
		self.checkDevTTYisUnlocked()
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
		if attach_status:
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Completed in ' + str(attach_cnt+4) + ' seconds'
			if (nRSRQ is not None) and (nRSRP is not None):
				html_cell += '\n   RSRQ = ' + str(-20+(nRSRQ/2)) + ' dB'
				html_cell += '\n   RSRP = ' + str(-140+nRSRP) + ' dBm</pre>'
			else:
				html_cell += '</pre>'
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'OK', 1, html_queue)
		else:
			html_cell = '<pre style="background-color:white">CAT-M module\nAttachment Failed</pre>'
			html_queue.put(html_cell)
			self.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)

	def PingCatM(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
			self.CreateHtmlTabFooter(False)
			sys.exit(1)
		try:
			statusQueue = SimpleQueue()
			lock = Lock()
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				logging.debug('Using the OAI EPC HSS: not implemented yet')
				self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
			else:
				self.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5)
				result = re.search('Allocated ipv4 addr: (?P<ipaddr>[0-9\.]+) from Pool', str(self.ssh.before))
				if result is not None:
					moduleIPAddr = result.group('ipaddr')
				else:
					return
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
			device_id = 'catm'
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + str(moduleIPAddr) + ' 2>&1 | stdbuf -o0 tee -a ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(moduleIPAddr) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, moduleIPAddr, device_id, statusQueue, message)
				return
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + moduleIPAddr + ') \u001B[0m')
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
					qMsg += '\nPacket Loss too high'
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
					packetLossOK = False
				elif float(packetloss) > 0:
					qMsg += '\nPacket Loss is not 0%'
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
			lock.release()
			self.close()
			html_cell = '<pre style="background-color:white">CAT-M module\nIP Address  : ' + moduleIPAddr + '\n' + qMsg + '</pre>'
			statusQueue.put(html_cell)
			if (packetLossOK):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
				self.AutoTerminateUEandeNB()
				self.CreateHtmlTabFooter(False)
				sys.exit(1)
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
988

Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
989
	def AttachUE_common(self, device_id, statusQueue, lock):
990
991
992
993
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
			time.sleep(2)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
994
995
			max_count = 45
			count = max_count
996
997
998
999
1000
			while count > 0:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1001
1002
1003
1004
1005
1006
					lock.acquire()
					statusQueue.put(-1)
					statusQueue.put(device_id)
					statusQueue.put('mDataConnectionState Not Found!')
					lock.release()
					break
1007
1008
1009
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState == 2:
					logging.debug('\u001B[1mUE (' + device_id + ') Attach Completed\u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1010
1011
1012
1013
1014
					lock.acquire()
					statusQueue.put(max_count - count)
					statusQueue.put(device_id)
					statusQueue.put('Attach Completed')
					lock.release()
1015
1016
1017
					break
				count = count - 1
				if count == 15 or count == 30:
Raphael Defosseux's avatar
Raphael Defosseux committed
1018
					logging.debug('\u001B[1;30;43m Retry UE (' + device_id + ') Flight Mode Off \u001B[0m')
1019
1020
1021
1022
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
					time.sleep(0.5)
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
					time.sleep(0.5)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1023
				logging.debug('\u001B[1mWait UE (' + device_id + ') a second until mDataConnectionState=2 (' + str(max_count-count) + ' times)\u001B[0m')
1024
1025
1026
				time.sleep(1)
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Attach Failed \u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1027
1028
1029
1030
1031
				lock.acquire()
				statusQueue.put(-1)
				statusQueue.put(device_id)
				statusQueue.put('Attach Failed')
				lock.release()
1032
1033
1034
1035
1036
1037
1038
1039
1040
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def AttachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1041
1042
1043
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
Raphael Defosseux's avatar
Raphael Defosseux committed
1044
			self.AutoTerminateUEandeNB()
1045
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1046
			sys.exit(1)
1047
		multi_jobs = []
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1048
1049
		status_queue = SimpleQueue()
		lock = Lock()
1050
		nb_ue_to_connect = 0
1051
		for device_id in self.UEDevices:
1052
1053
1054
1055
1056
1057
			if (self.nbMaxUEtoAttach == -1) or (nb_ue_to_connect < self.nbMaxUEtoAttach):
				p = Process(target = self.AttachUE_common, args = (device_id, status_queue, lock,))
				p.daemon = True
				p.start()
				multi_jobs.append(p)
			nb_ue_to_connect = nb_ue_to_connect + 1
1058
1059
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1060
1061

		if (status_queue.empty()):
1062
			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
1063
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1064
			self.AutoTerminateUEandeNB()
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
			sys.exit(1)
		else:
			attach_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					attach_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				if (count < 0):
1076
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + '</pre>'
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1077
				else:
1078
					html_cell = '<pre style="background-color:white">UE (' + device_id + ')\n' + message + ' in ' + str(count + 2) + ' seconds</pre>'
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1079
1080
1081
				html_queue.put(html_cell)
			if (attach_status):
				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
1082
1083
1084
1085
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					logging.debug('Waiting 5 seconds to fill up record file')
					time.sleep(5)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1086
1087
			else:
				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
Raphael Defosseux's avatar
Raphael Defosseux committed
1088
				self.AutoTerminateUEandeNB()
1089
				self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1090
				sys.exit(1)
1091
1092
1093
1094

	def DetachUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1095
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DetachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1106
1107
1108
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
Raphael Defosseux's avatar
Raphael Defosseux committed
1109
			self.AutoTerminateUEandeNB()
1110
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1111
			sys.exit(1)
1112
1113
		multi_jobs = []
		for device_id in self.UEDevices:
1114
			p = Process(target = self.DetachUE_common, args = (device_id,))
1115
1116
1117
1118
1119
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1120
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1121
1122
1123
1124
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
		if result is not None:
			logging.debug('Waiting 5 seconds to fill up record file')
			time.sleep(5)
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169

	def RebootUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			previousmDataConnectionStates = []
			# Save mDataConnectionState
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
				sys.exit(1)
			previousmDataConnectionStates.append(int(result.group('state')))
			# Reboot UE
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell reboot', '\$', 10)
			time.sleep(60)
			previousmDataConnectionState = previousmDataConnectionStates.pop(0)
			count = 180
			while count > 0:
				count = count - 1
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					mDataConnectionState = None
				else:
					mDataConnectionState = int(result.group('state'))
					logging.debug('mDataConnectionState = ' + result.group('state'))
				if mDataConnectionState is None or (previousmDataConnectionState == 2 and mDataConnectionState != 2):
					logging.debug('\u001B[1mWait UE (' + device_id + ') a second until reboot completion (' + str(180-count) + ' times)\u001B[0m')
					time.sleep(1)
				else:
					logging.debug('\u001B[1mUE (' + device_id + ') Reboot Completed\u001B[0m')
					break
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Reboot Failed \u001B[0m')
				sys.exit(1)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def RebootUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1170
1171
1172
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
1173
			self.CreateHtmlTabFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1174
			sys.exit(1)
1175
1176
		multi_jobs = []
		for device_id in self.UEDevices:
1177
			p = Process(target = self.RebootUE_common, args = (device_id,))
1178
1179
1180
1181
1182
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1183
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195

	def GetAllUEDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('adb devices', '\$', 15)
		self.UEDevices = re.findall("\\\\r\\\\n([A-Za-z0-9]+)\\\\tdevice",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.UEDevices) == 0:
				logging.debug('\u001B[1;37;41m UE Not Found! \u001B[0m')
				sys.exit(1)
1196
1197
		self.close()

1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
	def GetAllCatMDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
		self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.CatMDevices) == 0:
				logging.debug('\u001B[1;37;41m CAT-M UE Not Found! \u001B[0m')
				sys.exit(1)
		self.close()

1211
1212
1213
1214
	def GetAllUEIPAddresses(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
1215
		ue_ip_status = 0
1216
1217
1218
		self.UEIPAddresses = []
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		for device_id in self.UEDevices:
1219
1220
1221
1222
1223
1224
			count = 0
			while count < 4:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ip addr show | grep rmnet', '\$', 15)
				result = re.search('inet (?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+[0-9a-zA-Z\.\s]+', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
1225
					time.sleep(1)
1226
1227
1228
1229
1230
1231
					count += 1
				else:
					count = 10
			if count < 9:
				ue_ip_status -= 1
				continue
1232
1233
1234
1235
1236
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + device_id + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			for ueipaddress in self.UEIPAddresses:
				if ueipaddress == UE_IPAddress:
					logging.debug('\u001B[1mUE (' + device_id + ') IP Address ' + UE_IPAddress + 'has been existed!' + '\u001B[0m')
1237
1238
					ue_ip_status -= 1
					continue
1239
			self.UEIPAddresses.append(UE_IPAddress)
1240
		self.close()
1241
		return ue_ip_status
1242

Raphael Defosseux's avatar
Raphael Defosseux committed
1243
1244
1245
1246
1247
1248
1249
1250
1251
	def ping_iperf_wrong_exit(self, lock, UE_IPAddress, device_id, statusQueue, message):
		lock.acquire()
		statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
		statusQueue.put(message)
		lock.release()

	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue):
1252
1253
1254
1255
1256
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
1257
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee -a ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1258
1259
1260
1261
1262
1263
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
				return
1264
1265
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1266
1267
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1268
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1269
				return
1270
1271
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1272
1273
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1274
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1275
				return
1276
1277
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1278
1279
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1280
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1281
				return
1282
1283
1284
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1285
1286
1287
1288
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
1289
1290
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1291
1292
1293
1294
1295
1296
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
1297
1298
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1299
					qMsg += '\nPacket Loss too high'
1300
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1301
					packetLossOK = False
1302
				elif float(packetloss) > 0:
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1303
					qMsg += '\nPacket Loss is not 0%'
Raphael Defosseux's avatar
Raphael Defosseux committed
1304
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
Raphael Defosseux's avatar
CI:    
Raphael Defosseux committed
1305
1306
1307
1308
1309
1310
1311
1312
			if (packetLossOK):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(qMsg)
			lock.release()
1313
1314
1315
1316
1317
1318
1319
1320
1321
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def Ping(self):
		if self.<