ran.py 50.6 KB
Newer Older
Raphael Defosseux's avatar
Raphael Defosseux committed
1
#/*
Gabriele Perrone's avatar
Gabriele Perrone committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

31
32
33
34
35
36
37
38
39
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
import time
from multiprocessing import Process, Lock, SimpleQueue
hardy's avatar
hardy committed
40
41
import yaml

42
43
44
45

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
Gabriele Perrone's avatar
Gabriele Perrone committed
46
import sshconnection as SSH
47
48
49
50
51
52
53
import helpreadme as HELP
import constants as CONST

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class RANManagement():
Gabriele Perrone's avatar
Gabriele Perrone committed
54

55
	def __init__(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
56
57
		
		self.prematureExit = False
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.Build_eNB_args = ''
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
		self.Build_eNB_forced_workspace_cleanup = False
		self.Initialize_eNB_args = ''
Remi Hardy's avatar
Remi Hardy committed
80
		self.imageKind = ''
81
82
		self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
		self.eNB_instance = 0
83
		self.eNB_serverId = ['', '', '']
84
85
86
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
		self.eNBmbmsEnables = [False, False, False]
87
88
89
		self.eNBstatuses = [-1, -1, -1]
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
90
91
		self.flexranCtrlDeployed = False
		self.flexranCtrlIpAddress = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
92
		self.testCase_id = ''
93
		self.epcPcapFile = ''
94
		self.runtime_stats= ''
hardy's avatar
hardy committed
95
		self.datalog_rt_stats={}
Gabriele Perrone's avatar
Gabriele Perrone committed
96

hardy's avatar
hardy committed
97

98

Gabriele Perrone's avatar
Gabriele Perrone committed
99
100
101
102
#-----------------------------------------------------------
# RAN management functions
#-----------------------------------------------------------

Raphael Defosseux's avatar
Raphael Defosseux committed
103
	def BuildeNB(self, HTML):
104
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
105
			HELP.GenericHelp(CONST.Version)
106
			sys.exit('Insufficient Parameter')
107
		if self.eNB_serverId[self.eNB_instance] == '0':
108
109
110
111
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
112
		elif self.eNB_serverId[self.eNB_instance] == '1':
113
114
115
116
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
117
		elif self.eNB_serverId[self.eNB_instance] == '2':
118
119
120
121
122
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
123
			HELP.GenericHelp(CONST.Version)
124
			sys.exit('Insufficient Parameter')
125
		logging.debug('Building on server: ' + lIpAddr)
126
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
127
		mySSH.open(lIpAddr, lUserName, lPassWord)
hardy's avatar
hardy committed
128
129
130
		
		# Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB
		result = re.search('--eNBocp', self.Build_eNB_args)
131
		if result is not None:
132
			self.air_interface[self.eNB_instance] = 'ocp-enb'
hardy's avatar
hardy committed
133
134
135
		else:	
			result = re.search('--gNB', self.Build_eNB_args)
			if result is not None:
136
				self.air_interface[self.eNB_instance] = 'nr-softmodem'
hardy's avatar
hardy committed
137
			else:
138
				self.air_interface[self.eNB_instance] = 'lte-softmodem'
139
		
140
141
142
		# Worakround for some servers, we need to erase completely the workspace
		if self.Build_eNB_forced_workspace_cleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
Raphael Defosseux's avatar
Raphael Defosseux committed
143
		self.testCase_id = HTML.testCase_id
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
			full_ran_repo_name = self.ranRepository
		else:
			full_ran_repo_name = self.ranRepository + '.git'
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			mySSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
			if result is not None:
				mismatch = False
				mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
				result = re.search(self.ranCommitID, mySSH.getBefore())
				if result is None:
					mismatch = True
				mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
				if (self.ranAllowMerge):
					result = re.search('YES', mySSH.getBefore())
					if result is None:
						mismatch = True
					mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
					if self.ranTargetBranch == '':
						result = re.search('develop', mySSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, mySSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', mySSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
184
					HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
					return

		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('mkdir -p log', '\$', 5)
		mySSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		if self.backgroundBuild:
			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
208
			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
209
			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
210
			mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
211
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
212
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
213
214
			return
		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
215
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
216
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML)
217

Raphael Defosseux's avatar
Raphael Defosseux committed
218
	def WaitBuildeNBisFinished(self, HTML):
219
		if self.eNB_serverId[self.eNB_instance] == '0':
220
221
222
223
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
224
		elif self.eNB_serverId[self.eNB_instance] == '1':
225
226
227
228
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
229
		elif self.eNB_serverId[self.eNB_instance] == '2':
230
231
232
233
234
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
235
			HELP.GenericHelp(CONST.Version)
236
			sys.exit('Insufficient Parameter')
237
		logging.debug('Waiting for end of build on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
238
		mySSH = SSH.SSHConnection()
239
240
241
242
		mySSH.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
243
			mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6)
244
			result = re.search('build_oai', mySSH.getBefore())
hardy's avatar
hardy committed
245
			print(result)
246
247
248
249
250
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
251
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
252
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)], HTML)
253

Raphael Defosseux's avatar
Raphael Defosseux committed
254
255
	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
		HTML.testCase_id=testcaseId
hardy's avatar
hardy committed
256

257
		mySSH = SSH.SSHConnection()
258
		mySSH.open(lIpAddr, lUserName, lPassWord)
Gabriele Perrone's avatar
Gabriele Perrone committed
259
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
260
261
		mySSH.command('ls ran_build/build', '\$', 3)
		mySSH.command('ls ran_build/build', '\$', 3)
hardy's avatar
hardy committed
262
263

		#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem, ocp-enb)
264
		logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
265
		result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
266
		if result is None:
hardy's avatar
hardy committed
267
			buildStatus = False #if not, build failed
268
		else:
hardy's avatar
hardy committed
269
			buildStatus = True 
270
271
272
273
274
275
276
277
278
279
280
			# Generating a BUILD INFO file
			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			else:
				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
hardy's avatar
hardy committed
281
282
				
				
283
284
285
		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
286
		if self.eNB_serverId[self.eNB_instance] != '0':
287
288
289
290
291
292
293
294
295
296
297
298
			mySSH.command('cd cmake_targets', '\$', 5)
			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
			mySSH.close()
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
Remi Hardy's avatar
Remi Hardy committed
299
300
				#-qq quiet / -u update orcreate files
				mySSH.command('unzip -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
301
302
303
304
305
				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.close()
		else:
			mySSH.close()

hardy's avatar
hardy committed
306
		#generate logging info depending on buildStatus and air interface
307
		if buildStatus:
308
			logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
309
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
310
		else:
311
			logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
312
313
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
314
315
			sys.exit(1)

Raphael Defosseux's avatar
Raphael Defosseux committed
316
	def InitializeeNB(self, HTML, EPC):
317
		if self.eNB_serverId[self.eNB_instance] == '0':
318
319
320
321
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
322
		elif self.eNB_serverId[self.eNB_instance] == '1':
323
324
325
326
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
327
		elif self.eNB_serverId[self.eNB_instance] == '2':
328
329
330
331
332
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
333
			HELP.GenericHelp(CONST.Version)
334
			sys.exit('Insufficient Parameter')
335
		logging.debug('Starting eNB/gNB on server: ' + lIpAddr)
Raphael Defosseux's avatar
Raphael Defosseux committed
336

Raphael Defosseux's avatar
Raphael Defosseux committed
337
		self.testCase_id = HTML.testCase_id
Gabriele Perrone's avatar
Gabriele Perrone committed
338
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
339
		
Gabriele Perrone's avatar
Gabriele Perrone committed
340
		if (self.pStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
341
342
			HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
			HTML.CreateHtmlTabFooter(False)
343
344
345
			sys.exit(1)
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
346
347
348
349
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
350
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
351
352
353
354
355
			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
356
357
358
				self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
			mySSH.close()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
		extra_options = ''
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				mySSH.command('cd common/utils/T/tracer', '\$', 5)
				mySSH.command('make', '\$', 10)
				mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				mySSH.command('cd ' + lSourcePath, '\$', 5)
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
		ci_full_config_file = config_path + '/ci-' + config_file
		rruCheck = False
		result = re.search('^rru|^rcc|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		# do not reset board twice in IF4.5 case
		result = re.search('^rru|^enb|^du.band', str(config_file))
		if result is not None:
			mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
			result = re.search('type: b200', mySSH.getBefore())
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
				# Reloading FGPA bin firmware
				mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
		# Make a copy and adapt to EPC / eNB IP addresses
		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
397
398
		localMmeIpAddr = EPC.MmeIPAddress
		mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
399
		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
400
		mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
401
402
403
		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
404
		mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
405
		if (self.flexranCtrlInstalled and self.flexranCtrlStarted) or self.flexranCtrlDeployed:
406
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
407
			mySSH.command('sed -i -e \'s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/\' ' + ci_full_config_file, '\$', 2);
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
		else:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
		result = re.search('yes', mySSH.getBefore())
		if result is not None:
			self.eNBmbmsEnables[int(self.eNB_instance)] = True
			logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
		result = re.search('noS1', str(self.Initialize_eNB_args))
		eNBinNoS1 = False
		if result is not None:
			eNBinNoS1 = True
			logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
		# Launch eNB with the modified config file
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
424
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
425
			mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
426
			mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
427
428
		# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
		mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
429
430
		mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
431
		mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
		time.sleep(6)
		doLoop = True
		loopCounter = 20
		enbDidSync = False
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					mySSH.command('killall --signal SIGKILL record', '\$', 5)
				mySSH.close()
				doLoop = False
448
				logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
449
				HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
450
451
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
452
453
454
455
				if (result is not None):
					localEpcIpAddr = EPC.IPAddress
					localEpcUserName = EPC.UserName
					localEpcPassword = EPC.Password
456
					mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
457
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
458
459
					mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					if self.epcPcapFile  != '':
460
						time.sleep(0.5)
461
						mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
462
463
					mySSH.close()
					time.sleep(1)
464
465
					if self.epcPcapFile != '':
						copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
466
						if (copyin_res == 0):
467
							mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
468
469
470
				self.prematureExit = True
				return
			else:
471
				mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting|Started"', '\$', 4)
472
473
474
475
476
477
478
479
480
481
482
				if rruCheck:
					result = re.search('wait RUs', mySSH.getBefore())
				else:
					result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					enbDidSync = True
					time.sleep(10)

483
484
485
486
487
		rruCheck = False
		result = re.search('^rru|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		if enbDidSync and eNBinNoS1 and not rruCheck:
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			result = re.search('inet addr:1|inet 1', mySSH.getBefore())
			if result is not None:
				logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
			else:
				logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				mySSH.command('ifconfig oaitun_enm1', '\$', 4)
				result = re.search('inet addr', mySSH.getBefore())
				if result is not None:
					logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
		if enbDidSync:
503
			self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance])
504
505

		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
506
		HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
507
		logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527

	def CheckeNBProcess(self, status_queue):
		try:
			# At least the instance 0 SHALL be on!
			if self.eNBstatuses[0] == 0:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
			elif self.eNBstatuses[0] == 1:
				lIpAddr = self.eNB1IPAddress
				lUserName = self.eNB1UserName
				lPassWord = self.eNB1Password
			elif self.eNBstatuses[0] == 2:
				lIpAddr = self.eNB2IPAddress
				lUserName = self.eNB2UserName
				lPassWord = self.eNB2Password
			else:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
Gabriele Perrone's avatar
Gabriele Perrone committed
528
			mySSH = SSH.SSHConnection()
529
			mySSH.open(lIpAddr, lUserName, lPassWord)
530
531
			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface[self.eNB_instance] + ' | grep -v grep', '\$', 5)
			result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
532
533
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
Gabriele Perrone's avatar
Gabriele Perrone committed
534
				status_queue.put(CONST.ENB_PROCESS_FAILED)
535
			else:
Gabriele Perrone's avatar
Gabriele Perrone committed
536
				status_queue.put(CONST.ENB_PROCESS_OK)
537
538
539
540
			mySSH.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
541
	def TerminateeNB(self, HTML, EPC):
542
		if self.eNB_serverId[self.eNB_instance] == '0':
543
544
545
546
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
547
		elif self.eNB_serverId[self.eNB_instance] == '1':
548
549
550
551
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
552
		elif self.eNB_serverId[self.eNB_instance] == '2':
553
554
555
556
557
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
558
559
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
560
		logging.debug('Stopping eNB/gNB on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
561
		mySSH = SSH.SSHConnection()
562
563
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
564
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
565
566
567
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
568
569
		mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
		result = re.search('(-softmodem|ocp)', mySSH.getBefore())
570
		if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
571
			mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5)
572
			time.sleep(10)
hardy's avatar
hardy committed
573
574
			mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
			result = re.search('(-softmodem|ocp)', mySSH.getBefore())
575
			if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
576
				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
577
578
579
580
581
				time.sleep(5)
		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.close()
		# If tracer options is on, stopping tshark on EPC side
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
582
583
584
585
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
586
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
587
			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
588
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
589
			time.sleep(1)
590
591
592
593
			if self.epcPcapFile != '':
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
				mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
			mySSH.close()
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			mySSH.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
			mySSH.close()
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
609
610
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file, HTML)
			HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
611
612
613
614
615
616
617
618
619
620
621
			self.eNBLogFiles[int(self.eNB_instance)] = ''
		else:
			analyzeFile = False
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
				analyzeFile = True
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
			if analyzeFile:
				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
622
623
					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
624
625
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
626
				if self.eNB_serverId[self.eNB_instance] != '0':
627
628
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
Raphael Defosseux's avatar
Raphael Defosseux committed
629
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML)
630
				if (logStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
631
					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
632
633
634
635
					self.preamtureExit = True
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				else:
Raphael Defosseux's avatar
Raphael Defosseux committed
636
					HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
637
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
638
				HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
hardy's avatar
hardy committed
639
640
		if len(self.datalog_rt_stats)!=0:
			HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
641
642
643
644
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		self.eNBstatuses[int(self.eNB_instance)] = -1

	def LogCollecteNB(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
645
		mySSH = SSH.SSHConnection()
646
647
648
649
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
650
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60)
651
652
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
		mySSH.close()
Gabriele Perrone's avatar
Gabriele Perrone committed
653

Raphael Defosseux's avatar
Raphael Defosseux committed
654
	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
Gabriele Perrone's avatar
Gabriele Perrone committed
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
		if (not os.path.isfile('./' + eNBlogFile)):
			return -1
		enb_log_file = open('./' + eNBlogFile, 'r')
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
		uciStatMsgCount = 0
		pdcpFailure = 0
		ulschFailure = 0
676
677
		ulschAllocateCCEerror = 0
		uplinkSegmentsAborted = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
678
		ulschReceiveOK = 0
679
		gnbRxTxWakeUpFailure = 0
680
		gnbTxWriteThreadEnabled = False
Gabriele Perrone's avatar
Gabriele Perrone committed
681
682
683
		cdrxActivationMessageCount = 0
		dropNotEnoughRBs = 0
		mbmsRequestMsg = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
684
		htmleNBFailureMsg = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
685
686
687
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
Gabriele Perrone's avatar
Gabriele Perrone committed
688
		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
689
690
		X2HO_inNbProcedures = 0
		X2HO_outNbProcedures = 0
691
		global_status = CONST.ALL_PROCESSES_OK
692
693
694
695
696
697
		# Runtime statistics
		runTime = ''
		userTime = ''
		systemTime = ''
		maxPhyMemUsage = ''
		nbContextSwitches = ''
hardy's avatar
hardy committed
698
		#NSA FR1 check
hardy's avatar
hardy committed
699
		NSA_RAPROC_PUSCH_check = 0
hardy's avatar
hardy committed
700
701
		#dlsch and ulsch statistics (dictionary)
		dlsch_ulsch_stats = {}
702
		#real time statistics (dictionary)
703
		real_time_stats = {}
Remi Hardy's avatar
Remi Hardy committed
704
705
		#count "problem receiving samples" msg
		pb_receiving_samples_cnt = 0
Remi Hardy's avatar
Remi Hardy committed
706
	
hardy's avatar
hardy committed
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
		#the datalog config file has to be loaded
		datalog_rt_stats_file='datalog_rt_stats.yaml'
		if (os.path.isfile(datalog_rt_stats_file)):
			yaml_file=datalog_rt_stats_file
		elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)):
			yaml_file='ci-scripts/'+datalog_rt_stats_file
		else:
			logging.error("Datalog RT stats yaml file cannot be found")
			sys.exit("Datalog RT stats yaml file cannot be found")

		with open(yaml_file,'r') as f:
			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  


Gabriele Perrone's avatar
Gabriele Perrone committed
722
		for line in enb_log_file.readlines():
723
724
725
726
727
728
729
			# Runtime statistics
			result = re.search('Run time:' ,str(line))
			if result is not None:
				runTime = str(line).strip()
			if runTime != '':
				result = re.search('Time executing user inst', str(line))
				if result is not None:
730
731
					fields=line.split(':')
					userTime = 'userTime : ' + fields[1].replace('\n','')
732
733
				result = re.search('Time executing system inst', str(line))
				if result is not None:
734
735
					fields=line.split(':')
					systemTime = 'systemTime : ' + fields[1].replace('\n','')
736
737
				result = re.search('Max. Phy. memory usage:', str(line))
				if result is not None:
738
739
					fields=line.split(':')
					maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
740
741
				result = re.search('Number of context switch.*process origin', str(line))
				if result is not None:
742
743
					fields=line.split(':')
					nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
Gabriele Perrone's avatar
Gabriele Perrone committed
744
			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
Gabriele Perrone's avatar
Gabriele Perrone committed
745
746
				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
747
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
Gabriele Perrone's avatar
Gabriele Perrone committed
748
749
				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
750
751
					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
752
753
				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
754
755
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
Gabriele Perrone's avatar
Gabriele Perrone committed
756
757
				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
758
759
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
760
761
				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
762
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
763
					X2HO_inNbProcedures += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
764
			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
Gabriele Perrone's avatar
Gabriele Perrone committed
765
766
				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
767
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
768
769
770
771
772
773
774
775
776
					X2HO_outNbProcedures += 1

			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
Raphael Defosseux's avatar
Raphael Defosseux committed
777
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
778
					else:
Raphael Defosseux's avatar
Raphael Defosseux committed
779
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./ran_build/build/lte-softmodem', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
hardy's avatar
hardy committed
816
			if result is not None:				rrcReleaseRequest += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
841
842
843
			result = re.search('could not wakeup gNB rxtx process', str(line))
			if result is not None:
				gnbRxTxWakeUpFailure += 1
844
845
846
			result = re.search('tx write thread ready', str(line))
			if result is not None:
				gnbTxWriteThreadEnabled = True
847
			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
Gabriele Perrone's avatar
Gabriele Perrone committed
848
849
			if result is not None:
				ulschFailure += 1
850
851
852
853
854
855
			result = re.search('ERROR ALLOCATING CCEs', str(line))
			if result is not None:
				ulschAllocateCCEerror += 1
			result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
			if result is not None:
				uplinkSegmentsAborted += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
856
857
858
			result = re.search('ULSCH received ok', str(line))
			if result is not None:
				ulschReceiveOK += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
859
860
861
862
863
864
865
866
867
868
869
870
871
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
				if result is not None:
					mbmsRequestMsg += 1
hardy's avatar
hardy committed
872
873
874
875
			#FR1 NSA test : add new markers to make sure gNB is used
			result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI [0-9a-fA-F]+ received correctly, adding UE MAC Context UE_id [0-9]+\/RNTI [0-9a-fA-F]+', str(line))
			if result is not None:
				NSA_RAPROC_PUSCH_check = 1
hardy's avatar
hardy committed
876
			#dlsch and ulsch statistics
877
			#keys below are the markers we are loooking for, loop over this keys list
hardy's avatar
hardy committed
878
			#everytime these markers are found in the log file, the previous ones are overwritten in the dict
Remi Hardy's avatar
Remi Hardy committed
879
			#eventually we record and print only the last occurence 
hardy's avatar
hardy committed
880
			keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'}
hardy's avatar
hardy committed
881
			for k in keys:
882
				result = re.search(k, line)
hardy's avatar
hardy committed
883
				if result is not None:
Remi Hardy's avatar
Remi Hardy committed
884
885
					#remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char
					dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
hardy's avatar
hardy committed
886
887
			#real time statistics for gNB
			for k in rt_keys:
888
889
890
				result = re.search(k, line)     
				if result is not None:
					#remove 1- all useless char before relevant info  2- trailing char
hardy's avatar
hardy committed
891
					line=line.replace('[0m','')
892
893
					tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex 
					real_time_stats[k]=tmp.group(1)
hardy's avatar
hardy committed
894

Remi Hardy's avatar
Remi Hardy committed
895
896
897
898
899
			#count "problem receiving samples" msg
			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
			if result is not None:
				pb_receiving_samples_cnt += 1				

Gabriele Perrone's avatar
Gabriele Perrone committed
900
901
		enb_log_file.close()
		logging.debug('   File analysis completed')
902
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
Gabriele Perrone's avatar
Gabriele Perrone committed
903
904
905
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
906

907
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
Raphael Defosseux's avatar
Raphael Defosseux committed
908
909
910
911
			if ulschReceiveOK > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
912
913
914
915
			if gnbRxTxWakeUpFailure > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
916
917
918
			if gnbTxWriteThreadEnabled:
				statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
919
				htmleNBFailureMsg += statMsg + '\n'
hardy's avatar
hardy committed
920
			#FR1 NSA test : add new markers to make sure gNB is used
921
922
923
924
			if NSA_RAPROC_PUSCH_check:
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS '
				htmlMsg = statMsg+'\n'
			else:
925
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant'
926
927
928
				htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
929
930
931
932
933
934
			#problem receiving samples log
			statMsg = '[PHY] problem receiving samples msg count =  '+str(pb_receiving_samples_cnt)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg

935
936
937
938
			#ulsch and dlsch statistics
			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
				statMsg=''
				for key in dlsch_ulsch_stats: #for each dictionary key
Remi Hardy's avatar
bug fix    
Remi Hardy committed
939
					statMsg += dlsch_ulsch_stats[key] + '\n' 
940
941
					logging.debug(dlsch_ulsch_stats[key])
				htmleNBFailureMsg += statMsg
hardy's avatar
hardy committed
942

hardy's avatar
hardy committed
943
944
			#real time statistics
			datalog_rt_stats['Data']={}
945
			if len(real_time_stats)!=0: #check if dictionary is not empty
hardy's avatar
hardy committed
946
947
948
949
950
951
952
				for k in real_time_stats:
					tmp=re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+\d+;\s+(?P<max>\d+\.\d+) us;',real_time_stats[k])
					if tmp is not None:
						metric=tmp.group('metric')
						avg=float(tmp.group('avg'))
						max=float(tmp.group('max'))
						datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])]
953
954
955
956
957
958
959
				#once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward
				self.datalog_rt_stats=datalog_rt_stats
				#check if there is a fail => will render the test as failed
				for k in datalog_rt_stats['Data']:
					if float(datalog_rt_stats['Data'][k][2])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold
						#setting prematureExit is ok although not the best option
						self.prematureExit=True
960
961
962
963
			else:
				statMsg = 'No real time stats found in the log file\n'
				logging.debug('No real time stats found in the log file')
				htmleNBFailureMsg += statMsg
964

Gabriele Perrone's avatar
Gabriele Perrone committed
965
966
967
		if uciStatMsgCount > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
968
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
969
970
971
		if pdcpFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
972
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
973
974
975
		if ulschFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
976
			htmleNBFailureMsg += statMsg + '\n'
977
978
979
980
981
982
983
984
		if ulschAllocateCCEerror > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
		if uplinkSegmentsAborted > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
985
986
987
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
988
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
989
990
991
		if rrcSetupComplete > 0:
			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
992
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
993
994
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
995
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
996
997
998
		if rrcReleaseRequest > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
999
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1000
1001
1002
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1003
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1004
1005
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1006
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1007
1008
1009
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1010
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1011
1012
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1013
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1014
1015
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1016
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1017
1018
1019
1020
		if self.eNBmbmsEnables[int(self.eNB_instance)]:
			if mbmsRequestMsg > 0:
				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1021
				htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1022
1023
1024
		if X2HO_inNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1025
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1026
1027
1028
		if X2HO_outNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1029
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1030
1031
1032
1033
1034
1035
		if self.eNBOptions[int(self.eNB_instance)] != '':
			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
			if res1 is not None:
				if cdrxActivationMessageCount > 0:
					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1036
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1037
1038
1039
				else:
					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1040
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1041
1042
1043
		if rachCanceledProcedure > 0:
			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1044
			htmleNBFailureMsg += rachMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1045
1046
1047
1048
1049
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1050
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1051
1052
1053
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')