ran.py 49.3 KB
Newer Older
Raphael Defosseux's avatar
Raphael Defosseux committed
1
#/*
Gabriele Perrone's avatar
Gabriele Perrone committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

31
32
33
34
35
36
37
38
39
40
41
42
43
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
import time
from multiprocessing import Process, Lock, SimpleQueue

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
Gabriele Perrone's avatar
Gabriele Perrone committed
44
import sshconnection as SSH
45
46
47
48
49
50
51
import helpreadme as HELP
import constants as CONST

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class RANManagement():
Gabriele Perrone's avatar
Gabriele Perrone committed
52

53
	def __init__(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
54
55
		
		self.prematureExit = False
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.Build_eNB_args = ''
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
		self.Build_eNB_forced_workspace_cleanup = False
		self.Initialize_eNB_args = ''
Remi Hardy's avatar
Remi Hardy committed
78
		self.imageKind = ''
79
80
		self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
		self.eNB_instance = 0
81
		self.eNB_serverId = ['', '', '']
82
83
84
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
		self.eNBmbmsEnables = [False, False, False]
85
86
87
		self.eNBstatuses = [-1, -1, -1]
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
88
89
		self.flexranCtrlDeployed = False
		self.flexranCtrlIpAddress = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
90
		self.testCase_id = ''
91
		self.epcPcapFile = ''
92
		self.runtime_stats= ''
Gabriele Perrone's avatar
Gabriele Perrone committed
93

hardy's avatar
hardy committed
94

95

Gabriele Perrone's avatar
Gabriele Perrone committed
96
97
98
99
#-----------------------------------------------------------
# RAN management functions
#-----------------------------------------------------------

Raphael Defosseux's avatar
Raphael Defosseux committed
100
	def BuildeNB(self, HTML):
101
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
102
			HELP.GenericHelp(CONST.Version)
103
			sys.exit('Insufficient Parameter')
104
		if self.eNB_serverId[self.eNB_instance] == '0':
105
106
107
108
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
109
		elif self.eNB_serverId[self.eNB_instance] == '1':
110
111
112
113
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
114
		elif self.eNB_serverId[self.eNB_instance] == '2':
115
116
117
118
119
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
120
			HELP.GenericHelp(CONST.Version)
121
			sys.exit('Insufficient Parameter')
122
		logging.debug('Building on server: ' + lIpAddr)
123
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
124
		mySSH.open(lIpAddr, lUserName, lPassWord)
hardy's avatar
hardy committed
125
126
127
		
		# Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB
		result = re.search('--eNBocp', self.Build_eNB_args)
128
		if result is not None:
129
			self.air_interface[self.eNB_instance] = 'ocp-enb'
hardy's avatar
hardy committed
130
131
132
		else:	
			result = re.search('--gNB', self.Build_eNB_args)
			if result is not None:
133
				self.air_interface[self.eNB_instance] = 'nr-softmodem'
hardy's avatar
hardy committed
134
			else:
135
				self.air_interface[self.eNB_instance] = 'lte-softmodem'
136
		
137
138
139
		# Worakround for some servers, we need to erase completely the workspace
		if self.Build_eNB_forced_workspace_cleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
Raphael Defosseux's avatar
Raphael Defosseux committed
140
		self.testCase_id = HTML.testCase_id
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
			full_ran_repo_name = self.ranRepository
		else:
			full_ran_repo_name = self.ranRepository + '.git'
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			mySSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
			if result is not None:
				mismatch = False
				mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
				result = re.search(self.ranCommitID, mySSH.getBefore())
				if result is None:
					mismatch = True
				mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
				if (self.ranAllowMerge):
					result = re.search('YES', mySSH.getBefore())
					if result is None:
						mismatch = True
					mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
					if self.ranTargetBranch == '':
						result = re.search('develop', mySSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, mySSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', mySSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
181
					HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
					return

		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('mkdir -p log', '\$', 5)
		mySSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		if self.backgroundBuild:
			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
205
			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
206
			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
207
			mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
208
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
209
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
210
211
			return
		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
212
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
213
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML)
214

Raphael Defosseux's avatar
Raphael Defosseux committed
215
	def WaitBuildeNBisFinished(self, HTML):
216
		if self.eNB_serverId[self.eNB_instance] == '0':
217
218
219
220
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
221
		elif self.eNB_serverId[self.eNB_instance] == '1':
222
223
224
225
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
226
		elif self.eNB_serverId[self.eNB_instance] == '2':
227
228
229
230
231
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
232
			HELP.GenericHelp(CONST.Version)
233
			sys.exit('Insufficient Parameter')
234
		logging.debug('Waiting for end of build on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
235
		mySSH = SSH.SSHConnection()
236
237
238
239
		mySSH.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
240
			mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6)
241
			result = re.search('build_oai', mySSH.getBefore())
hardy's avatar
hardy committed
242
			print(result)
243
244
245
246
247
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
248
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
249
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)], HTML)
250

Raphael Defosseux's avatar
Raphael Defosseux committed
251
252
	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
		HTML.testCase_id=testcaseId
hardy's avatar
hardy committed
253

254
		mySSH = SSH.SSHConnection()
255
		mySSH.open(lIpAddr, lUserName, lPassWord)
Gabriele Perrone's avatar
Gabriele Perrone committed
256
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
257
258
		mySSH.command('ls ran_build/build', '\$', 3)
		mySSH.command('ls ran_build/build', '\$', 3)
hardy's avatar
hardy committed
259
260

		#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem, ocp-enb)
261
		logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
262
		result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
263
		if result is None:
hardy's avatar
hardy committed
264
			buildStatus = False #if not, build failed
265
		else:
hardy's avatar
hardy committed
266
			buildStatus = True 
267
268
269
270
271
272
273
274
275
276
277
			# Generating a BUILD INFO file
			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			else:
				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
hardy's avatar
hardy committed
278
279
				
				
280
281
282
		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
283
		if self.eNB_serverId[self.eNB_instance] != '0':
284
285
286
287
288
289
290
291
292
293
294
295
			mySSH.command('cd cmake_targets', '\$', 5)
			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
			mySSH.close()
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
Remi Hardy's avatar
Remi Hardy committed
296
297
				#-qq quiet / -u update orcreate files
				mySSH.command('unzip -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
298
299
300
301
302
				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.close()
		else:
			mySSH.close()

hardy's avatar
hardy committed
303
		#generate logging info depending on buildStatus and air interface
304
		if buildStatus:
305
			logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
306
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
307
		else:
308
			logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
309
310
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
311
312
			sys.exit(1)

Raphael Defosseux's avatar
Raphael Defosseux committed
313
	def InitializeeNB(self, HTML, EPC):
314
		if self.eNB_serverId[self.eNB_instance] == '0':
315
316
317
318
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
319
		elif self.eNB_serverId[self.eNB_instance] == '1':
320
321
322
323
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
324
		elif self.eNB_serverId[self.eNB_instance] == '2':
325
326
327
328
329
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
330
			HELP.GenericHelp(CONST.Version)
331
			sys.exit('Insufficient Parameter')
332
		logging.debug('Starting eNB/gNB on server: ' + lIpAddr)
Raphael Defosseux's avatar
Raphael Defosseux committed
333

Raphael Defosseux's avatar
Raphael Defosseux committed
334
		self.testCase_id = HTML.testCase_id
Gabriele Perrone's avatar
Gabriele Perrone committed
335
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
336
		
Gabriele Perrone's avatar
Gabriele Perrone committed
337
		if (self.pStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
338
339
			HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
			HTML.CreateHtmlTabFooter(False)
340
341
342
			sys.exit(1)
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
343
344
345
346
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
347
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
348
349
350
351
352
			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
353
354
355
				self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
			mySSH.close()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
		extra_options = ''
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				mySSH.command('cd common/utils/T/tracer', '\$', 5)
				mySSH.command('make', '\$', 10)
				mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				mySSH.command('cd ' + lSourcePath, '\$', 5)
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
		ci_full_config_file = config_path + '/ci-' + config_file
		rruCheck = False
		result = re.search('^rru|^rcc|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		# do not reset board twice in IF4.5 case
		result = re.search('^rru|^enb|^du.band', str(config_file))
		if result is not None:
			mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
			result = re.search('type: b200', mySSH.getBefore())
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
				# Reloading FGPA bin firmware
				mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
		# Make a copy and adapt to EPC / eNB IP addresses
		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
394
395
		localMmeIpAddr = EPC.MmeIPAddress
		mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
396
		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
397
		mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
398
399
400
		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
401
		mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
402
		if (self.flexranCtrlInstalled and self.flexranCtrlStarted) or self.flexranCtrlDeployed:
403
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
404
			mySSH.command('sed -i -e \'s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/\' ' + ci_full_config_file, '\$', 2);
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
		else:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
		result = re.search('yes', mySSH.getBefore())
		if result is not None:
			self.eNBmbmsEnables[int(self.eNB_instance)] = True
			logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
		result = re.search('noS1', str(self.Initialize_eNB_args))
		eNBinNoS1 = False
		if result is not None:
			eNBinNoS1 = True
			logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
		# Launch eNB with the modified config file
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
421
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
422
			mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
423
			mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
424
425
		# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
		mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
426
427
		mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
428
		mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
		time.sleep(6)
		doLoop = True
		loopCounter = 20
		enbDidSync = False
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					mySSH.command('killall --signal SIGKILL record', '\$', 5)
				mySSH.close()
				doLoop = False
445
				logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
446
				HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
447
448
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
449
450
451
452
				if (result is not None):
					localEpcIpAddr = EPC.IPAddress
					localEpcUserName = EPC.UserName
					localEpcPassword = EPC.Password
453
					mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
454
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
455
456
					mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					if self.epcPcapFile  != '':
457
						time.sleep(0.5)
458
						mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
459
460
					mySSH.close()
					time.sleep(1)
461
462
					if self.epcPcapFile != '':
						copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
463
						if (copyin_res == 0):
464
							mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
465
466
467
				self.prematureExit = True
				return
			else:
468
				mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting|Started"', '\$', 4)
469
470
471
472
473
474
475
476
477
478
479
				if rruCheck:
					result = re.search('wait RUs', mySSH.getBefore())
				else:
					result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					enbDidSync = True
					time.sleep(10)

480
481
482
483
484
		rruCheck = False
		result = re.search('^rru|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		if enbDidSync and eNBinNoS1 and not rruCheck:
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			result = re.search('inet addr:1|inet 1', mySSH.getBefore())
			if result is not None:
				logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
			else:
				logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				mySSH.command('ifconfig oaitun_enm1', '\$', 4)
				result = re.search('inet addr', mySSH.getBefore())
				if result is not None:
					logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
		if enbDidSync:
500
			self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance])
501
502

		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
503
		HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
504
		logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524

	def CheckeNBProcess(self, status_queue):
		try:
			# At least the instance 0 SHALL be on!
			if self.eNBstatuses[0] == 0:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
			elif self.eNBstatuses[0] == 1:
				lIpAddr = self.eNB1IPAddress
				lUserName = self.eNB1UserName
				lPassWord = self.eNB1Password
			elif self.eNBstatuses[0] == 2:
				lIpAddr = self.eNB2IPAddress
				lUserName = self.eNB2UserName
				lPassWord = self.eNB2Password
			else:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
Gabriele Perrone's avatar
Gabriele Perrone committed
525
			mySSH = SSH.SSHConnection()
526
			mySSH.open(lIpAddr, lUserName, lPassWord)
527
528
			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface[self.eNB_instance] + ' | grep -v grep', '\$', 5)
			result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
529
530
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
Gabriele Perrone's avatar
Gabriele Perrone committed
531
				status_queue.put(CONST.ENB_PROCESS_FAILED)
532
			else:
Gabriele Perrone's avatar
Gabriele Perrone committed
533
				status_queue.put(CONST.ENB_PROCESS_OK)
534
535
536
537
			mySSH.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
538
	def TerminateeNB(self, HTML, EPC):
539
		if self.eNB_serverId[self.eNB_instance] == '0':
540
541
542
543
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
544
		elif self.eNB_serverId[self.eNB_instance] == '1':
545
546
547
548
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
549
		elif self.eNB_serverId[self.eNB_instance] == '2':
550
551
552
553
554
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
555
556
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
557
		logging.debug('Stopping eNB/gNB on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
558
		mySSH = SSH.SSHConnection()
559
560
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
561
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
562
563
564
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
565
566
		mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
		result = re.search('(-softmodem|ocp)', mySSH.getBefore())
567
		if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
568
			mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5)
569
			time.sleep(10)
hardy's avatar
hardy committed
570
571
			mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
			result = re.search('(-softmodem|ocp)', mySSH.getBefore())
572
			if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
573
				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
574
575
576
577
578
				time.sleep(5)
		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.close()
		# If tracer options is on, stopping tshark on EPC side
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
579
580
581
582
		if (result is not None):
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
583
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
584
			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
585
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
586
			time.sleep(1)
587
588
589
590
			if self.epcPcapFile != '':
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
				mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
			mySSH.close()
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			mySSH.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
			mySSH.close()
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
606
607
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file, HTML)
			HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
608
609
610
611
612
613
614
615
616
617
618
			self.eNBLogFiles[int(self.eNB_instance)] = ''
		else:
			analyzeFile = False
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
				analyzeFile = True
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
			if analyzeFile:
				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
619
620
					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
621
622
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
623
				if self.eNB_serverId[self.eNB_instance] != '0':
624
625
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
Raphael Defosseux's avatar
Raphael Defosseux committed
626
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML)
627
				if (logStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
628
					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
629
630
631
632
					self.preamtureExit = True
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				else:
Raphael Defosseux's avatar
Raphael Defosseux committed
633
					HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
634
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
635
				HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
636
637
638
639
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		self.eNBstatuses[int(self.eNB_instance)] = -1

	def LogCollecteNB(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
640
		mySSH = SSH.SSHConnection()
641
642
643
644
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
645
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60)
646
647
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
		mySSH.close()
Gabriele Perrone's avatar
Gabriele Perrone committed
648

Raphael Defosseux's avatar
Raphael Defosseux committed
649
	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
Gabriele Perrone's avatar
Gabriele Perrone committed
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
		if (not os.path.isfile('./' + eNBlogFile)):
			return -1
		enb_log_file = open('./' + eNBlogFile, 'r')
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
		uciStatMsgCount = 0
		pdcpFailure = 0
		ulschFailure = 0
671
672
		ulschAllocateCCEerror = 0
		uplinkSegmentsAborted = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
673
		ulschReceiveOK = 0
674
		gnbRxTxWakeUpFailure = 0
675
		gnbTxWriteThreadEnabled = False
Gabriele Perrone's avatar
Gabriele Perrone committed
676
677
678
		cdrxActivationMessageCount = 0
		dropNotEnoughRBs = 0
		mbmsRequestMsg = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
679
		htmleNBFailureMsg = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
680
681
682
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
Gabriele Perrone's avatar
Gabriele Perrone committed
683
		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
684
685
		X2HO_inNbProcedures = 0
		X2HO_outNbProcedures = 0
686
		global_status = CONST.ALL_PROCESSES_OK
687
688
689
690
691
692
		# Runtime statistics
		runTime = ''
		userTime = ''
		systemTime = ''
		maxPhyMemUsage = ''
		nbContextSwitches = ''
hardy's avatar
hardy committed
693
		#NSA FR1 check
hardy's avatar
hardy committed
694
		NSA_RAPROC_PUSCH_check = 0
hardy's avatar
hardy committed
695
696
		#dlsch and ulsch statistics (dictionary)
		dlsch_ulsch_stats = {}
697
		#real time statistics (dictionary)
698
		real_time_stats = {}
Remi Hardy's avatar
Remi Hardy committed
699
700
		#count "problem receiving samples" msg
		pb_receiving_samples_cnt = 0
Remi Hardy's avatar
Remi Hardy committed
701
	
Gabriele Perrone's avatar
Gabriele Perrone committed
702
		for line in enb_log_file.readlines():
703
704
705
706
707
708
709
			# Runtime statistics
			result = re.search('Run time:' ,str(line))
			if result is not None:
				runTime = str(line).strip()
			if runTime != '':
				result = re.search('Time executing user inst', str(line))
				if result is not None:
710
711
					fields=line.split(':')
					userTime = 'userTime : ' + fields[1].replace('\n','')
712
713
				result = re.search('Time executing system inst', str(line))
				if result is not None:
714
715
					fields=line.split(':')
					systemTime = 'systemTime : ' + fields[1].replace('\n','')
716
717
				result = re.search('Max. Phy. memory usage:', str(line))
				if result is not None:
718
719
					fields=line.split(':')
					maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
720
721
				result = re.search('Number of context switch.*process origin', str(line))
				if result is not None:
722
723
					fields=line.split(':')
					nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
Gabriele Perrone's avatar
Gabriele Perrone committed
724
			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
Gabriele Perrone's avatar
Gabriele Perrone committed
725
726
				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
727
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
Gabriele Perrone's avatar
Gabriele Perrone committed
728
729
				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
730
731
					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
732
733
				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
734
735
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
Gabriele Perrone's avatar
Gabriele Perrone committed
736
737
				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
738
739
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
740
741
				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
742
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
743
					X2HO_inNbProcedures += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
744
			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
Gabriele Perrone's avatar
Gabriele Perrone committed
745
746
				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
747
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
748
749
750
751
752
753
754
755
756
					X2HO_outNbProcedures += 1

			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
Raphael Defosseux's avatar
Raphael Defosseux committed
757
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
758
					else:
Raphael Defosseux's avatar
Raphael Defosseux committed
759
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./ran_build/build/lte-softmodem', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
hardy's avatar
hardy committed
796
			if result is not None:				rrcReleaseRequest += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
821
822
823
			result = re.search('could not wakeup gNB rxtx process', str(line))
			if result is not None:
				gnbRxTxWakeUpFailure += 1
824
825
826
			result = re.search('tx write thread ready', str(line))
			if result is not None:
				gnbTxWriteThreadEnabled = True
827
			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
Gabriele Perrone's avatar
Gabriele Perrone committed
828
829
			if result is not None:
				ulschFailure += 1
830
831
832
833
834
835
			result = re.search('ERROR ALLOCATING CCEs', str(line))
			if result is not None:
				ulschAllocateCCEerror += 1
			result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
			if result is not None:
				uplinkSegmentsAborted += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
836
837
838
			result = re.search('ULSCH received ok', str(line))
			if result is not None:
				ulschReceiveOK += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
839
840
841
842
843
844
845
846
847
848
849
850
851
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
				if result is not None:
					mbmsRequestMsg += 1
hardy's avatar
hardy committed
852
853
854
855
			#FR1 NSA test : add new markers to make sure gNB is used
			result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI [0-9a-fA-F]+ received correctly, adding UE MAC Context UE_id [0-9]+\/RNTI [0-9a-fA-F]+', str(line))
			if result is not None:
				NSA_RAPROC_PUSCH_check = 1
hardy's avatar
hardy committed
856
			#dlsch and ulsch statistics
857
			#keys below are the markers we are loooking for, loop over this keys list
hardy's avatar
hardy committed
858
			#everytime these markers are found in the log file, the previous ones are overwritten in the dict
Remi Hardy's avatar
Remi Hardy committed
859
			#eventually we record and print only the last occurence 
860
			keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled', 'scheduling timing stats'}
hardy's avatar
hardy committed
861
			for k in keys:
862
				result = re.search(k, line)
hardy's avatar
hardy committed
863
				if result is not None:
Remi Hardy's avatar
Remi Hardy committed
864
865
					#remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char
					dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
866
867
868
			#real time statistics
			#same method as above
			keys = {'feprx','feptx_prec','feptx_ofdm','feptx_total','L1 Tx processing','DLSCH encoding','L1 Rx processing','PUSCH inner-receiver','PUSCH decoding'}   
869
870
871
872
873
874
			for k in keys:
				result = re.search(k, line)     
				if result is not None:
					#remove 1- all useless char before relevant info  2- trailing char
					tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex 
					real_time_stats[k]=tmp.group(1)
Remi Hardy's avatar
Remi Hardy committed
875
876
877
878
879
			#count "problem receiving samples" msg
			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
			if result is not None:
				pb_receiving_samples_cnt += 1				

Gabriele Perrone's avatar
Gabriele Perrone committed
880
881
		enb_log_file.close()
		logging.debug('   File analysis completed')
882
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
Gabriele Perrone's avatar
Gabriele Perrone committed
883
884
885
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
886

887
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
Raphael Defosseux's avatar
Raphael Defosseux committed
888
889
890
891
			if ulschReceiveOK > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
892
893
894
895
			if gnbRxTxWakeUpFailure > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
896
897
898
			if gnbTxWriteThreadEnabled:
				statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
899
				htmleNBFailureMsg += statMsg + '\n'
hardy's avatar
hardy committed
900
			#FR1 NSA test : add new markers to make sure gNB is used
901
902
903
904
			if NSA_RAPROC_PUSCH_check:
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS '
				htmlMsg = statMsg+'\n'
			else:
905
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant'
906
907
908
				htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
909
910
911
912
913
914
			#problem receiving samples log
			statMsg = '[PHY] problem receiving samples msg count =  '+str(pb_receiving_samples_cnt)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg

915
916
917
918
			#ulsch and dlsch statistics
			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
				statMsg=''
				for key in dlsch_ulsch_stats: #for each dictionary key
Remi Hardy's avatar
bug fix    
Remi Hardy committed
919
					statMsg += dlsch_ulsch_stats[key] + '\n' 
920
921
					logging.debug(dlsch_ulsch_stats[key])
				htmleNBFailureMsg += statMsg
hardy's avatar
hardy committed
922

923
924
925
926
927
928
929
			#real time statistics statistics
			if len(real_time_stats)!=0: #check if dictionary is not empty
				statMsg=''
				for key in real_time_stats: #for each dictionary key
					statMsg += real_time_stats[key] + '\n' 
					logging.debug(real_time_stats[key])
				htmleNBFailureMsg += statMsg
930
931
932
933
			else:
				statMsg = 'No real time stats found in the log file\n'
				logging.debug('No real time stats found in the log file')
				htmleNBFailureMsg += statMsg
934

Gabriele Perrone's avatar
Gabriele Perrone committed
935
936
937
		if uciStatMsgCount > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
938
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
939
940
941
		if pdcpFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
942
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
943
944
945
		if ulschFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
946
			htmleNBFailureMsg += statMsg + '\n'
947
948
949
950
951
952
953
954
		if ulschAllocateCCEerror > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
		if uplinkSegmentsAborted > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
955
956
957
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
958
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
959
960
961
		if rrcSetupComplete > 0:
			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
962
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
963
964
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
965
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
966
967
968
		if rrcReleaseRequest > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
969
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
970
971
972
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
973
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
974
975
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
976
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
977
978
979
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
980
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
981
982
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
983
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
984
985
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
986
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
987
988
989
990
		if self.eNBmbmsEnables[int(self.eNB_instance)]:
			if mbmsRequestMsg > 0:
				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
991
				htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
992
993
994
		if X2HO_inNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
995
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
996
997
998
		if X2HO_outNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
999
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1000
1001
1002
1003
1004
1005
		if self.eNBOptions[int(self.eNB_instance)] != '':
			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
			if res1 is not None:
				if cdrxActivationMessageCount > 0:
					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1006
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1007
1008
1009
				else:
					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1010
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1011
1012
1013
		if rachCanceledProcedure > 0:
			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1014
			htmleNBFailureMsg += rachMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1015
1016
1017
1018
1019
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1020
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1021
1022
1023
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1024
					htmleNBFailureMsg += rruMsg + '\n'
1025
					self.prematureExit = True
1026
					global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
Gabriele Perrone's avatar
Gabriele Perrone committed
1027
1028
		if foundSegFault:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
1029
			global_status = CONST.ENB_PROCESS_SEG_FAULT
Gabriele Perrone's avatar
Gabriele Perrone committed
1030
1031
		if foundAssertion:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1032
			htmleNBFailureMsg += msgAssertion
1033
			global_status = CONST.ENB_PROCESS_ASSERTION
Gabriele Perrone's avatar
Gabriele Perrone committed
1034
1035
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1036
			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues!\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1037
1038
1039
		if rlcDiscardBuffer > 0:
			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1040
			htmleNBFailureMsg += rlcMsg + '\n'
1041
			global_status = CONST.ENB_PROCESS_REALTIME_ISSUE
Raphael Defosseux's avatar
Raphael Defosseux committed
1042
		HTML.htmleNBFailureMsg=html