run_exec_lte-softmodem_tests.py 78.7 KB
Newer Older
1
#! /usr/bin/python
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#******************************************************************************

#    OpenAirInterface 
#    Copyright(c) 1999 - 2014 Eurecom

#    OpenAirInterface is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.


#    OpenAirInterface is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#   You should have received a copy of the GNU General Public License
#   along with OpenAirInterface.The full GNU General Public License is 
#   included in this distribution in the file called "COPYING". If not, 
#   see <http://www.gnu.org/licenses/>.

#  Contact Information
#  OpenAirInterface Admin: openair_admin@eurecom.fr
#  OpenAirInterface Tech : openair_tech@eurecom.fr
#  OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr
  
#  Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

# *******************************************************************************/

Rohit Gupta's avatar
Rohit Gupta committed
32
# \author Rohit Gupta
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# \version 0.1
# @ingroup _test

import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock 
import xml.etree.ElementTree as ET
Rohit Gupta's avatar
Rohit Gupta committed
47
import re
48

49
50
import numpy as np

51
52
53
54
55
import log

from  openair import *

import paramiko
56

57
58
import subprocess
import commands
59
sys.path.append('/opt/ssh')
60
sys.path.append(os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/'))
61

62
from lib_autotest import *
63
64
import ssh
from ssh import SSHSession
Rohit Gupta's avatar
Rohit Gupta committed
65
import argparse
66

Rohit Gupta's avatar
Rohit Gupta committed
67
68
69
70
# \brief write a string to a file
# \param filename name of file
# \param string string to write
# \mode file opening mode (default=write)
71
72
73
74
def write_file(filename, string, mode="w"):
   text_file = open(filename, mode)
   text_file.write(string)
   text_file.close()
Rohit Gupta's avatar
Rohit Gupta committed
75
76
77
78
79
80
81
82
 
# \brief function to check if test case passed throughput test
# \param filename name of file which has throughput results (usually from iperf -s ...
# \param min_tput minimum throughput
# \param max_tuput maximum throughput
# \param average average throughput
# \param min_duration minimum duration of throughput
#The throughput values found in file must be higher than values from from arguments 2,3,4,5
83
84
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
85
   
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
   if os.path.exists(filename):
      with open (filename, "r") as myfile:
         data=myfile.read()
      p=re.compile('(\d*.\d*) Mbits/sec')
      array=p.findall(data)
      array = [ float(x) for x in array ]
      duration = array.__len__()
      if duration !=0:
        min_list = min(array) 
        max_list = max(array)
        average_list = np.mean(array)
      else:
        min_list = 0
        max_list = 0
        average_list=0
101
      tput_string=' ( '+ "min=%0.2f"  % min_list + ' Mbps / ' + "max=%0.2f" %  max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' 
102
      if (min_list >= min_tput and  max_list >= max_tput and average_list >= average and duration >= min_duration):
103
        return True , tput_string
104
      else:
105
        return False , tput_string
106
   else: 
107
      return False , tput_string
108

Rohit Gupta's avatar
Rohit Gupta committed
109
# \brief Convert string to float or return None if there is exception    
110
111
112
113
114
115
def try_convert_to_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

Rohit Gupta's avatar
Rohit Gupta committed
116
117
118
# \brief get throughput statistics from log file
# \param search_expr search expression found in test_case_list.xml file
# \param logfile_traffic logfile which has traffic statistics
119
120
def tput_test_search_expr (search_expr, logfile_traffic):
   result=0
121
   tput_string=''
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
   if search_expr !='':
       if search_expr.find('throughput_test')!= -1 :
          p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
          min_tput=p.findall(search_expr)
          if min_tput.__len__()==1:
             min_tput = min_tput[0]
          else:
             min_tput = None

          p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
          max_tput=p.findall(search_expr)
          if max_tput.__len__()==1:
             max_tput = max_tput[0]
          else:
             max_tput = None

          p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
          avg_tput=p.findall(search_expr)
          if avg_tput.__len__()==1:
             avg_tput=avg_tput[0]
          else:
             avg_tput = None

          p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
          duration=p.findall(search_expr)
          if duration.__len__()==1:
             duration = duration[0]
          else:
             duration = None
          
          min_tput = try_convert_to_float(min_tput)
          max_tput = try_convert_to_float(max_tput)
          avg_tput = try_convert_to_float(avg_tput)
          duration = try_convert_to_float(duration)
          
          if (min_tput != None and max_tput != None  and avg_tput != None  and duration != None ):
158
             result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
Rohit Gupta's avatar
Rohit Gupta committed
159
160
161
   else: 
      result=1

162
   return result, tput_string
163
      
Rohit Gupta's avatar
Rohit Gupta committed
164
165
166
167
168
169
170
# \brief function to copy files to/from remote machine
# \param username user with which to make sftp connection
# \param password password of user
# \param hostname host to connect
# \ports port of remote machine on which server is listening
# \paramList This is list of operations as a set {operation: "get/put", localfile: "filename", remotefile: "filename"
# \param logfile Ignored currently and set once at the beginning of program
Rohit Gupta's avatar
Rohit Gupta committed
171
172
173
def sftp_module (username, password, hostname, ports, paramList,logfile): 
   #localD = localfile
   #remoteD = remotefile
174
175
176
177
   #fd, paramiko_logfile  = tempfile.mkstemp()
   #res = os.close(fd )
   #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
   error = ""
Rohit Gupta's avatar
Rohit Gupta committed
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
   #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
   transport = paramiko.Transport(hostname, ports)
   transport.connect(username = username, password = password)
   sftp = paramiko.SFTPClient.from_transport(transport)
   #  index =0 
   for param in paramList:
      try:
        operation = param["operation"] 
        localD = param["localfile"]
        remoteD = param["remotefile"]
        if operation == "put":
          sftp.put(remotepath=remoteD, localpath=localD)
        elif operation == "get":
          sftp.get(remotepath=remoteD, localpath=localD)
        else :
          print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
          print "hostname = " + hostname
          print "ports = " + ports
          print "localfile = " + localD
          print "remotefile = " + remoteD
          print "operation = " + operation
          sys.exit()
      except Exception, e:
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'  
         error = error + traceback.format_exc()
204
         print error
Rohit Gupta's avatar
Rohit Gupta committed
205
206
207

   sftp.close()
   transport.close() 
208
209
210
211
212
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
   res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
   write_file(logfile, error, "a")
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')

Rohit Gupta's avatar
Rohit Gupta committed
213
214
215
# \brief bash script stub put at the end of scripts to terminate it 
# \param timeout_cmd terminate script after timeout_cmd seconds
# \param terminate_missing_procs if True terminate all the processes launched by script if one of them terminates prematurely (due to error)
216
def finalize_deploy_script (timeout_cmd, terminate_missing_procs='False'):
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
  cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
  if terminate_missing_procs == 'True':
    cmd = cmd +  """
    #The code below checks if one the processes launched in background has crashed.
    #If it does, then the code below terminates all the child processes created by this script
    declare -i wakeup_interval=1
    declare -i step=0
    echo \"Array pid =  ${array_exec_pid[@]}\"
    while [ "$step" -lt "$timeout_cmd" ]
      do
       declare -i break_while_loop=0
       #Iterate over each process ID in array_exec_pid
       for i in "${array_exec_pid[@]}"
       do
        numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
        echo "PID = $i, numchild = $numchild"
        if  [ "$numchild" -eq "0" ] ; then
            echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
            break_while_loop=1
            break
        fi
     done
    if  [ "$break_while_loop" -eq "1" ] ; then
             break
    fi
    step=$(( step + wakeup_interval ))
    sleep $wakeup_interval
    done
    echo "Final time step (Duration of test case) = $step "
Rohit Gupta's avatar
Rohit Gupta committed
246
    date
247
248
249
    """
  else:
    #We do not terminate the script if one of the processes has existed prematurely
Rohit Gupta's avatar
Rohit Gupta committed
250
    cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date  \n'
251
252
253
  
  return cmd

Rohit Gupta's avatar
Rohit Gupta committed
254
255
256
257
258
# \brief run python script and update config file params of test case
# \param oai module with already open connection
# \param config_string config string taken from xml file
# \param logdirRepo directory of remote repository
# \param python_script python script location
259
def update_config_file(oai, config_string, logdirRepo, python_script):
260
  cmd=""
261
262
263
264
265
266
267
268
  if config_string :
    stringArray = config_string.splitlines()
    #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
    for string in stringArray:
       #split the string based on space now
       string1=string.split()
       cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + '  ' + string1[1] +  ' '+ string1[2] + '\n'
       #cmd = cmd + 'perl -p -i  -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\'   ' + logdirRepo + '/' +string1[0] + '\n'
269
270
  return cmd
  #result = oai.send_recv(cmd)
271

Rohit Gupta's avatar
Rohit Gupta committed
272
273
274
275
276
277
278
279
280
# \brief thread safe sshsession wrapper due to occasional connection issues with ssh
# \param machine name of machine
# \param username user login for remote machine
# \param key_file file name which has keys to enable passwordless login
# \param password password for remote machine
# \param logdir_remote remote directory
# \param logdir_local_base local directory
# \param operation operation to perform (get_all, put_all) transfers recursively for directories
def SSHSessionWrapper(machine, username, key_file, password, logdir_remote, logdir_local_base, operation):
281
282
283
284
  max_tries = 100
  i=0
  while i <= max_tries:
    i = i +1
285
286
    try:
       ssh = SSHSession(machine , username, key_file, password)
287
       if operation == "get_all":
Rohit Gupta's avatar
Rohit Gupta committed
288
          ssh.get_all(logdir_remote , logdir_local_base)
289
       elif operation == "put_all":
Rohit Gupta's avatar
Rohit Gupta committed
290
          ssh.put_all(logdir_local_base, logdir_remote )
291
292
293
       else:
          print "Error: Uknown operation in SSHSessionWrapper. Exiting now..."
          sys.exit(1)
294
295
296
297
       break 
    except Exception, e:
       error=''
       error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
Rohit Gupta's avatar
Rohit Gupta committed
298
       error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote + '\n logdir_local_base = ' + logdir_local_base 
299
300
       error = error + traceback.format_exc()
       print error
301
       print " Retrying again in 1 seconds"
Rohit Gupta's avatar
Rohit Gupta committed
302
       time.sleep(1)
303
       print "Continuing ..."
304
305
306
       if i ==max_tries:
          print "Fatal Error: Max no of retries reached. Exiting now..."
          sys.exit(1)
307

308
309

 
Rohit Gupta's avatar
Rohit Gupta committed
310
311
312
313
314
# \briefFunction to clean old programs that might be running from earlier execution
# \param oai - parameter for making connection to machine
# \parm programList list of programs that must be terminated before execution of any test case 
# \param CleanUpAluLteBox program to terminate AlU Bell Labs LTE Box
# \param ExmimoRfStop String to stop EXMIMO card (specified in test_case_list.xml)
315
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo):
316
  cmd = 'killall -9 -q -r ' + programList
317
318
319
320
  result = oai.send(cmd, True)
  print "Killing old programs..." + result
  programArray = programList.split()
  programListJoin = '|'.join(programArray)
321
  cmd = " ( date ;echo \"Starting cleaning old programs.. \" ; dmesg|tail ; echo \"Current disk space.. \" ; df -h )>& " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
322
  result=oai.send_recv(cmd)
323
324
325
326
  cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
  #result = oai.send_recv(cmd)
  #print result
  result = oai.send_expect_false(cmd, 'Match found', False)
327
  print "Looking for old programs..." + result
328
  res=oai.send_recv(CleanUpAluLteBox, True)
329
330
331
332
  cmd= " echo \"Starting EXmimoRF Stop... \"  >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
  oai.send_recv(cmd)
  cmd  = "( " + "cd " + logdirOAI5GRepo + " ; source oaienv ;  "  +  ExmimoRfStop + " ) >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync "
  print "cleanoldprograms cmd = " + cmd
333
  res=oai.send_recv(cmd, False, timeout=600)
334
335
336
  cmd= " echo \"Stopping EXmimoRF Stop... \" >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
  oai.send_recv(cmd)

337
338
339
  #res = oai.send_recv(ExmimoRfStop, False)
  cmd = " ( date ;echo \"Finished cleaning old programs.. \" ; dmesg | tail)>> $HOME/.oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
  res=oai.send_recv(cmd)
340

Rohit Gupta's avatar
Rohit Gupta committed
341
342
343
344
345
346
347
348
349
# \brief Class thread to launch a generic command on remote machine
# \param threadID number of thread (for book keeping)
# \param threadname string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param username username with which to login
# \param password password with which to login
# \param cmd command as a string to run on remote machine
# \parma sudo if True sudo is set
# \param timeout timeout of command in seconds 
350
class oaiThread (threading.Thread):
351
    def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
352
353
        threading.Thread.__init__(self)
        self.threadID = threadID
354
355
356
357
        self.threadname = threadname
        self.machine = machine
        self.username = username
        self.password = password
358
359
360
361
        self.cmd = cmd
        self.sudo = sudo
        self.timeout = timeout
    def run(self):
362
363
        try:
          oai = openair('localdomain',self.machine)
364
          oai.connect(self.username, self.password)
365
          print "Starting " + self.threadname + " on machine " + self.machine
366
367
368
369
370
371
372
          result = oai.send_recv(self.cmd, self.sudo, self.timeout)
          print "result = " + result
          print "Exiting " + self.threadname
          oai.disconnect()
        except Exception, e:
           error=''
           error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
373
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + str(self.timeout) + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) +  '\n username = ' + self.username + '\n'  
374
375
376
           error = error + traceback.format_exc()
           print error

377

Rohit Gupta's avatar
Rohit Gupta committed
378
379
380
381
382
383
384
385
386
387
388
# \brief This class runs test cases with class {execution, compilatation}
# \param threadID number of thread (for book keeping)
# \param name string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param testcasename name of test case to run on remote machine
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param user username with which to login
# \param password password with which to login
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
389
class testCaseThread_generic (threading.Thread):
390
   def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop):
Rohit Gupta's avatar
Rohit Gupta committed
391
       threading.Thread.__init__(self)
392
       self.threadID = threadID
Rohit Gupta's avatar
Rohit Gupta committed
393
394
395
396
397
       self.name = name
       self.testcasename = testcasename
       self.timeout = timeout
       self.machine = machine
       self.logdirOAI5GRepo = logdirOAI5GRepo
398
399
400
       self.oldprogramList = oldprogramList
       self.CleanupAluLteBox = CleanupAluLteBox
       self.password=password
Rohit Gupta's avatar
Rohit Gupta committed
401
       self.ExmimoRfStop = ExmimoRfStop
402
       self.user = user
Rohit Gupta's avatar
Rohit Gupta committed
403
404
405
406
407
   def run(self):
     try:
       mypassword=''
       #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
       addpass = 'echo \'' + mypassword + '\' | '
408
       #user = getpass.getuser()
Rohit Gupta's avatar
Rohit Gupta committed
409
       print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) 
410
       oai = openair('localdomain',self.machine)
411
412
       oai.connect(self.user, self.password)
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
413
414
415
       logdir_local = os.environ.get('OPENAIR_DIR')
       logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
       logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
416
417
418
419
420
421
422
423
424
425
426
427
428
       logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
       logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
       logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
       logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
       #print "logdir_local_testcase = " + logdir_local_testcase
       #print "logdir_remote_testcase = " + logdir_remote_testcase
       #if os.path.exists(logdir_local_testcase) == True :
       #    os.removedirs(logdir_local_testcase)
       #os.mkdir(logdir_local_testcase)
       os.system("rm -fr " + logdir_local_testcase )
       os.system("mkdir -p " +  logdir_local_testcase)
       cmd = "mkdir -p " + logdir_remote_testcase
       res = oai.send_recv(cmd, False, self.timeout) 
Rohit Gupta's avatar
Rohit Gupta committed
429
       #print "res = " + res
430
431

       cmd =  "( cd " +  self.logdirOAI5GRepo + " \n "
Rohit Gupta's avatar
Rohit Gupta committed
432
       cmd = cmd + "source oaienv \n"
433
434
435
436
437
438
       cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
       cmd = cmd + " ) >& "   + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase +  "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase 
      
       #print "cmd = " + cmd
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res
Rohit Gupta's avatar
Rohit Gupta committed
439
       #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
440
441
       write_file(logfile_task_testcasename, cmd, mode="w")
       #Now we copy all the remote files
442
443
       #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
       #ssh.get_all(logdir_remote_testcase , logdir_local_base)
444
       SSHSessionWrapper(self.machine, self.user, None, self.password, logdir_remote_testcase, logdir_local_base, "get_all")
445
       print "Finishing test case : " + self.testcasename + " On machine " + self.machine
446
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
447
       #oai.kill(user,mypassword)
448
449
450
       oai.disconnect()
     except Exception, e:
         error=''
451
         error = error + ' In Class = testCaseThread_generic,  function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
452
         error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo +  '\n' + '\n timeout = ' + str(self.timeout)  + '\n user = ' + self.user
453
         error = error + traceback.format_exc()
454
         print error
455
456
         print "Continuing with next test case..."
         #sys.exit()
Rohit Gupta's avatar
Rohit Gupta committed
457
458


459
       
Rohit Gupta's avatar
Rohit Gupta committed
460
461
462
# \bried function to run a command as a sudo
# \param cmd command as a string
# \param password password to be supplied   
463
464
465
def addsudo (cmd, password=""):
  cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
  return cmd
466

Rohit Gupta's avatar
Rohit Gupta committed
467
468
469
470
471
472
473
474
475
476
477
# \brief handler for executing test cases (compilation, execution)
# \param name of testcase
# \param threadListGeneric list of threads which are already running on remote machines
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param MachineList list of all machines on which generic test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
478
def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, user, password, CleanupAluLteBox,timeout, ExmimoRfStop):
479
480
481
  try:
    mypassword=password
    MachineListFree=[]
Rohit Gupta's avatar
Rohit Gupta committed
482
    threadListNew=[]
483
484
    while MachineListFree.__len__() == 0 :
       MachineListBusy=[]
Rohit Gupta's avatar
Rohit Gupta committed
485
486
       MachineListFree=[]
       threadListNew=[]
487
488
489
       #first we need to find the list of free machines that we could run our test case
       if threadListGeneric.__len__() ==0 :
       #This means no thread is started yet
490
          MachineListFree = MachineList[:]
491
492
493
494
       else :
          for param in threadListGeneric :
             thread_id = param["thread_id"]
             machine = param["Machine"]
495
             testcasenameold = param["testcasename"]
496
             thread_id.join(1)
497
             if thread_id.isAlive() == True:
Rohit Gupta's avatar
Rohit Gupta committed
498
499
                threadListNew.append(param)
                print "thread_id is alive: testcasename: " + testcasenameold +  " on machine "+ machine
500
501
                if machine not in MachineListBusy:
                   MachineListBusy.append(machine)
502
             else :
Rohit Gupta's avatar
Rohit Gupta committed
503
504
505
506
                print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
                #threadListGeneric.remove(param)
                #if machine not in MachineListFree:
                #   MachineListFree.append(machine)
507
508
509
510
511
       #Now we check if there is at least one free machine
       MachineListFree = MachineList[:]
       for machine in MachineListBusy:
          if machine in MachineListFree:
            MachineListFree.remove(machine)
Rohit Gupta's avatar
Rohit Gupta committed
512
513
514
       print "MachineListFree = " + ','.join(MachineListFree)
       print "MachineListBusy = " + ','.join(MachineListBusy)
       print "MachineList = " + ','.join(MachineList)
515
    machine = MachineListFree[0]
516
    thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop)
517
518
    param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
    thread.start()
Rohit Gupta's avatar
Rohit Gupta committed
519
520
    threadListNew.append(param)
    return threadListNew
521
  except Exception, e:
522
523
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
524
     error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) +  '\n' + 'user = ' + user
525
526
     error = error + traceback.format_exc()
     print error
527
528
     print "Continuing..."
     #sys.exit(1)
529

Rohit Gupta's avatar
Rohit Gupta committed
530
531
532
# \brief Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
# \param threadListGeneric list of threads which are running on remote machines
# \param timeout time to wait on threads in seconds
533
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
534
   threadListGenericNew=[]
535
536
537
538
539
540
   for param in threadListGeneric:
      thread_id = param["thread_id"]
      machine = param["Machine"]
      testcasenameold = param["testcasename"]
      thread_id.join(timeout)
      if thread_id.isAlive() == True:
541
         threadListGenericNew.append(param)
542
543
544
545
546
         print "thread_id on machine: " + machine + "  is still alive: testcasename: " + testcasenameold
         print " Exiting now..."
         sys.exit(1)
      else:
         print "thread_id on machine: " + machine + "  is stopped: testcasename: " + testcasenameold
547
548
         #threadListGeneric.remove(param)
   return threadListGenericNew
549

Rohit Gupta's avatar
Rohit Gupta committed
550
551
552
553
554
555
556
557
558
559
560

# \brief handler for executing test cases (lte-softmodem)
# \param testcase name of testcase
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed
# \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed
# \param MachineList list of all machines on which test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
561
# \param nruns_lte-softmodem global parameter to override number of runs (nruns) within the test case
562
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, timeout_cmd):
563
564
565
566
567
568
  #We ignore the password sent to this function for secuirity reasons for password present in log files
  #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
  # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
  mypassword=''
  #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
  addpass = 'echo \'' + mypassword + '\' | '
569
  #user = getpass.getuser()
570
  testcasename = testcase.get('id')
571
  testcaseclass = testcase.findtext('class',default='')
572
573
  if timeout_cmd == '':
     timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
574
575
576
  timeout_cmd = int(float(timeout_cmd))
  #Timeout_thread is more than that of cmd to have room for compilation time, etc
  timeout_thread = timeout_cmd + 300 
577
578
579
580
  if nruns_lte_softmodem == '':
    nruns = testcase.findtext('nruns',default='')
  else:
    nruns = nruns_lte_softmodem
581
  nruns = int(float(nruns))
582
  tags = testcase.findtext('tags',default='')
Rohit Gupta's avatar
Rohit Gupta committed
583
584
585
586
587
588
589
590
591
592
593

  RRHMachine = testcase.findtext('RRH',default='')
  RRH_compile_prog = testcase.findtext('RRH_compile_prog',default='')
  RRH_compile_prog_args = testcase.findtext('RRH_compile_prog_args',default='')
  RRH_pre_exec = testcase.findtext('RRH_pre_exec',default='')
  RRH_pre_exec_args = testcase.findtext('RRH_pre_exec_args',default='')
  RRH_main_exec = testcase.findtext('RRH_main_exec',default='')
  RRH_main_exec_args = testcase.findtext('RRH_main_exec_args',default='')
  RRH_terminate_missing_procs = testcase.findtext('RRH_terminate_missing_procs',default='True')


594
595
596
597
598
599
600
601
602
603
604
  eNBMachine = testcase.findtext('eNB',default='')
  eNB_config_file = testcase.findtext('eNB_config_file',default='')
  eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
  eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
  eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
  eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
  eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
  eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
  eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
  eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
  eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='True')
605
  eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')
606
  if re.compile('\w+').match(eNB_search_expr_true) != None:
607
      eNB_search_expr_true = eNB_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's' 
608
609
610
611
612
613
614
615
616
617
618
619

  UEMachine = testcase.findtext('UE',default='')
  UE_config_file = testcase.findtext('UE_config_file',default='')
  UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
  UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
  UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
  UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
  UE_main_exec = testcase.findtext('UE_main_exec',default='')
  UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
  UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
  UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
  UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='True')
620
  UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
Rohit Gupta's avatar
Rohit Gupta committed
621
  UE_stop_script =  testcase.findtext('UE_stop_script','')
622
  if re.compile('\w+').match(UE_search_expr_true) != None:
623
      UE_search_expr_true = UE_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640

  EPCMachine = testcase.findtext('EPC',default='')
  EPC_config_file = testcase.findtext('EPC_config_file',default='')
  EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
  EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
  HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
  HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
  
  EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
  EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')  
  EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
  EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')  
  HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
  HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')  
  EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
  EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
  EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='True')
641
  EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')
642
  if re.compile('\w+').match(EPC_search_expr_true) != None:
643
     EPC_search_expr_true = EPC_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
644
645
646
647

  index_eNBMachine = MachineList.index(eNBMachine)
  index_UEMachine = MachineList.index(UEMachine)
  index_EPCMachine = MachineList.index(EPCMachine)
Rohit Gupta's avatar
Rohit Gupta committed
648
  cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
649
650
  oai_eNB = openair('localdomain', eNBMachine)
  oai_eNB.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
651
  res= oai_eNB.send_recv(cmd)
652
653
  oai_UE = openair('localdomain', UEMachine)
  oai_UE.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
654
  res = oai_eNB.send_recv(cmd)
655
656
  oai_EPC = openair('localdomain', EPCMachine)
  oai_EPC.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
657
  res = oai_eNB.send_recv(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
658
659
660
661
662
663
  if RRHMachine != '':
    cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
    index_RRHMachine = MachineList.index(RRHMachine)
    oai_RRH = openair('localdomain', RRHMachine)
    oai_RRH.connect(user, password)
    res= oai_RRH.send_recv(cmd)
664
665
666
  #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
  logdir_local = os.environ.get('OPENAIR_DIR')
  if logdir_local is None:
     print "Environment variable OPENAIR_DIR not set correctly"
     sys.exit()
   
  #Make the log directory of test case
  #cmd = 'mkdir -p ' + logdir_eNB
  #result = oai_eNB.send_recv(cmd)
  #cmd = 'mkdir -p ' +  logdir_UE
  #result = oai_UE.send_recv(cmd)
  #cmd = 'mkdir -p ' + logdir_EPC
  #result = oai_EPC.send_recv(cmd)
  
  print "Updating the config files for ENB/UE/EPC..."
  #updating the eNB/UE/EPC configuration file from the test case 
  #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
  #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
  #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
685
686
687
  test_result=1
  test_result_string=''
  start_time=time.time()
688
  for run in range(0,nruns):
689
690
    run_result=1
    run_result_string=''
691
    logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
Rohit Gupta's avatar
Rohit Gupta committed
692
    logdir_RRH = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
693
694
695
696
    logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    #Make the log directory of test case
Rohit Gupta's avatar
Rohit Gupta committed
697
698
699
    if RRHMachine != '':
      cmd = 'rm -fr ' + logdir_RRH + ' ; mkdir -p ' + logdir_RRH
      result = oai_RRH.send_recv(cmd)
700
701
702
703
704
705
706
707
    cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
    result = oai_eNB.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' +  logdir_UE
    result = oai_UE.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
    result = oai_EPC.send_recv(cmd)
    cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
    result = os.system(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740

    if RRHMachine != '':
       logfile_compile_RRH = logdir_RRH + '/RRH_compile' + '_' + str(run) + '_.log'
       logfile_exec_RRH = logdir_RRH + '/RRH_exec' + '_' + str(run) + '_.log'
       logfile_pre_exec_RRH = logdir_RRH + '/RRH_pre_exec' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile_out = logdir_RRH + '/RRH_task_compile_out' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile = logdir_local_testcase + '/RRH_task_compile' + '_' + str(run) + '_.log'
       logfile_task_RRH_out = logdir_RRH + '/RRH_task_out' + '_' + str(run) + '_.log'
       logfile_task_RRH = logdir_local_testcase + '/RRH_task' + '_' + str(run) + '_.log'
       task_RRH_compile = ' ( uname -a ; date \n'
       task_RRH_compile = task_RRH_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
       task_RRH_compile = task_RRH_compile + 'env |grep OPENAIR  \n'
       if RRH_compile_prog != "":
         task_RRH_compile  = task_RRH_compile +  ' ( ' + RRH_compile_prog + ' '+ RRH_compile_prog_args + ' ) > ' + logfile_compile_RRH + ' 2>&1 \n'
       task_RRH_compile =  task_RRH_compile + ' date ) > ' + logfile_task_RRH_compile_out + ' 2>&1  '
       write_file(logfile_task_RRH_compile, task_RRH_compile, mode="w")

       task_RRH = ' ( uname -a ; date \n'
       task_RRH = task_RRH + ' export OPENAIR_TESTDIR=' + logdir_RRH + '\n'
       task_RRH = task_RRH + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
       task_RRH = task_RRH + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

       if RRH_pre_exec != "":
          task_RRH  = task_RRH +  ' ( date; ' + RRH_pre_exec + ' '+ RRH_pre_exec_args + ' ) > ' + logfile_pre_exec_RRH + ' 2>&1 \n'
       if RRH_main_exec != "":
          task_RRH = task_RRH + ' ( date; ' + addsudo(RRH_main_exec + ' ' + RRH_main_exec_args, mypassword) + ' ) > ' + logfile_exec_RRH + ' 2>&1 & \n'
          task_RRH = task_RRH + 'array_exec_pid+=($!) \n'
          task_RRH = task_RRH + 'echo eNB_main_exec PID = $! \n'
       #terminate the eNB test case after timeout_cmd seconds
       task_RRH  = task_RRH + finalize_deploy_script (timeout_cmd, RRH_terminate_missing_procs) + ' \n'
       task_RRH  = task_RRH + 'handle_ctrl_c' + '\n' 
       task_RRH  = task_RRH + ' ) > ' + logfile_task_RRH_out + ' 2>&1  '
       write_file(logfile_task_RRH, task_RRH, mode="w")
741
742
743
744
745
    
    logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
    logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
746
747
    logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
748
749
    logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
    logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
750
    logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
751
752
    logfile_tshark_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.log'
    logfile_pcap_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap'
753
    logfile_pcap_zip_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap.zip'
Rohit Gupta's avatar
Rohit Gupta committed
754
    logfile_pcap_tmp_eNB = '/tmp/' + '/eNB_tshark' + '_' + str(run) + '_.pcap'
Rohit Gupta's avatar
Rohit Gupta committed
755
756
757
758
759

    task_eNB_compile = ' ( uname -a ; date \n'
    task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR  \n'
    task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
760
    if eNB_compile_prog != "":
Rohit Gupta's avatar
Rohit Gupta committed
761
762
763
764
765
       task_eNB_compile  = task_eNB_compile +  ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
    task_eNB_compile =  task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1  '
    write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")

    task_eNB = ' ( uname -a ; date \n'
766
    task_eNB = task_eNB + ' export OPENAIR_TESTDIR=' + logdir_eNB + '\n'
Rohit Gupta's avatar
Rohit Gupta committed
767
768
769
    task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB = task_eNB + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

770
    if eNB_pre_exec != "":
771
       task_eNB  = task_eNB +  ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
772
    if eNB_main_exec != "":
773
       task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
774
775
776
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
    if eNB_traffic_exec != "":
777
778
       cmd_traffic = eNB_traffic_exec + ' ' + eNB_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
779
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
780
       task_eNB = task_eNB + ' (date;  ' + cmd_traffic + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n'
781
782
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
Rohit Gupta's avatar
Rohit Gupta committed
783

784
    task_eNB = task_eNB + ' (date; sudo rm -f ' + logfile_pcap_tmp_eNB + ' ; sudo -E tshark -i lo -s 65535 -a duration:' + str(timeout_cmd-10)+ ' -w ' + logfile_pcap_tmp_eNB+ ' ; sudo -E chown ' + user + ' ' + logfile_pcap_tmp_eNB + ' ; zip -j -9  ' + logfile_pcap_zip_eNB + ' ' + logfile_pcap_tmp_eNB + '   ) > ' + logfile_tshark_eNB + ' 2>&1 & \n '
Rohit Gupta's avatar
Rohit Gupta committed
785
786
    task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
    task_eNB = task_eNB + 'echo eNB_tshark_exec PID = $! \n'
787
788
789
790
791
792
793
794
795
796
797
798
799
800
    #terminate the eNB test case after timeout_cmd seconds
    task_eNB  = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
    #task_eNB  = task_eNB + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_eNB  = task_eNB + 'handle_ctrl_c' + '\n' 
    task_eNB  = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1  '
    write_file(logfile_task_eNB, task_eNB, mode="w")

    #task_eNB =  'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB 
    logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
    logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'    
    logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
    logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
801
802
    logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
803
    logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
804
805
806
807
808
809
810
811
812
813
814
815
816
817

    task_UE_compile = ' ( uname -a ; date \n'
    task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
    task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE_compile = task_UE_compile + 'source oaienv \n'
    task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
    task_UE_compile = task_UE_compile + 'env |grep OPENAIR  \n'
    task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if UE_compile_prog != "":
       task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
    task_UE_compile  = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
    write_file(logfile_task_UE_compile, task_UE_compile, mode="w")

    task_UE = ' ( uname -a ; date \n'
818
819
820
821
822
823
    task_UE = task_UE + 'array_exec_pid=()' + '\n'
    task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE = task_UE + 'source oaienv \n'
    task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
    task_UE = task_UE + 'env |grep OPENAIR  \n'
    if UE_pre_exec != "":
824
       task_UE  = task_UE +  ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
825
    if UE_main_exec != "":
826
       task_UE = task_UE + ' ( date;  ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword)  + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
827
828
829
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
    if UE_traffic_exec != "":
830
831
       cmd_traffic = UE_traffic_exec + ' ' + UE_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
832
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
833
       task_UE = task_UE + ' ( date;  ' + cmd_traffic + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
    #terminate the UE test case after timeout_cmd seconds
    task_UE  = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
    #task_UE  = task_UE + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_UE  = task_UE + 'handle_ctrl_c' + '\n' 
    task_UE  = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
    write_file(logfile_task_UE, task_UE, mode="w")
    #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE

    logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
    logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
    logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
    logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
    logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
    logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
    logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
852
853
    logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
854
    logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
855
856
857

    task_EPC_compile = ' ( uname -a ; date \n'
    task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
858
    task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + ' ; source oaienv \n'
Rohit Gupta's avatar
Rohit Gupta committed
859
860
861
862
863
864
865
866
867
868
    task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
    task_EPC_compile = task_EPC_compile +  'source BUILD/TOOLS/build_helper \n'
    if EPC_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args +  ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
    if HSS_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
    task_EPC_compile  = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' 
    write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
    
    task_EPC = ' ( uname -a ; date \n'
869
    task_EPC = task_EPC + ' export OPENAIRCN_TESTDIR=' + logdir_EPC + '\n'
870
    task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
871
    task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n'
872
873
    task_EPC = task_EPC +  'source BUILD/TOOLS/build_helper \n'
    if EPC_pre_exec != "":
874
       task_EPC  = task_EPC +  ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
875
    if HSS_main_exec !=  "":
876
       task_EPC  = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS  +  ' 2>&1   & \n'
877
878
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
879
    if EPC_main_exec !=  "":
880
       task_EPC  = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC  +  ' 2>&1   & \n'
881
882
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
883
    if EPC_traffic_exec !=  "":
884
885
       cmd_traffic = EPC_traffic_exec + ' ' + EPC_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
886
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
887
       task_EPC  = task_EPC + '( date; ' + cmd_traffic + ' ) > ' + logfile_traffic_EPC  +  ' 2>&1   & \n' 
888
889
890
891
892
893
894
895
896
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'  
       task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
    #terminate the EPC test case after timeout_cmd seconds   
    task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
    #task_EPC  = task_EPC + 'sleep ' +  str(timeout_cmd) + '\n'
    task_EPC  = task_EPC + 'handle_ctrl_c' '\n' 
    task_EPC  = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' 
    write_file(logfile_task_EPC, task_EPC, mode="w")
    
Rohit Gupta's avatar
Rohit Gupta committed
897
898
899
900
    #first we compile all the programs
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE_compile, False, timeout_thread) 
Rohit Gupta's avatar
Rohit Gupta committed
901
902
    if RRHMachine != '':
        thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH_compile, False, timeout_thread) 
Rohit Gupta's avatar
Rohit Gupta committed
903
904
905
906
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
Rohit Gupta's avatar
Rohit Gupta committed
907
908
    if RRHMachine != '':
        threads.append(thread_RRH)
Rohit Gupta's avatar
Rohit Gupta committed
909
910
911
912
    # Start new Threads
    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
Rohit Gupta's avatar
Rohit Gupta committed
913
914
    if RRHMachine != '':
        thread_RRH.start()
Rohit Gupta's avatar
Rohit Gupta committed
915
916
917
918
919
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()

    #Now we execute all the threads
920
921
922
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE, False, timeout_thread) 
Rohit Gupta's avatar
Rohit Gupta committed
923
924
    if RRHMachine != '':
        thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH, False, timeout_thread) 
925
926
927
928
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
Rohit Gupta's avatar
Rohit Gupta committed
929
930
    if RRHMachine != '':
        threads.append(thread_RRH)
931
932
933
934
935
    # Start new Threads

    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
Rohit Gupta's avatar
Rohit Gupta committed
936
937
    if RRHMachine != '':
        thread_RRH.start()
938
939
940
941
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()
    #Now we get the log files from remote machines on the local machine
Rohit Gupta's avatar
Rohit Gupta committed
942
    if RRHMachine != '':
943
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC, oai_RRH] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC, logdir_RRH], logdirOAI5GRepo)
Rohit Gupta's avatar
Rohit Gupta committed
944
    else:
945
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC], logdirOAI5GRepo)       
Rohit Gupta's avatar
Rohit Gupta committed
946
947
948
949
950
951
952
953
954
955
956
957
958
959
    logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
    logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'

    if UE_stop_script != "":
      cmd = ' ( uname -a ; date \n'
      cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
      cmd = cmd + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'
      cmd = cmd + UE_stop_script + '\n'
      cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' 
      write_file(logfile_UE_stop_script , cmd, mode="w")
      thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password  , cmd, False, timeout_thread)
      thread_UE.start()
      thread_UE.join()
   
Rohit Gupta's avatar
Rohit Gupta committed
960
961
962
963
964
965
966
967
968
969
970
971
972
    #Now we change the permissions of the logfiles to avoid some of them being with root permissions
    cmd = 'sudo -E chown -R ' + user + ' ' + logdir_eNB
    res= oai_eNB.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_eNB + "> in eNB machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_UE
    res= oai_UE.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_UE + "> in UE machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_EPC
    res= oai_EPC.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_EPC + "> in EPC machine..." + res

Rohit Gupta's avatar
Rohit Gupta committed
973
974
975
976
    if RRHMachine != '':
       cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_RRH
       res= oai_RRH.send_recv(cmd)
       print "Changing permissions of logdir <" + logdir_RRH + "> in RRH machine..." + res
977

978
    print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
979
    SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
980

981
    print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
982
    SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
983

984
    print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
985
    SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
986

Rohit Gupta's avatar
Rohit Gupta committed
987
988
989
    if RRHMachine != '':
       print "Copying files from RRHMachine : " + RRHMachine + "logdir_RRH = " + logdir_RRH
       SSHSessionWrapper(RRHMachine, user, None, password, logdir_RRH, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
990

Rohit Gupta's avatar
Rohit Gupta committed
991
    
992
    #Currently we only perform throughput tests
993
994
995
    tput_run_string=''
    result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
    tput_run_string = tput_run_string + tput_string
996
    run_result=run_result&result
997
    result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
998
    run_result=run_result&result
999
1000
    tput_run_string = tput_run_string + tput_string
    result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
1001
    run_result=run_result&result
1002
    tput_run_string = tput_run_string + tput_string
Rohit Gupta's avatar
Rohit Gupta committed
1003
    
1004
    if run_result == 1:  
Rohit Gupta's avatar
Rohit Gupta committed
1005
      run_result_string = ' RUN_'+str(run) + ' = PASS'
1006
    else:
Rohit Gupta's avatar
Rohit Gupta committed
1007
      run_result_string = ' RUN_'+str(run) + ' = FAIL'
1008
1009
1010
1011
1012
1013
1014
1015

    #If there is assertion, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"assertion\" " + logdir_local_testcase + " | cat " 
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if not cmd_out:
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Assert)'

1016
1017
1018
1019
1020
1021
1022
    #If there is thread busy error, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"thread busy\" " + logdir_local_testcase + " | cat "
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if not cmd_out:
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Thread_Busy)'

1023
    run_result_string = run_result_string + tput_run_string
1024
1025
1026
1027

    test_result=test_result & run_result
    test_result_string=test_result_string + run_result_string

1028
1029
1030
    oai_eNB.disconnect()
    oai_UE.disconnect()
    oai_EPC.disconnect()
1031
    #We need to close the new ssh session that was created  
1032
1033
    #if index_eNBMachine == index_EPCMachine:
    #    oai_EPC.disconnect()
1034
1035
1036
  #Now we finalize the xml file of the test case
  end_time=time.time()
  duration= end_time - start_time
Rohit Gupta's avatar
Rohit Gupta committed
1037
  xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
1038
1039
1040
1041
  if test_result ==0: 
    result='FAIL'
  else:
    result = 'PASS'
1042
  xml="\n<testcase classname=\'"+ testcaseclass +  "\' name=\'" + testcasename + "."+tags +  "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
1043
1044
  write_file(xmlFile, xml, mode="w")

1045

Rohit Gupta's avatar
Rohit Gupta committed
1046
1047
1048
1049
# \brief This function searches if test case is present in list of test cases that need to be executed by user
# \param testcasename the test case to search for
# \param testcasegroup list that is passed from the arguments
# \param test_case_exclude list of test cases excluded from execution (specified in test_case_list.xml)
1050
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
1051
    
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
    if test_case_exclude != "":
       testcase_exclusion_list=test_case_exclude.split()
       for entry in testcase_exclusion_list:
          if entry.find('+') >=0:
            match = re.search(entry, testcasename)
            if match:
               print "\nSkipping test case as it is found in black list: " + testcasename
               return False
          else:
             match = entry.find(testcasename)
             if match >=0:
                print "\nSkipping test case as it is found in black list: " + testcasename
                return False
Rohit Gupta's avatar
Rohit Gupta committed
1065
    if testcasegroup == '':
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
         return True
    else:
      testcaselist = testcasegroup.split()
      for entry in testcaselist:
        if entry.find('+') >=0:
           match = re.search(entry, testcasename)
           if match:
              return True
        else:
           match = testcasename.find(entry)
           if match >=0:
Rohit Gupta's avatar
Rohit Gupta committed
1077
1078
             return True
    return False
1079

Rohit Gupta's avatar
Rohit Gupta committed
1080
1081
1082
1083
1084
1085
1086
# \brief thread that cleans up remote machines from pre-existing test case executions
# \param threadID number of thread (for book keeping)
# \param threadname name of thread (for book keeping)
# \param oai handler that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
1087
class oaiCleanOldProgramThread (threading.Thread):
1088
    def __init__(self, threadID, threadname, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo):
1089
1090
1091
1092
1093
1094
1095
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.threadname = threadname
        self.oai = oai
        self.CleanUpOldProgs = CleanUpOldProgs
        self.CleanUpAluLteBox = CleanUpAluLteBox
        self.ExmimoRfStop = ExmimoRfStop
1096
1097
        self.logdir = logdir
        self.logdirOAI5GRepo = logdirOAI5GRepo
1098
1099
    def run(self):
        try:
1100
          cleanOldPrograms(self.oai, self.CleanUpOldProgs, self.CleanUpAluLteBox, self.ExmimoRfStop, self.logdir, self.logdirOAI5GRepo)
1101
1102
1103
1104
1105
1106
        except Exception, e:
           error=''
           error = error + ' In class oaiCleanOldProgramThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n CleanUpOldProgs = ' + self.CleanUpOldProgs + '\n CleanUpAluLteBox = ' + self.CleanUpAluLteBox + '\n ExmimoRfStop = ' + self.ExmimoRfStop + '\n'  
           error = error + traceback.format_exc()
           print error