run_exec_lte-softmodem_tests.py

    run_result=run_result&result
    tput_run_string = tput_run_string + tput_string
    
    if run_result == 1:  
      run_result_string = ' RUN_'+str(run) + ' = PASS'
    else:
      run_result_string = ' RUN_'+str(run) + ' = FAIL'

    #If there is assertion, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"assertion\" " + logdir_local_testcase + " | cat " 
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if len(cmd_out) !=0 :
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Assert)'

    #If there is thread busy error, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"thread busy\" " + logdir_local_testcase + " | cat "
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if len(cmd_out) !=0:
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Thread_Busy)'

    #If there is Segmentation fault, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"segmentation fault\" " + logdir_local_testcase + " | cat "
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if len(cmd_out) !=0:
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(SEGFAULT)'

    run_result_string = run_result_string + tput_run_string

    test_result=test_result & run_result
    test_result_string=test_result_string + run_result_string

    oai_eNB.disconnect()
    oai_UE.disconnect()
    oai_EPC.disconnect()
    #We need to close the new ssh session that was created  
    #if index_eNBMachine == index_EPCMachine:
    #    oai_EPC.disconnect()
  #Now we finalize the xml file of the test case
  end_time=time.time()
  duration= end_time - start_time
  xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
  if test_result ==0: 
    result='FAIL'
  else:
    result = 'PASS'
  xml="\n<testcase classname=\'"+ testcaseclass +  "\' name=\'" + testcasename + "."+tags +  "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
  write_file(xmlFile, xml, mode="w")


# \brief This function searches if test case is present in list of test cases that need to be executed by user
# \param testcasename the test case to search for
# \param testcasegroup list that is passed from the arguments
# \param test_case_exclude list of test cases excluded from execution (specified in test_case_list.xml)
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
    
    if test_case_exclude != "":
       testcase_exclusion_list=test_case_exclude.split()
       for entry in testcase_exclusion_list:
          if entry.find('+') >=0:
            match = re.search(entry, testcasename)
            if match:
               print "\nSkipping test case as it is found in black list: " + testcasename
               return False
          else:
             match = entry.find(testcasename)
             if match >=0:
                print "\nSkipping test case as it is found in black list: " + testcasename
                return False
    if testcasegroup == '':
         return True
    else:
      testcaselist = testcasegroup.split()
      for entry in testcaselist:
        if entry.find('+') >=0:
           match = re.search(entry, testcasename)
           if match:
              return True
        else:
           match = testcasename.find(entry)
           if match >=0:
             return True
    return False

# \brief thread that cleans up remote machines from pre-existing test case executions
# \param threadID number of thread (for book keeping)
# \param threadname name of thread (for book keeping)
# \param oai handler that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
class oaiCleanOldProgramThread (threading.Thread):
    def __init__(self, threadID, threadname, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.threadname = threadname
        self.oai = oai
        self.CleanUpOldProgs = CleanUpOldProgs
        self.CleanUpAluLteBox = CleanUpAluLteBox
        self.ExmimoRfStop = ExmimoRfStop
        self.logdir = logdir
        self.logdirOAI5GRepo = logdirOAI5GRepo
    def run(self):
        try:
          cleanOldPrograms(self.oai, self.CleanUpOldProgs, self.CleanUpAluLteBox, self.ExmimoRfStop, self.logdir, self.logdirOAI5GRepo)
        except Exception, e:
           error=''
           error = error + ' In class oaiCleanOldProgramThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n CleanUpOldProgs = ' + self.CleanUpOldProgs + '\n CleanUpAluLteBox = ' + self.CleanUpAluLteBox + '\n ExmimoRfStop = ' + self.ExmimoRfStop + '\n'  
           error = error + traceback.format_exc()
           print error
           print "There is error in cleaning up old programs....."
           #sys.exit(1)

# \brief Run parallel threads in all machines for clean up old execution of test cases
# \param oai_list list of handlers that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
def cleanOldProgramsAllMachines(oai_list, CleanOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir_list, logdirOAI5GRepo):
   threadId=0
   threadList=[]
   index=0
   if len(oai_list)!=len(logdir_list) :
       logdir_list=[logdir[0]]*len(oai_list)

   for oai in oai_list:
      threadName="cleanup_thread_"+str(threadId)
      thread=oaiCleanOldProgramThread(threadId, threadName, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir_list[index],logdirOAI5GRepo)
      threadList.append(thread)
      thread.start()
      threadId = threadId + 1
      index = index+1
   for t in threadList:
      t.join()


debug = 0
pw =''
i = 0
dlsim=0
localshell=0
GitOAI5GRepo=''
GitOAI5GRepoBranch=''
GitOAI5GHeadVersion=''
user=''
pw=''
testcasegroup=''
NFSResultsShare=''
cleanUpRemoteMachines=False
openairdir_local = os.environ.get('OPENAIR_DIR')
if openairdir_local is None:
   print "Environment variable OPENAIR_DIR not set correctly"
   sys.exit()
locallogdir = openairdir_local + '/cmake_targets/autotests/log'
MachineList = ''
MachineListGeneric=''
flag_remove_logdir=False
flag_start_testcase=False
nruns_lte_softmodem=''
flag_skip_git_head_check=False
flag_skip_oai_install=False
Timeout_cmd=''
print "Number of arguments argc = " + str(len(sys.argv))
#for index in range(1,len(sys.argv) ):
#  print "argv_" + str(index) + " : " + sys.argv[index]

i=1
while i < len (sys.argv):
    arg=sys.argv[i]
    if arg == '-r':
        flag_remove_logdir=True
    elif arg == '-s' :
        flag_start_testcase=True
    elif arg == '-g' :
        testcasegroup = sys.argv[i+1].replace("\"","")
        i = i +1   
    elif arg == '-c':
        cleanUpRemoteMachines=True
    elif arg == '-5GRepo':
        GitOAI5GRepo = sys.argv[i+1]
        i = i +1
    elif arg == '-5GRepoBranch':
        GitOAI5GRepoBranch = sys.argv[i+1]
        i = i +1
    elif arg == '-5GRepoHeadVersion':
        GitOAI5GHeadVersion = sys.argv[i+1]
        #We now find the branch that corresponds to this Git Head Commit
        cmd = "git show-ref --head " + " | grep " + GitOAI5GHeadVersion
        cmd_out = subprocess.check_output ([cmd], shell=True)
        cmd_out=cmd_out.replace("\n","")
        cmd_out = cmd_out.split('/')
        GitOAI5GRepoBranch = cmd_out[-1]
        if GitOAI5GRepoBranch == '':
           print "Error extracting GitBranch from head commit. Exiting now..."
           sys.exit(1)
        i = i +1
    elif arg == '-u':
        user = sys.argv[i+1]
        i = i +1
    elif arg == '-p': 
        pw = sys.argv[i+1]
        i = i +1
    elif arg == '-n': 
        NFSResultsShare = sys.argv[i+1]
        i = i +1
    elif arg == '--nrun_lte_softmodem': 
        nruns_lte_softmodem = sys.argv[i+1]
        i = i +1
    elif arg == '-MachineList':
        MachineList =  sys.argv[i+1]
        MachineList = MachineList.replace("\"","")
        MachineList = MachineList.replace("\'","")
        i = i +1
    elif arg == '-MachineListGeneric':
        MachineListGeneric =  sys.argv[i+1]
        MachineListGeneric = MachineListGeneric.replace("\"","")
        MachineListGeneric = MachineListGeneric.replace("\'","")
        i = i +1
    elif arg == '--skip-git-head-check':
        flag_skip_git_head_check=True
    elif arg == '--timeout_cmd': 
        Timeout_cmd = sys.argv[i+1]
        i = i +1
    elif arg == '--skip-oai-install':
        flag_skip_oai_install=True
    elif arg == '-h' :
        print "-s:  This flag *MUST* be set to start the test cases"
        print "-r:  Remove the log directory in autotests"
        print "-g:  Run test cases in a group"
        print "-c:  Run cleanup scripts on remote machines and exit"
        print "-5GRepo:  Repository for OAI 5G to use to run tests (overrides GitOAI5GRepo in test_case_list.xml)"
        print "-5GRepoBranch:  Branch for OAI 5G Repository to run tests (overrides the branch in test_case_list.xml)"
        print "-5GRepoHeadVersion:  Head commit on which to run tests (overrides the branch in test_case_list.xml)"
        print "-u:  use the user name passed as argument"
        print "-p:  use the password passed as an argument"
        print "-n:  Set the NFS share passed as an argument"
        print "--nrun_lte_softmodem:  Set the number of runs for lte-softmodem test case class"
        print "-MachineList : overrides the MachineList parameter in test_case_list.xml"
        print "-MachineListGeneric : overrides the MachineListGeneric  parameter in test_case_list.xml"
        print "--skip-git-head-check: skip checking of GitHead remote/local branch (only for debugging)"
        print "--timeout_cmd: Override the default parameter (timeout_cmd) in test_case_list.xml. This parameter is in seconds and should be > 120"
        print "--skip-oai-install: Skips the openairinterface5g installer"
        sys.exit()
    else :
        print "Unrecongnized Option: <" + arg + ">. Use -h to see valid options"
        sys.exit()
    i= i + 1     

try:  
   os.environ["OPENAIR1_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR1_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR2_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR2_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR_TARGETS"]
except KeyError: 
   print "Please set the environment variable OPENAIR_TARGETS in the .bashrc"
   sys.exit(1)

print "Killing zombie ssh sessions from earlier sessions..."
cmd='ps aux |grep \"/usr/bin/ssh -q -l guptar\"| awk \'{print $2}\' | sudo xargs kill -9  '

os.system(cmd)

if flag_start_testcase == False:
  print "You need to start the testcase by passing option -s. Use -h to see all options. Aborting now..."
  sys.exit(1)

# get the oai object
host = os.uname()[1]
#oai = openair('localdomain','calisson')
oai_list = []

#start_time = time.time()  # datetime.datetime.now()
if user=='':
  user = getpass.getuser()
if pw=='':
  pw = getpass.getpass()

print "Killing zombie ssh sessions from earlier sessions..."
cmd='ps aux |grep \"/usr/bin/ssh -q -l guptar\"|tr -s \" \" :|cut -f 2 -d :|xargs kill -9 '
cmd = cmd + '; ps aux |grep \"/usr/bin/ssh -q -l ' + user + '\"|tr -s \" \" :|cut -f 2 -d :|xargs kill -9 '
os.system(cmd)

print "host = " + host 
print "user = " + user
xmlInputFile=os.environ.get('OPENAIR_DIR')+"/cmake_targets/autotests/test_case_list.xml"
NFSResultsDir = '/mnt/sradio'
cleanupOldProgramsScript = '$OPENAIR_DIR/cmake_targets/autotests/tools/remove_old_programs.bash'
logdir = '/tmp/' + 'OAITestFrameWork-' + user + '/'
logdirOAI5GRepo = logdir + 'openairinterface5g/'
logdirOpenaircnRepo = logdir + 'openair-cn/'

if flag_remove_logdir == True:
   print "Removing directory: " + locallogdir
   os.system(' rm -fr ' + locallogdir + '; mkdir -p ' +  locallogdir  )

paramiko_logfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/log/paramiko.log')
res=os.system(' echo > ' + paramiko_logfile)
paramiko.util.log_to_file(paramiko_logfile)

#pw=getpass.getpass()

#Now we parse the xml file for basic configuration
xmlTree = ET.parse(xmlInputFile)
xmlRoot = xmlTree.getroot()



if MachineList =='':
   MachineList = xmlRoot.findtext('MachineList',default='')
NFSResultsShare = xmlRoot.findtext('NFSResultsShare',default='')
GitOpenaircnRepo = xmlRoot.findtext('GitOpenair-cnRepo',default='')

if GitOAI5GRepo == '':
   GitOAI5GRepo = xmlRoot.findtext('GitOAI5GRepo',default='')

if GitOAI5GRepoBranch == '':
   GitOAI5GRepoBranch = xmlRoot.findtext('GitOAI5GRepoBranch',default='')

GitOpenaircnRepoBranch = xmlRoot.findtext('GitOpenair-cnRepoBranch',default='')
CleanUpOldProgs = xmlRoot.findtext('CleanUpOldProgs',default='')
CleanUpAluLteBox = xmlRoot.findtext('CleanUpAluLteBox',default='')
Timeout_execution = int (xmlRoot.findtext('Timeout_execution'))
if MachineListGeneric == '':
   MachineListGeneric = xmlRoot.findtext('MachineListGeneric',default='')
TestCaseExclusionList = xmlRoot.findtext('TestCaseExclusionList',default='')
ExmimoRfStop = xmlRoot.findtext('ExmimoRfStop',default='')
if nruns_lte_softmodem == '':
   nruns_lte_softmodem = xmlRoot.findtext('nruns_lte-softmodem',default='')

print "MachineList = " + MachineList
print "GitOpenair-cnRepo = " + GitOpenaircnRepo
print "GitOAI5GRepo = " + GitOAI5GRepo
print "GitOAI5GBranch = " + GitOAI5GRepoBranch
print "GitOpenaircnRepoBranch = " + GitOpenaircnRepoBranch
print "NFSResultsShare = " + NFSResultsShare
print "nruns_lte_softmodem = " + nruns_lte_softmodem
print "Timeout_cmd = " + Timeout_cmd

if GitOAI5GHeadVersion == '':
  cmd = "git show-ref --heads -s "+ GitOAI5GRepoBranch
  GitOAI5GHeadVersion = subprocess.check_output ([cmd], shell=True)
  GitOAI5GHeadVersion=GitOAI5GHeadVersion.replace("\n","")

print "GitOAI5GHeadVersion = " + GitOAI5GHeadVersion
print "CleanUpOldProgs = " + CleanUpOldProgs
print "Timeout_execution = " + str(Timeout_execution)

if GitOAI5GHeadVersion == '':
  print "Error getting the OAI5GBranch Head version...Exiting"
  sys.exit()

NFSTestsResultsDir = NFSResultsShare + '/'+ GitOAI5GRepoBranch + '/' + GitOAI5GHeadVersion

print "NFSTestsResultsDir = " + NFSTestsResultsDir

MachineList = MachineList.split()
MachineListGeneric = MachineListGeneric.split()

#index=0
for machine in MachineList: 
  oai_list.append( openair('localdomain',machine))
  #index = index + 1


print "\nTesting the sanity of machines used for testing..."
if localshell == 0:
    try:
        index=0
        for machine in MachineList:
           print '\n******* Note that the user <'+user+'> should be a sudoer *******\n'
           print '******* Connecting to the machine <'+machine+'> to perform the test *******\n'
           if not pw :
              print "username: " + user 
              #pw = getpass.getpass() 
              #print "password: " + pw            
           else :
              print "username: " + user 
              #print "password: " + pw 
           # issues in ubuntu 12.04
           oai_list[index].connect(user,pw)
           print '\nChecking for sudo permissions on machine <'+machine+'>...'
           result = oai_list[index].send_expect_false('sudo -S -v','may not run sudo',True)
           print "Sudo permissions..." + result
           
           print '\nCleaning Older running programs : ' + CleanUpOldProgs
           cleanOldPrograms(oai_list[index], CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, '$HOME', '/tmp')

           #result = oai_list[index].send('mount ' + NFSResultsDir, True)
           #print "Mounting NFS Share " + NFSResultsDir + "..." + result

           # Check if NFS share is mounted correctly.
           #print 'Checking if NFS Share<' + NFSResultsDir + '> is mounted correctly...'
           #cmd = 'if grep -qs '+NFSResultsDir+ ' /proc/mounts; then  echo \'' + NFSResultsDir  + ' is mounted\' ; fi'
           #search_expr = NFSResultsDir + ' is mounted'
           #print "cmd = " + cmd
           #print "search_expr = " + search_expr
           #result = oai_list[index].send_expect(cmd, search_expr)
           #print "Mount NFS_Results_Dir..." + result
           index = index + 1
           
           #oai.connect2(user,pw) 
           #oai.get_shell()
    except :
        print 'Fail to connect to the machine: '+ machine 
        sys.exit(1)
else:
    pw = ''
    oai_list[0].connect_localshell()

#We now prepare the machines for testing
index=0
threads_init_setup=[]
for oai in oai_list:
  try:
      print "setting up machine: " + MachineList[index]
      #print oai_list[oai].send_recv('echo \''+pw+'\' |sudo -S -v')
      #print oai_list[oai].send_recv('sudo su')
      #print oai_list[oai].send_recv('who am i') 
      #cleanUpPrograms(oai_list[oai]
      cmd = 'sudo -S -E rm -fr ' + logdir + ' ; mkdir -p ' + logdir 
      result = oai.send_recv(cmd)
     
      setuplogfile  = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
      setup_script  = locallogdir  + '/setup_script_' + MachineList[index] +  '_.txt'

      #Sometimes git fails so the script below retries in that case
      localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/git-retry.sh')
      remotefile = logdir + '/git-retry.sh'
      paramList=[]
      port=22
      paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} )
      sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
      sftp_module (user, pw, MachineList[index], port, paramList, sftp_log)


      cmd = ' ( \n'
      #cmd = cmd  + 'rm -fR ' +  logdir + '\n'
      #cmd = cmd + 'mkdir -p ' + logdir + '\n'
      cmd = cmd + 'cd '+ logdir   + '\n'
      cmd = cmd + 'sudo apt-get install -y git \n'
      cmd = cmd + 'chmod 700 ' + logdir + '/git-retry.sh \n' 
      cmd = cmd + logdir + '/git-retry.sh clone  '+ GitOAI5GRepo  +' \n'
      cmd = cmd + logdir + '/git-retry.sh clone '+ GitOpenaircnRepo + ' \n'
      cmd = cmd +  'cd ' + logdirOAI5GRepo  + '\n'
      cmd = cmd + 'git checkout ' + GitOAI5GRepoBranch   + '\n'                      
      #cmd = cmd + 'git checkout ' + GitOAI5GHeadVersion   + '\n'
      cmd = cmd + 'git_head=`git ls-remote |grep \'' + GitOAI5GRepoBranch + '\'` \n'
      cmd = cmd + 'git_head=($git_head) \n'
      cmd = cmd + 'git_head=${git_head[0]} \n'
      cmd = cmd + 'echo \"GitOAI5GHeadVersion_remote = $git_head\" \n'
      cmd = cmd + 'echo \"GitOAI5GHeadVersion_local = ' + GitOAI5GHeadVersion + '\" \n'
      if flag_skip_git_head_check==True:
         cmd = cmd + 'echo \"skipping GitHead check...\" \n '
      else:
         cmd = cmd + 'if [ \"$git_head\" != \"'+ GitOAI5GHeadVersion + '\" ]; then echo \"error: Git openairinterface5g head version does not match\" ; fi \n'
      cmd = cmd + 'source oaienv'   + '\n'
      if flag_skip_oai_install == False:
         cmd = cmd + 'source $OPENAIR_DIR/cmake_targets/tools/build_helper \n'
         cmd = cmd + 'echo \"Installing core OAI dependencies...Start\" \n'
         cmd = cmd + '$OPENAIR_DIR/cmake_targets/build_oai -I --install-optional-packages \n'
         cmd = cmd + 'echo \"Installing core OAI dependencies...Finished\" \n'
         #cmd = cmd + 'echo \"Installing BLADERF OAI dependencies...Start\" \n'
         #cmd = cmd + 'check_install_bladerf_driver \n'
         #cmd = cmd + 'echo \"Installing BLADERF OAI dependencies...Finished\" \n'
         #cmd = cmd + 'echo \"Installing USRP OAI dependencies...Start\" \n'
         #cmd = cmd + 'check_install_usrp_uhd_driver \n'
         #cmd = cmd + 'echo \"Installing USRP OAI dependencies...Finished\" \n'
      cmd = cmd +  'cd ' + logdirOpenaircnRepo  + '\n'
      cmd = cmd +  'git checkout ' + GitOpenaircnRepoBranch  + '\n'
      cmd = cmd +  'env |grep OPENAIR'  + '\n'
      cmd = cmd + ' cd ' + logdir   + '\n'
      cmd = cmd + ' ) > ' +  setuplogfile + ' 2>&1 \n'
      #cmd = cmd + 'echo \' ' + cmd  + '\' > ' + setup_script + ' 2>&1 \n '
      #result = oai_list[index].send_recv(cmd, False, 300 )
      write_file(setup_script, cmd, mode="w")
      tempThread = oaiThread(index, 'thread_setup_'+str(index)+'_' + MachineList[index] , MachineList[index] , user, pw, cmd, False, 3000)
      threads_init_setup.append(tempThread )
      tempThread.start()
      index = index + 1
  except Exception, e:
         print 'There is error in one of the commands to setup the machine '+ MachineList[index] 
         error=''
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + traceback.format_exc()
         print error
         sys.exit(1)

#Now we wait for all the threads to complete
index = 0
for t in threads_init_setup:
   t.join()
   setuplogfile  = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
   setup_script  = locallogdir  + '/setup_script_' + MachineList[index] +  '_.txt'
   localfile = locallogdir + '/setup_log_' + MachineList[index] + '_.txt'
   remotefile = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
   port = 22
   
   paramList=[]
   sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
   paramList.append ( {"operation":'get', "localfile":localfile, "remotefile":remotefile} )
   #sftp_module (user, pw, MachineList[index], port, localfile, remotefile, sftp_log, "get")

   #Now we copy test_case_list.xml on the remote machines
   localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml')
   remotefile = logdirOAI5GRepo + '/cmake_targets/autotests/test_case_list.xml'
   paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} )
   sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
   sftp_module (user, pw, MachineList[index], port, paramList, sftp_log)

   cmd =  '  cd ' + logdirOAI5GRepo + ' ; source oaienv ; env|grep OPENAIR \n'
   res = oai_list[index].send_recv(cmd)
   index  = index +1
   if os.path.exists(localfile) == 0:
      print "Setup log file <" + localfile + "> missing for machine <" + MachineList[index] + ">.  Please check the setup log files. Exiting now"
      sys.exit(1)

#Now we process all the test cases
#Now we check if there was error in setup files

status, out = commands.getstatusoutput('grep ' +  ' -il \'error\' ' + locallogdir + '/setup_log*')
if (out != '') :
  print "There is error in setup of machines"
  print "status  = " + str(status) + "\n Check files for error = " + out
  print "Exiting now..."
  sys.exit(1)

cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, '$HOME' , logdirOAI5GRepo)
if cleanUpRemoteMachines == True:
  sys.exit(0)

threadListGlobal=[]
testcaseList=xmlRoot.findall('testCase')
#print testcaseList
for testcase in testcaseList:
  try:
    testcasename = testcase.get('id')
    testcaseclass = testcase.findtext('class',default='')
    desc = testcase.findtext('desc',default='')
    #print "Machine list top level = " + ','.join(MachineList)
    if search_test_case_group(testcasename, testcasegroup, TestCaseExclusionList) == True:
      if testcaseclass == 'lte-softmodem' :
        eNBMachine = testcase.findtext('eNB',default='')
        UEMachine = testcase.findtext('UE',default='')
        EPCMachine = testcase.findtext('EPC',default='')
        #index_eNBMachine = MachineList.index(eNBMachine)
        #index_UEMachine = MachineList.index(UEMachine)
        #index_EPCMachine = MachineList.index(EPCMachine)
        if (eNBMachine not in MachineList)|(UEMachine not in MachineList)|(UEMachine not in MachineList):
           print "One of the machines is not in the machine list"
           print "eNBMachine : " + eNBMachine + "UEMachine : " + UEMachine + "EPCMachine : " + EPCMachine + "MachineList : " + ','.join(MachineList)
        print "testcasename = " + testcasename + " class = " + testcaseclass
        threadListGlobal = wait_testcaseclass_generic_threads(threadListGlobal, Timeout_execution)
        #cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)
        handle_testcaseclass_softmodem (testcase, CleanUpOldProgs, logdirOAI5GRepo, logdirOpenaircnRepo, MachineList, user, pw, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, Timeout_cmd )
        
        #The lines below are copied from below to trace the failure of some of the machines in test setup. These lines below need to be removed in long term
        print "Creating xml file for overall results..."
        cmd = "cat $OPENAIR_DIR/cmake_targets/autotests/log/*/*.xml > $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml "
        res=os.system(cmd)
        os.system('sync')
        print "Now copying files to NFS Share"
        oai_localhost = openair('localdomain','localhost')
        oai_localhost.connect(user,pw)
        cmd = ' mkdir -p ' + NFSTestsResultsDir
        res = oai_localhost.send_recv(cmd)

        print "Copying files from GilabCI Runner Machine : " + host + " .locallogdir = " + locallogdir + ", NFSTestsResultsDir = " + NFSTestsResultsDir
        SSHSessionWrapper('localhost', user, None, pw , NFSTestsResultsDir , locallogdir, "put_all")
        oai_localhost.disconnect()

      elif (testcaseclass == 'compilation'): 
        threadListGlobal = handle_testcaseclass_generic (testcasename, threadListGlobal, CleanUpOldProgs, logdirOAI5GRepo, MachineListGeneric, user, pw, CleanUpAluLteBox,Timeout_execution, ExmimoRfStop)
      elif (testcaseclass == 'execution'): 
        threadListGlobal = handle_testcaseclass_generic (testcasename, threadListGlobal, CleanUpOldProgs, logdirOAI5GRepo, MachineListGeneric, user, pw, CleanUpAluLteBox, Timeout_execution, ExmimoRfStop)
      else :
        print "Unknown test case class: " + testcaseclass
        sys.exit()

  except Exception, e:
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
     error = error + '\n testcasename = ' + testcasename + '\n testcaseclass = ' + testcaseclass + '\n desc = ' + 'desc' + '\n'  
     error = error + traceback.format_exc()
     print error
     print "Continuing to next test case..."
     #sys.exit(1)


print "Exiting the test cases execution now. Waiting for existing threads to complete..."

for param in threadListGlobal:
   thread_id = param["thread_id"]
   thread_id.join()

print "Creating xml file for overall results..."
cmd = "cat $OPENAIR_DIR/cmake_targets/autotests/log/*/*.xml > $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml "
res=os.system(cmd)

print "Now copying files to NFS Share"
oai_localhost = openair('localdomain','localhost')
oai_localhost.connect(user,pw)
cmd = 'mkdir -p ' + NFSTestsResultsDir
res = oai_localhost.send_recv(cmd)

print "Copying files from GilabCI Runner Machine : " + host + " .locallogdir = " + locallogdir + ", NFSTestsResultsDir = " + NFSTestsResultsDir
SSHSessionWrapper('localhost', user, None, pw , NFSTestsResultsDir , locallogdir, "put_all")

cmd = "cat " + NFSTestsResultsDir + "/log/*/*.xml > " + NFSTestsResultsDir + "/log/results_autotests.xml"
res = oai_localhost.send_recv(cmd)
 
oai_localhost.disconnect()

sys.exit()