run_exec_lte-softmodem_tests.py

#! /usr/bin/python
#******************************************************************************

#    OpenAirInterface 
#    Copyright(c) 1999 - 2014 Eurecom

#    OpenAirInterface is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.


#    OpenAirInterface is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#   You should have received a copy of the GNU General Public License
#   along with OpenAirInterface.The full GNU General Public License is 
#   included in this distribution in the file called "COPYING". If not, 
#   see <http://www.gnu.org/licenses/>.

#  Contact Information
#  OpenAirInterface Admin: openair_admin@eurecom.fr
#  OpenAirInterface Tech : openair_tech@eurecom.fr
#  OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr
  
#  Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

# *******************************************************************************/

# \file test01.py
# \brief test 01 for OAI
# \author Navid Nikaein
# \date 2013 - 2015
# \version 0.1
# @ingroup _test

import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock 
import xml.etree.ElementTree as ET
import re

import numpy as np

import log

from  openair import *

import paramiko

import subprocess
import commands
sys.path.append('/opt/ssh')

import ssh
from ssh import SSHSession

def write_file(filename, string, mode="w"):
   text_file = open(filename, mode)
   text_file.write(string)
   text_file.close()
   
#$1 name of file (assuming created with iperf -s -u ....
#$2 minimum throughput
#$3 maximum throughput
#$4 average throughput
#$5 minimum duration of throughput
#The throughput values found in file must be higher than values from from 2,3,4,5
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
   
   if os.path.exists(filename):
      with open (filename, "r") as myfile:
         data=myfile.read()
      p=re.compile('(\d*.\d*) Mbits/sec')
      array=p.findall(data)
      array = [ float(x) for x in array ]
      duration = array.__len__()
      if duration !=0:
        min_list = min(array) 
        max_list = max(array)
        average_list = np.mean(array)
      else:
        min_list = 0
        max_list = 0
        average_list=0
      tput_string=' ( '+ "min=%0.2f"  % min_list + ' Mbps / ' + "max=%0.2f" %  max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' 
      if (min_list >= min_tput and  max_list >= max_tput and average_list >= average and duration >= min_duration):
        return True , tput_string
      else:
        return False , tput_string
   else: 
      return False , tput_string

    
def try_convert_to_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

def tput_test_search_expr (search_expr, logfile_traffic):
   result=0
   tput_string=''
   if search_expr !='':
       if search_expr.find('throughput_test')!= -1 :
          p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
          min_tput=p.findall(search_expr)
          if min_tput.__len__()==1:
             min_tput = min_tput[0]
          else:
             min_tput = None

          p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
          max_tput=p.findall(search_expr)
          if max_tput.__len__()==1:
             max_tput = max_tput[0]
          else:
             max_tput = None

          p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
          avg_tput=p.findall(search_expr)
          if avg_tput.__len__()==1:
             avg_tput=avg_tput[0]
          else:
             avg_tput = None

          p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
          duration=p.findall(search_expr)
          if duration.__len__()==1:
             duration = duration[0]
          else:
             duration = None
          
          min_tput = try_convert_to_float(min_tput)
          max_tput = try_convert_to_float(max_tput)
          avg_tput = try_convert_to_float(avg_tput)
          duration = try_convert_to_float(duration)
          
          if (min_tput != None and max_tput != None  and avg_tput != None  and duration != None ):
             result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
   else: 
      result=1

   return result, tput_string
      

def sftp_module (username, password, hostname, ports, paramList,logfile): 
   #localD = localfile
   #remoteD = remotefile
   #fd, paramiko_logfile  = tempfile.mkstemp()
   #res = os.close(fd )
   #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
   error = ""
   #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
 


   transport = paramiko.Transport(hostname, ports)
   transport.connect(username = username, password = password)
   sftp = paramiko.SFTPClient.from_transport(transport)
   

   #  index =0 
   for param in paramList:
      try:
        operation = param["operation"] 
        localD = param["localfile"]
        remoteD = param["remotefile"]
        if operation == "put":
          sftp.put(remotepath=remoteD, localpath=localD)
        elif operation == "get":
          sftp.get(remotepath=remoteD, localpath=localD)
        else :
          print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
          print "hostname = " + hostname
          print "ports = " + ports
          print "localfile = " + localD
          print "remotefile = " + remoteD
          print "operation = " + operation
          sys.exit()
      except Exception, e:
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'  
         error = error + traceback.format_exc()
         print error

   sftp.close()
   transport.close() 
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
   res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
   write_file(logfile, error, "a")
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')

def finalize_deploy_script (timeout_cmd, terminate_missing_procs='True'):
  cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
  if terminate_missing_procs == 'True':
    cmd = cmd +  """
    #The code below checks if one the processes launched in background has crashed.
    #If it does, then the code below terminates all the child processes created by this script
    declare -i wakeup_interval=1
    declare -i step=0
    echo \"Array pid =  ${array_exec_pid[@]}\"
    while [ "$step" -lt "$timeout_cmd" ]
      do
       declare -i break_while_loop=0
       #Iterate over each process ID in array_exec_pid
       for i in "${array_exec_pid[@]}"
       do
        numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
        echo "PID = $i, numchild = $numchild"
        if  [ "$numchild" -eq "0" ] ; then
            echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
            break_while_loop=1
            break
        fi
     done
    if  [ "$break_while_loop" -eq "1" ] ; then
             break
    fi
    step=$(( step + wakeup_interval ))
    sleep $wakeup_interval
    done
    echo "Final time step (Duration of test case) = $step "
    date
    """
  else:
    #We do not terminate the script if one of the processes has existed prematurely
    cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date  \n'
  
  return cmd

def update_config_file(oai, config_string, logdirRepo, python_script):
  if config_string :
    stringArray = config_string.splitlines()
    cmd=""
    #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
    for string in stringArray:
       #split the string based on space now
       string1=string.split()
       cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + '  ' + string1[1] +  ' '+ string1[2] + '\n'
       #cmd = cmd + 'perl -p -i  -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\'   ' + logdirRepo + '/' +string1[0] + '\n'
    return cmd
    #result = oai.send_recv(cmd)

def SSHSessionWrapper(machine, username, key_file, password, logdir_remote_testcase, logdir_local_base):
  while True:
    try:
       ssh = SSHSession(machine , username, key_file, password)
       ssh.get_all(logdir_remote_testcase , logdir_local_base)
       break 
    except Exception, e:
       error=''
       error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
       error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote_testcase + '\n logdir_local_base = ' + logdir_local_base 
       error = error + traceback.format_exc()
       print error
       print " Trying again in 1 seconds"
       time.sleep(1)
       print "Continuing ..."


 
#Function to clean old programs that might be running from earlier execution
#oai - parameter for making connection to machine
#programList - list of programs that must be terminated before execution of any test case 
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop):
  cmd = 'killall -q -r ' + programList
  result = oai.send(cmd, True)
  print "Killing old programs..." + result
  programArray = programList.split()
  programListJoin = '|'.join(programArray)
  cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
  #result = oai.send_recv(cmd)
  #print result
  result = oai.send_expect_false(cmd, 'Match found', False)
  print result
  res=oai.send_recv(CleanUpAluLteBox, True)
  res = oai.send_recv(ExmimoRfStop, False)

class myThread (threading.Thread):
    def __init__(self, threadID, name, counter):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.counter = counter
    def run(self):
        print "Starting " + self.name


class oaiThread (threading.Thread):
    def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.threadname = threadname
        self.machine = machine
        self.username = username
        self.password = password
        self.cmd = cmd
        self.sudo = sudo
        self.timeout = timeout
    def run(self):
        try:
          oai = openair('localdomain',self.machine)
          oai.connect(user, self.password)
          print "Starting " + self.threadname + " on machine " + self.machine
          result = oai.send_recv(self.cmd, self.sudo, self.timeout)
          print "result = " + result
          print "Exiting " + self.threadname
          oai.disconnect()
        except Exception, e:
           error=''
           error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + self.timeout + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) +  '\n'  
           error = error + traceback.format_exc()
           print error


#This class runs test cases with class execution, compilatation
class testCaseThread_generic (threading.Thread):
   def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, password, timeout, ExmimoRfStop):
       threading.Thread.__init__(self)
       self.threadID = threadID
       self.name = name
       self.testcasename = testcasename
       self.timeout = timeout
       self.machine = machine
       self.logdirOAI5GRepo = logdirOAI5GRepo
       self.oldprogramList = oldprogramList
       self.CleanupAluLteBox = CleanupAluLteBox
       self.password=password
       self.ExmimoRfStop = ExmimoRfStop
   def run(self):
     try:
       mypassword=''
       #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
       addpass = 'echo \'' + mypassword + '\' | '
       user = getpass.getuser()
       print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) 
       oai = openair('localdomain',self.machine)
       oai.connect(user, self.password)
       cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
       logdir_local = os.environ.get('OPENAIR_DIR')
       logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
       logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
       logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
       logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
       logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
       logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
       #print "logdir_local_testcase = " + logdir_local_testcase
       #print "logdir_remote_testcase = " + logdir_remote_testcase
       #if os.path.exists(logdir_local_testcase) == True :
       #    os.removedirs(logdir_local_testcase)
       #os.mkdir(logdir_local_testcase)
       os.system("rm -fr " + logdir_local_testcase )
       os.system("mkdir -p " +  logdir_local_testcase)
       cmd = "mkdir -p " + logdir_remote_testcase
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res

       cmd =  "( cd " +  self.logdirOAI5GRepo + " \n "
       cmd = cmd + "source oaienv \n"
       cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
       cmd = cmd + " ) >& "   + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase +  "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase 
      
       #print "cmd = " + cmd
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res
       #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
       write_file(logfile_task_testcasename, cmd, mode="w")
       #Now we copy all the remote files
       #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
       #ssh.get_all(logdir_remote_testcase , logdir_local_base)
       SSHSessionWrapper(self.machine, user, None, self.password, logdir_remote_testcase, logdir_local_base)
       print "Finishing test case : " + self.testcasename + " On machine " + self.machine
       cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
       #oai.kill(user,mypassword)
       oai.disconnect()
     except Exception, e:
         error=''
         error = error + ' In Class = testCaseThread_generic,  function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo +  '\n' + '\n timeout = ' + str(self.timeout)  
         error = error + traceback.format_exc()
         print error
         print "Continuing with next test case..."
         #sys.exit()


       
   
def addsudo (cmd, password=""):
  cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
  return cmd

def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, password, CleanupAluLteBox,timeout, ExmimoRfStop):
  try:
    mypassword=password
    MachineListFree=[]
    threadListNew=[]
    while MachineListFree.__len__() == 0 :
       MachineListBusy=[]
       MachineListFree=[]
       threadListNew=[]
       #first we need to find the list of free machines that we could run our test case
       if threadListGeneric.__len__() ==0 :
       #This means no thread is started yet
          MachineListFree = MachineList[:]
       else :
          for param in threadListGeneric :
             thread_id = param["thread_id"]
             machine = param["Machine"]
             testcasenameold = param["testcasename"]
             thread_id.join(1)
             if thread_id.isAlive() == True:
                threadListNew.append(param)
                print "thread_id is alive: testcasename: " + testcasenameold +  " on machine "+ machine
                if machine not in MachineListBusy:
                   MachineListBusy.append(machine)
             else :
                print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
                #threadListGeneric.remove(param)
                #if machine not in MachineListFree:
                #   MachineListFree.append(machine)
       #Now we check if there is at least one free machine
       MachineListFree = MachineList[:]
       for machine in MachineListBusy:
          if machine in MachineListFree:
            MachineListFree.remove(machine)
       print "MachineListFree = " + ','.join(MachineListFree)
       print "MachineListBusy = " + ','.join(MachineListBusy)
       print "MachineList = " + ','.join(MachineList)
    machine = MachineListFree[0]
    thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, password, timeout, ExmimoRfStop)
    param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
    thread.start()
    threadListNew.append(param)
    return threadListNew
  except Exception, e:
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
     error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) +  '\n'  
     error = error + traceback.format_exc()
     print error
     print "Continuing..."
     #sys.exit(1)

#Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
   threadListGenericNew=[]
   for param in threadListGeneric:
      thread_id = param["thread_id"]
      machine = param["Machine"]
      testcasenameold = param["testcasename"]
      thread_id.join(timeout)
      if thread_id.isAlive() == True:
         threadListGenericNew.append(param)
         print "thread_id on machine: " + machine + "  is still alive: testcasename: " + testcasenameold
         print " Exiting now..."
         sys.exit(1)
      else:
         print "thread_id on machine: " + machine + "  is stopped: testcasename: " + testcasenameold
         #threadListGeneric.remove(param)
   return threadListGenericNew

#Function to handle test case class : lte-softmodem
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, password, CleanUpAluLteBox, ExmimoRfStop):
  #We ignore the password sent to this function for secuirity reasons for password present in log files
  #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
  # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
  mypassword=''
  #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
  addpass = 'echo \'' + mypassword + '\' | '
  user = getpass.getuser()
  testcasename = testcase.get('id')
  testcaseclass = testcase.findtext('class',default='')
  timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
  timeout_cmd = int(float(timeout_cmd))
  #Timeout_thread is more than that of cmd to have room for compilation time, etc
  timeout_thread = timeout_cmd + 300 
  nruns = testcase.findtext('nruns',default='')
  nruns = int(float(nruns))
  tags = testcase.findtext('tags',default='')
  eNBMachine = testcase.findtext('eNB',default='')
  eNB_config_file = testcase.findtext('eNB_config_file',default='')
  eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
  eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
  eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
  eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
  eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
  eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
  eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
  eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
  eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='True')
  eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')

  UEMachine = testcase.findtext('UE',default='')
  UE_config_file = testcase.findtext('UE_config_file',default='')
  UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
  UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
  UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
  UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
  UE_main_exec = testcase.findtext('UE_main_exec',default='')
  UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
  UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
  UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
  UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='True')
  UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
  UE_stop_script =  testcase.findtext('UE_stop_script','')

  EPCMachine = testcase.findtext('EPC',default='')
  EPC_config_file = testcase.findtext('EPC_config_file',default='')
  EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
  EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
  HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
  HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
  
  EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
  EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')  
  EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
  EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')  
  HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
  HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')  
  EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
  EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
  EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='True')
  EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')

  index_eNBMachine = MachineList.index(eNBMachine)
  index_UEMachine = MachineList.index(UEMachine)
  index_EPCMachine = MachineList.index(EPCMachine)
  cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
  oai_eNB = openair('localdomain', eNBMachine)
  oai_eNB.connect(user, password)
  res= oai_eNB.send_recv(cmd)
  oai_UE = openair('localdomain', UEMachine)
  oai_UE.connect(user, password)
  res = oai_eNB.send_recv(cmd)
  oai_EPC = openair('localdomain', EPCMachine)
  oai_EPC.connect(user, password)
  res = oai_eNB.send_recv(cmd)

  #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
  logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
  logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename
  logdir_local = os.environ.get('OPENAIR_DIR')
  if logdir_local is None:
     print "Environment variable OPENAIR_DIR not set correctly"
     sys.exit()
   
  #Make the log directory of test case
  #cmd = 'mkdir -p ' + logdir_eNB
  #result = oai_eNB.send_recv(cmd)
  #cmd = 'mkdir -p ' +  logdir_UE
  #result = oai_UE.send_recv(cmd)
  #cmd = 'mkdir -p ' + logdir_EPC
  #result = oai_EPC.send_recv(cmd)
  
  print "Updating the config files for ENB/UE/EPC..."
  #updating the eNB/UE/EPC configuration file from the test case 
  #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
  #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
  #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
  test_result=1
  test_result_string=''
  start_time=time.time()
  for run in range(0,nruns):
    run_result=1
    run_result_string=''
    logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    #Make the log directory of test case
    cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
    result = oai_eNB.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' +  logdir_UE
    result = oai_UE.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
    result = oai_EPC.send_recv(cmd)
    cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
    result = os.system(cmd)
    
    logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
    logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
    logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
    logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
    logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' 

    task_eNB_compile = ' ( uname -a ; date \n'
    task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR  \n'
    task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if eNB_compile_prog != "":
       task_eNB_compile  = task_eNB_compile +  ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
    task_eNB_compile =  task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1  '
    write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")

    task_eNB = ' ( uname -a ; date \n'
    task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB = task_eNB + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

    if eNB_pre_exec != "":
       task_eNB  = task_eNB +  ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
    if eNB_main_exec != "":
       task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
    if eNB_traffic_exec != "":
       task_eNB = task_eNB + ' (date;  ' + eNB_traffic_exec + ' ' + eNB_traffic_exec_args + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n '
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
    #terminate the eNB test case after timeout_cmd seconds
    task_eNB  = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
    #task_eNB  = task_eNB + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_eNB  = task_eNB + 'handle_ctrl_c' + '\n' 
    task_eNB  = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1  '
    write_file(logfile_task_eNB, task_eNB, mode="w")

    #task_eNB =  'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB 
    logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
    logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'    
    logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
    logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
    logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
    logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' 

    task_UE_compile = ' ( uname -a ; date \n'
    task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
    task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE_compile = task_UE_compile + 'source oaienv \n'
    task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
    task_UE_compile = task_UE_compile + 'env |grep OPENAIR  \n'
    task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if UE_compile_prog != "":
       task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
    task_UE_compile  = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
    write_file(logfile_task_UE_compile, task_UE_compile, mode="w")

    task_UE = ' ( uname -a ; date \n'
    task_UE = task_UE + 'array_exec_pid=()' + '\n'
    task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE = task_UE + 'source oaienv \n'
    task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
    task_UE = task_UE + 'env |grep OPENAIR  \n'
    if UE_pre_exec != "":
       task_UE  = task_UE +  ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
    if UE_main_exec != "":
       task_UE = task_UE + ' ( date;  ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword)  + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
    if UE_traffic_exec != "":
       task_UE = task_UE + ' ( date;  ' + UE_traffic_exec + ' ' + UE_traffic_exec_args + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
    #terminate the UE test case after timeout_cmd seconds
    task_UE  = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
    #task_UE  = task_UE + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_UE  = task_UE + 'handle_ctrl_c' + '\n' 
    task_UE  = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
    write_file(logfile_task_UE, task_UE, mode="w")
    #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE

    logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
    logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
    logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
    logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
    logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
    logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
    logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
    logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' 

    task_EPC_compile = ' ( uname -a ; date \n'
    task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
    task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + ' ; source oaienv \n'
    task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
    task_EPC_compile = task_EPC_compile +  'source BUILD/TOOLS/build_helper \n'
    if EPC_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args +  ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
    if HSS_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
    task_EPC_compile  = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' 
    write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
    
    task_EPC = ' ( uname -a ; date \n'
    task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
    task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n'
    task_EPC = task_EPC +  'source BUILD/TOOLS/build_helper \n'
    if EPC_pre_exec != "":
       task_EPC  = task_EPC +  ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
    if HSS_main_exec !=  "":
       task_EPC  = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS  +  ' 2>&1   & \n'
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
    if EPC_main_exec !=  "":
       task_EPC  = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC  +  ' 2>&1   & \n'
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
    if EPC_traffic_exec !=  "":
       task_EPC  = task_EPC + '( date; ' + EPC_traffic_exec + ' ' + EPC_traffic_exec_args + ' ) > ' + logfile_traffic_EPC  +  ' 2>&1   & \n' 
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'  
       task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
    #terminate the EPC test case after timeout_cmd seconds   
    task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
    #task_EPC  = task_EPC + 'sleep ' +  str(timeout_cmd) + '\n'
    task_EPC  = task_EPC + 'handle_ctrl_c' '\n' 
    task_EPC  = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' 
    write_file(logfile_task_EPC, task_EPC, mode="w")
    
    #first we compile all the programs
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE_compile, False, timeout_thread) 
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
    # Start new Threads
    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()

    #Now we execute all the threads
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE, False, timeout_thread) 

    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
    # Start new Threads

    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()

    #Wait for all the compile threads to complete
    for t in threads:
       t.join()
    #Now we get the log files from remote machines on the local machine

    cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
    cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
    cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
    logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
    logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'

    if UE_stop_script != "":
      cmd = ' ( uname -a ; date \n'
      cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
      cmd = cmd + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'
      cmd = cmd + UE_stop_script + '\n'
      cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' 
      write_file(logfile_UE_stop_script , cmd, mode="w")
      thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password  , cmd, False, timeout_thread)
      thread_UE.start()
      thread_UE.join()
   

    print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
    #ssh = SSHSession(EPCMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_EPC , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
    SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)

    print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
    #ssh = SSHSession(eNBMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
    SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)

    print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
    #ssh = SSHSession(UEMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_UE , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
    SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)


    
    #Currently we only perform throughput tests
    tput_run_string=''
    result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
    tput_run_string = tput_run_string + tput_string
    run_result=run_result&result
    result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
    run_result=run_result&result
    tput_run_string = tput_run_string + tput_string
    result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
    run_result=run_result&result
    tput_run_string = tput_run_string + tput_string
    
    if run_result == 1:  
      run_result_string = ' RUN_'+str(run) + ' = PASS'
    else:
      run_result_string = ' RUN_'+str(run) + ' = FAIL'
    
    run_result_string = run_result_string + tput_run_string

    test_result=test_result & run_result
    test_result_string=test_result_string + run_result_string

    oai_eNB.disconnect()
    oai_UE.disconnect()
    oai_EPC.disconnect()
    #We need to close the new ssh session that was created  
    #if index_eNBMachine == index_EPCMachine:
    #    oai_EPC.disconnect()
  #Now we finalize the xml file of the test case
  end_time=time.time()
  duration= end_time - start_time
  xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
  if test_result ==0: 
    result='FAIL'
  else:
    result = 'PASS'
  xml="\n<testcase classname=\'"+ testcaseclass +  "\' name=\'" + testcasename + "."+tags +  "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
  write_file(xmlFile, xml, mode="w")


#This function searches if test case is present in list of test cases that need to be executed by user
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
    
    if test_case_exclude != "":
       testcase_exclusion_list=test_case_exclude.split()
       for entry in testcase_exclusion_list:
          if entry.find('+') >=0:
            match = re.search(entry, testcasename)
            if match:
               print "\nSkipping test case as it is found in black list: " + testcasename
               return False
          else:
             match = entry.find(testcasename)
             if match >=0:
                print "\nSkipping test case as it is found in black list: " + testcasename
                return False
    if testcasegroup == '':
         return True
    else:
      testcaselist = testcasegroup.split()
      for entry in testcaselist:
        if entry.find('+') >=0:
           match = re.search(entry, testcasename)
           if match:
              return True
        else:
           match = testcasename.find(entry)
           if match >=0:
             return True
    return False

def cleanOldProgramsAllMachines(oai_list, CleanOldProgs, CleanUpAluLteBox, ExmimoRfStop):
   for index in oai_list:
      cleanOldPrograms(oai_list[index], CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)

#thread1 = myThread(1, "Thread-1", 1)
debug = 0
pw =''
i = 0
dlsim=0
localshell=0
is_compiled = 0
timeout=2000
GitOAI5GRepoBranch=''
xmlInputFile="./test_case_list.xml"
NFSResultsDir = '/mnt/sradio'
cleanupOldProgramsScript = '$OPENAIR_DIR/cmake_targets/autotests/tools/remove_old_programs.bash'
testcasegroup=''
cleanUpRemoteMachines=False
logdir = '/tmp/' + 'OAITestFrameWork-' + getpass.getuser() + '/'
logdirOAI5GRepo = logdir + 'openairinterface5g/'
logdirOpenaircnRepo = logdir + 'openair-cn/'

openairdir_local = os.environ.get('OPENAIR_DIR')
if openairdir_local is None:
   print "Environment variable OPENAIR_DIR not set correctly"
   sys.exit()
locallogdir = openairdir_local + '/cmake_targets/autotests/log/'
#Remove  the contents of local log directory
#os.system(' rm -fr ' + locallogdir + '; mkdir -p ' +  locallogdir  )
flag_remove_logdir=False
i=1
while i < len (sys.argv):
    arg=sys.argv[i]
    if arg == '-d':
        debug = 1
    elif arg == '-dd':
        debug = 2
    elif arg == '-p' :
        prompt2 = sys.argv[i+1]
        i = i +1
    elif arg == '-r':
        flag_remove_logdir=True 
    elif arg == '-w' :
        pw = sys.argv[i+1]
        i = i +1  
    elif arg == '-P' :
        dlsim = 1
    elif arg == '-l' :
        localshell = 1
    elif arg == '-c' :
        is_compiled = 1
    elif arg == '-t' :
        timeout = sys.argv[i+1]
        i = i +1  
    elif arg == '-g' :
        testcasegroup = sys.argv[i+1].replace("\"","")
        i = i +1   
    elif arg == '-c':
        cleanUpRemoteMachines=True
    elif arg == '-5GRepoBranch':
        GitOAI5GRepoBranch = sys.argv[i+1]
        i = i +1
    elif arg == '-h' :
        print "-d:  low debug level"
        print "-dd: high debug level"
        print "-p:  set the prompt"
        print "-r:  Remove the log directory in autotests"
        print "-g:  Run test cases in a group"
        print "-c:  Run cleanup scripts on remote machines"
        print "-w:  set the password for ssh to localhost"
        print "-l:  use local shell instead of ssh connection"
        print "-t:  set the time out in second for commands"
        print "-5GRepoBranch:  Branch for OAI 5G Repository to run tests (overrides the branch in test_case_list.xml)"
        sys.exit()
    else :
        print "Unrecongnized Option: <" + arg + ">. Use -h to see valid options"
        sys.exit()
    i= i + 1     

try:  
   os.environ["OPENAIR1_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR1_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR2_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR2_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR_TARGETS"]
except KeyError: 
   print "Please set the environment variable OPENAIR_TARGETS in the .bashrc"
   sys.exit(1)

if flag_remove_logdir == True:
   print "Removing directory: " + locallogdir
   os.system(' rm -fr ' + locallogdir + '; mkdir -p ' +  locallogdir  )



paramiko_logfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/log/paramiko.log')
res=os.system(' echo > ' + paramiko_logfile)
paramiko.util.log_to_file(paramiko_logfile)

# get the oai object
host = os.uname()[1]
#oai = openair('localdomain','calisson')
oai_list = {}


#start_time = time.time()  # datetime.datetime.now()
user = getpass.getuser()
print "host = " + host 
print "user = " + user
pw=getpass.getpass()

#Now we parse the xml file for basic configuration
xmlTree = ET.parse(xmlInputFile)
xmlRoot = xmlTree.getroot()




MachineList = xmlRoot.findtext('MachineList',default='')
NFSResultsShare = xmlRoot.findtext('NFSResultsShare',default='')
GitOpenaircnRepo = xmlRoot.findtext('GitOpenair-cnRepo',default='')
GitOAI5GRepo = xmlRoot.findtext('GitOAI5GRepo',default='')

if GitOAI5GRepoBranch == '':
   GitOAI5GRepoBranch = xmlRoot.findtext('GitOAI5GRepoBranch',default='')

GitOpenaircnRepoBranch = xmlRoot.findtext('GitOpenair-cnRepoBranch',default='')
CleanUpOldProgs = xmlRoot.findtext('CleanUpOldProgs',default='')
CleanUpAluLteBox = xmlRoot.findtext('CleanUpAluLteBox',default='')
Timeout_execution = int (xmlRoot.findtext('Timeout_execution'))
MachineListGeneric = xmlRoot.findtext('MachineListGeneric',default='')
TestCaseExclusionList = xmlRoot.findtext('TestCaseExclusionList',default='')
ExmimoRfStop = xmlRoot.findtext('ExmimoRfStop',default='')
print "MachineList = " + MachineList
print "GitOpenair-cnRepo = " + GitOpenaircnRepo
print "GitOAI5GRepo = " + GitOAI5GRepo
print "GitOAI5GBranch = " + GitOAI5GRepoBranch
print "GitOpenaircnRepoBranch = " + GitOpenaircnRepoBranch
print "NFSResultsShare = " + NFSResultsShare
cmd = "git show-ref --heads -s "+ GitOAI5GRepoBranch
GitOAI5GHeadVersion = subprocess.check_output ([cmd], shell=True)
GitOAI5GHeadVersion=GitOAI5GHeadVersion.replace("\n","")
print "GitOAI5GHeadVersion = " + GitOAI5GHeadVersion
print "CleanUpOldProgs = " + CleanUpOldProgs
print "Timeout_execution = " + str(Timeout_execution)

if GitOAI5GHeadVersion == '':
  print "Error getting the OAI5GBranch Head version...Exiting"
  sys.exit()

MachineList = MachineList.split()
MachineListGeneric = MachineListGeneric.split()

index=0
for machine in MachineList: 
  oai_list[index] = openair('localdomain',machine)
  index = index + 1


#myThread (1,"sddsf", 1)


#thread1 = oaiThread1(1, "Thread-1", 1)
#def __init__(self, threadID, name, counter, oai, cmd, sudo, timeout):

#sys.exit()







print "\nTesting the sanity of machines used for testing..."
if localshell == 0:
    try:
        index=0
        for machine in MachineList:
           print '\n******* Note that the user <'+user+'> should be a sudoer *******\n'
           print '******* Connecting to the machine <'+machine+'> to perform the test *******\n'
           if not pw :
              print "username: " + user 
              #pw = getpass.getpass() 
              #print "password: " + pw            
           else :
              print "username: " + user 
              #print "password: " + pw 
           # issues in ubuntu 12.04
           oai_list[index].connect(user,pw)
           #print "result = " + result
           

           #print '\nCleaning Older running programs : ' + CleanUpOldProgs
           #cleanOldPrograms(oai_list[index], CleanUpOldProgs)



           print '\nChecking for sudo permissions on machine <'+machine+'>...'
           result = oai_list[index].send_expect_false('sudo -S -v','may not run sudo',True)
           print "Sudo permissions..." + result
           
           print '\nCleaning Older running programs : ' + CleanUpOldProgs
           cleanOldPrograms(oai_list[index], CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)

           result = oai_list[index].send('mount ' + NFSResultsDir, True)
           print "Mounting NFS Share " + NFSResultsDir + "..." + result

           # Check if NFS share is mounted correctly.
           print 'Checking if NFS Share<' + NFSResultsDir + '> is mounted correctly...'
           #result = oai_list[index].send_expect('mount | grep ' + NFSResultsDir,  NFSResultsDir )
           cmd = 'if grep -qs '+NFSResultsDir+ ' /proc/mounts; then  echo \'' + NFSResultsDir  + ' is mounted\' ; fi'
           search_expr = NFSResultsDir + ' is mounted'
           print "cmd = " + cmd
           print "search_expr = " + search_expr
           result = oai_list[index].send_expect(cmd, search_expr)
           print "Mount NFS_Results_Dir..." + result
           index = index + 1
           
           #oai.connect2(user,pw) 
           #oai.get_shell()
    except :
        print 'Fail to connect to the machine: '+ machine 
        sys.exit(1)
else:
    pw = ''
    oai_list[0].connect_localshell()





cpu_freq = int(oai_list[0].cpu_freq())
if timeout == 2000 : 
    if cpu_freq <= 2000 : 
        timeout = 3000
    elif cpu_freq < 2700 :
        timeout = 2000 
    elif cpu_freq < 3300 :
        timeout = 1500
print "cpu freq(MHz): " + str(cpu_freq) + "timeout(s): " + str(timeout)

# The log files are stored in branch/version/



#result = oai_list[0].send('uname -a ' )
#print result

#We now prepare the machines for testing
#index=0
threads_init_setup=[]
for index in oai_list:
  try:
      print "setting up machine: " + MachineList[index]
      #print oai_list[oai].send_recv('echo \''+pw+'\' |sudo -S -v')
      #print oai_list[oai].send_recv('sudo su')
      #print oai_list[oai].send_recv('who am i') 
      #cleanUpPrograms(oai_list[oai]
      cmd = 'sudo -S -E rm -fr ' + logdir + ' ; mkdir -p ' + logdir 
      result = oai_list[index].send_recv(cmd)
     
      setuplogfile  = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
      setup_script  = locallogdir  + '/setup_script_' + MachineList[index] +  '_.txt'
      cmd = ' ( \n'
      #cmd = cmd  + 'rm -fR ' +  logdir + '\n'
      #cmd = cmd + 'mkdir -p ' + logdir + '\n'
      cmd = cmd + 'cd '+ logdir   + '\n'
      cmd = cmd + 'git config --global http.sslVerify false \n' 
      cmd = cmd + 'git clone --depth 1 '+ GitOAI5GRepo  + ' -b ' + GitOAI5GRepoBranch +' \n'
      cmd = cmd + 'git clone '+ GitOpenaircnRepo + ' -b ' +GitOpenaircnRepoBranch +  ' \n'
      cmd = cmd +  'cd ' + logdirOAI5GRepo  + '\n'
      cmd = cmd + 'git checkout ' + GitOAI5GRepoBranch   + '\n'                      
      #cmd = cmd + 'git checkout ' + GitOAI5GHeadVersion   + '\n'
      cmd = cmd + 'git_head=`git ls-remote |grep \'' + GitOAI5GRepoBranch + '\'` \n'
      cmd = cmd + 'git_head=($git_head) \n'
      cmd = cmd + 'git_head=${git_head[0]} \n'
      cmd = cmd + 'echo \"GitOAI5GHeadVersion_remote = $git_head\"'
      cmd = cmd + 'echo \"GitOAI5GHeadVersion_local = ' + GitOAI5GHeadVersion + '\" \n'
      cmd = cmd + 'if [ \"$git_head\" != \"'+ GitOAI5GHeadVersion + '\" ]; then echo \"error: Git openairinterface5g head version does not match\" ; fi \n'
      cmd = cmd + 'source oaienv'   + '\n'
      cmd = cmd +  'cd ' + logdirOpenaircnRepo  + '\n'
      cmd = cmd +  'git checkout ' + GitOpenaircnRepoBranch  + '\n'
      cmd = cmd +  'env |grep OPENAIR'  + '\n'
      cmd = cmd + ' cd ' + logdir   + '\n'
      cmd = cmd + ' ) > ' +  setuplogfile + ' 2>&1 \n'
      #cmd = cmd + 'echo \' ' + cmd  + '\' > ' + setup_script + ' 2>&1 \n '
      #result = oai_list[index].send_recv(cmd, False, 300 )
      write_file(setup_script, cmd, mode="w")
      tempThread = oaiThread(index, 'thread_setup_'+str(index)+'_' + MachineList[index] , MachineList[index] , user, pw, cmd, False, 300)
      threads_init_setup.append(tempThread )
      tempThread.start()
      
      #localfile = locallogdir + '/setup_log_' + MachineList[index] + '_.txt'
      #remotefile = logdir  + '/setup_log_' + MachineList[index] + '_.txt'

      #sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
      #sftp_module (user, pw, MachineList[index], 22, localfile, remotefile, sftp_log, "get")


      #Now we copy test_case_list.xml on the remote machines
      #localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml')
      #remotefile = logdirOAI5GRepo + '/cmake_targets/autotests/test_case_list.xml'

      #sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
      #sftp_module (user, pw, MachineList[index], 22, localfile, remotefile, sftp_log, "put")


      #print oai_list[index].send('rm -fR ' +  logdir)
      #print oai_list[index].send('mkdir -p ' + logdir)
      #print oai_list[index].send('cd '+ logdir)
      #print oai_list[index].send('git clone '+ GitOAI5GRepo )
      #print oai_list[index].send('git clone '+ GitOpenaircnRepo)
      #print oai_list[index].send('cd ' + logdirOAI5GRepo)
      #print oai_list[index].send('git checkout ' + GitOAI5GHeadVersion)
      #print oai_list[index].send('source oaienv')
      #print oai_list[index].send('cd ' + logdirOpenaircnRepo)
      #print oai_list[index].send('git checkout ' + GitOpenaircnRepoBranch)
      #print oai_list[index].send_recv('cd ' + logdirOAI5GRepo)
      #print oai_list[index].send_recv('source oaienv')
      #print oai_list[index].send_recv('env |grep OPENAIR')

      #print '\nCleaning Older running programs : ' + CleanUpOldProgs
      #cleanOldPrograms(oai_list[index], CleanUpOldProgs)
  except Exception, e:
         print 'There is error in one of the commands to setup the machine '+ MachineList[index] 
         error=''
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + traceback.format_exc()
         print error
         sys.exit(1)

#Now we wait for all the threads to complete
index = 0
for t in threads_init_setup:
   t.join()
   setuplogfile  = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
   setup_script  = locallogdir  + '/setup_script_' + MachineList[index] +  '_.txt'
   localfile = locallogdir + '/setup_log_' + MachineList[index] + '_.txt'
   remotefile = logdir  + '/setup_log_' + MachineList[index] + '_.txt'
   port = 22
   
   paramList=[]
   sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
   paramList.append ( {"operation":'get', "localfile":localfile, "remotefile":remotefile} )
   #sftp_module (user, pw, MachineList[index], port, localfile, remotefile, sftp_log, "get")

   #Now we copy test_case_list.xml on the remote machines
   localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml')
   remotefile = logdirOAI5GRepo + '/cmake_targets/autotests/test_case_list.xml'
   paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} )
   sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log')
   sftp_module (user, pw, MachineList[index], port, paramList, sftp_log)

   cmd =  '  cd ' + logdirOAI5GRepo + ' ; source oaienv ; env|grep OPENAIR \n'
   res = oai_list[index].send_recv(cmd)
   index  = index +1
   if os.path.exists(localfile) == 0:
      print "Setup log file <" + localfile + "> missing for machine <" + MachineList[index] + ">.  Please check the setup log files. Exiting now"
      sys.exit(1)

#Now we process all the test cases
#Now we check if there was error in setup files

status, out = commands.getstatusoutput('grep ' +  ' -il \'error\' ' + locallogdir + '/setup_log*')
if (out != '') :
  print "There is error in setup of machines"
  print "status  = " + str(status) + "\n Check files for error = " + out
  print sys.exit(1)

if cleanUpRemoteMachines == True:
  cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)
  sys.exit(0)

threadListGlobal=[]
testcaseList=xmlRoot.findall('testCase')
#print testcaseList
for testcase in testcaseList:
  try:
    testcasename = testcase.get('id')
    testcaseclass = testcase.findtext('class',default='')
    desc = testcase.findtext('desc',default='')
    #print "Machine list top level = " + ','.join(MachineList)
    if search_test_case_group(testcasename, testcasegroup, TestCaseExclusionList) == True:
      cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)
      if testcaseclass == 'lte-softmodem' :
        eNBMachine = testcase.findtext('eNB',default='')
        UEMachine = testcase.findtext('UE',default='')
        EPCMachine = testcase.findtext('EPC',default='')
        #index_eNBMachine = MachineList.index(eNBMachine)
        #index_UEMachine = MachineList.index(UEMachine)
        #index_EPCMachine = MachineList.index(EPCMachine)
        if (eNBMachine not in MachineList)|(UEMachine not in MachineList)|(UEMachine not in MachineList):
           print "One of the machines is not in the machine list"
           print "eNBMachine : " + eNBMachine + "UEMachine : " + UEMachine + "EPCMachine : " + EPCMachine + "MachineList : " + ','.join(MachineList)
        print "testcasename = " + testcasename