Newer
Older
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#******************************************************************************
# OpenAirInterface
# Copyright(c) 1999 - 2014 Eurecom
# OpenAirInterface is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# OpenAirInterface is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with OpenAirInterface.The full GNU General Public License is
# included in this distribution in the file called "COPYING". If not,
# see <http://www.gnu.org/licenses/>.
# Contact Information
# OpenAirInterface Admin: openair_admin@eurecom.fr
# OpenAirInterface Tech : openair_tech@eurecom.fr
# OpenAirInterface Dev : openair4g-devel@lists.eurecom.fr
# Address : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE
# *******************************************************************************/
# \file test01.py
# \brief test 01 for OAI
# \author Navid Nikaein
# \date 2013 - 2015
# \version 0.1
# @ingroup _test
import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock
import xml.etree.ElementTree as ET
import log
from openair import *
import paramiko
sys.path.append('/opt/ssh')
import ssh
from ssh import SSHSession
def write_file(filename, string, mode="w"):
text_file = open(filename, mode)
text_file.write(string)
text_file.close()
#$1 name of file (assuming created with iperf -s -u ....
#$2 minimum throughput
#$3 maximum throughput
#$4 average throughput
#$5 minimum duration of throughput
#The throughput values found in file must be higher than values from from 2,3,4,5
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
if os.path.exists(filename):
with open (filename, "r") as myfile:
data=myfile.read()
p=re.compile('(\d*.\d*) Mbits/sec')
array=p.findall(data)
array = [ float(x) for x in array ]
duration = array.__len__()
if duration !=0:
min_list = min(array)
max_list = max(array)
average_list = np.mean(array)
else:
min_list = 0
max_list = 0
average_list=0
tput_string=' ( '+ "min=%0.2f" % min_list + ' Mbps / ' + "max=%0.2f" % max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) '
if (min_list >= min_tput and max_list >= max_tput and average_list >= average and duration >= min_duration):
def try_convert_to_float(string, fail=None):
try:
return float(string)
except Exception:
return fail;
def tput_test_search_expr (search_expr, logfile_traffic):
result=0
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
if search_expr !='':
if search_expr.find('throughput_test')!= -1 :
p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
min_tput=p.findall(search_expr)
if min_tput.__len__()==1:
min_tput = min_tput[0]
else:
min_tput = None
p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
max_tput=p.findall(search_expr)
if max_tput.__len__()==1:
max_tput = max_tput[0]
else:
max_tput = None
p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
avg_tput=p.findall(search_expr)
if avg_tput.__len__()==1:
avg_tput=avg_tput[0]
else:
avg_tput = None
p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
duration=p.findall(search_expr)
if duration.__len__()==1:
duration = duration[0]
else:
duration = None
min_tput = try_convert_to_float(min_tput)
max_tput = try_convert_to_float(max_tput)
avg_tput = try_convert_to_float(avg_tput)
duration = try_convert_to_float(duration)
if (min_tput != None and max_tput != None and avg_tput != None and duration != None ):
result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
def sftp_module (username, password, hostname, ports, paramList,logfile):
#localD = localfile
#remoteD = remotefile
#fd, paramiko_logfile = tempfile.mkstemp()
#res = os.close(fd )
#paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
error = ""
#The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
transport = paramiko.Transport(hostname, ports)
transport.connect(username = username, password = password)
sftp = paramiko.SFTPClient.from_transport(transport)
# index =0
for param in paramList:
try:
operation = param["operation"]
localD = param["localfile"]
remoteD = param["remotefile"]
if operation == "put":
sftp.put(remotepath=remoteD, localpath=localD)
elif operation == "get":
sftp.get(remotepath=remoteD, localpath=localD)
else :
print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
print "hostname = " + hostname
print "ports = " + ports
print "localfile = " + localD
print "remotefile = " + remoteD
print "operation = " + operation
sys.exit()
except Exception, e:
error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e)
error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'
error = error + traceback.format_exc()
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
write_file(logfile, error, "a")
res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')
def finalize_deploy_script (timeout_cmd, terminate_missing_procs='True'):
cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
if terminate_missing_procs == 'True':
cmd = cmd + """
#The code below checks if one the processes launched in background has crashed.
#If it does, then the code below terminates all the child processes created by this script
declare -i wakeup_interval=1
declare -i step=0
echo \"Array pid = ${array_exec_pid[@]}\"
while [ "$step" -lt "$timeout_cmd" ]
do
declare -i break_while_loop=0
#Iterate over each process ID in array_exec_pid
for i in "${array_exec_pid[@]}"
do
numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
echo "PID = $i, numchild = $numchild"
if [ "$numchild" -eq "0" ] ; then
echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
break_while_loop=1
break
fi
done
if [ "$break_while_loop" -eq "1" ] ; then
break
fi
step=$(( step + wakeup_interval ))
sleep $wakeup_interval
done
echo "Final time step (Duration of test case) = $step "
"""
else:
#We do not terminate the script if one of the processes has existed prematurely
cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date \n'
return cmd
def update_config_file(oai, config_string, logdirRepo, python_script):
if config_string :
stringArray = config_string.splitlines()
cmd=""
#python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
for string in stringArray:
#split the string based on space now
string1=string.split()
cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + ' ' + string1[1] + ' '+ string1[2] + '\n'
#cmd = cmd + 'perl -p -i -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\' ' + logdirRepo + '/' +string1[0] + '\n'
return cmd
#result = oai.send_recv(cmd)
def SSHSessionWrapper(machine, username, key_file, password, logdir_remote_testcase, logdir_local_base):
while True:
try:
ssh = SSHSession(machine , username, key_file, password)
ssh.get_all(logdir_remote_testcase , logdir_local_base)
break
except Exception, e:
error=''
error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e)
error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote_testcase + '\n logdir_local_base = ' + logdir_local_base
error = error + traceback.format_exc()
print error
print " Trying again in 1 seconds"
time.sleep(1)
print "Continuing ..."
#Function to clean old programs that might be running from earlier execution
#oai - parameter for making connection to machine
#programList - list of programs that must be terminated before execution of any test case
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop):
cmd = 'killall -q -r ' + programList
result = oai.send(cmd, True)
print "Killing old programs..." + result
programArray = programList.split()
programListJoin = '|'.join(programArray)
cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
#result = oai.send_recv(cmd)
#print result
result = oai.send_expect_false(cmd, 'Match found', False)
print result
res=oai.send_recv(CleanUpAluLteBox, True)
class myThread (threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
print "Starting " + self.name
class oaiThread (threading.Thread):
def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
threading.Thread.__init__(self)
self.threadID = threadID
self.threadname = threadname
self.machine = machine
self.username = username
self.password = password
self.cmd = cmd
self.sudo = sudo
self.timeout = timeout
def run(self):
try:
oai = openair('localdomain',self.machine)
oai.connect(user, self.password)
print "Starting " + self.threadname + " on machine " + self.machine
result = oai.send_recv(self.cmd, self.sudo, self.timeout)
print "result = " + result
print "Exiting " + self.threadname
oai.disconnect()
except Exception, e:
error=''
error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e)
error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + self.timeout + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) + '\n'
error = error + traceback.format_exc()
print error
#This class runs test cases with class execution, compilatation
class testCaseThread_generic (threading.Thread):
def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, password, timeout, ExmimoRfStop):
self.name = name
self.testcasename = testcasename
self.timeout = timeout
self.machine = machine
self.logdirOAI5GRepo = logdirOAI5GRepo
self.oldprogramList = oldprogramList
self.CleanupAluLteBox = CleanupAluLteBox
self.password=password
def run(self):
try:
mypassword=''
#addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
addpass = 'echo \'' + mypassword + '\' | '
user = getpass.getuser()
print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout)
oai = openair('localdomain',self.machine)
oai.connect(user, self.password)
cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
logdir_local = os.environ.get('OPENAIR_DIR')
logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
#print "logdir_local_testcase = " + logdir_local_testcase
#print "logdir_remote_testcase = " + logdir_remote_testcase
#if os.path.exists(logdir_local_testcase) == True :
# os.removedirs(logdir_local_testcase)
#os.mkdir(logdir_local_testcase)
os.system("rm -fr " + logdir_local_testcase )
os.system("mkdir -p " + logdir_local_testcase)
cmd = "mkdir -p " + logdir_remote_testcase
res = oai.send_recv(cmd, False, self.timeout)
cmd = "( cd " + self.logdirOAI5GRepo + " \n "
cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
cmd = cmd + " ) >& " + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase + "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase
#print "cmd = " + cmd
res = oai.send_recv(cmd, False, self.timeout)
#print "res = " + res
#print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
write_file(logfile_task_testcasename, cmd, mode="w")
#Now we copy all the remote files
#ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
#ssh.get_all(logdir_remote_testcase , logdir_local_base)
SSHSessionWrapper(self.machine, user, None, self.password, logdir_remote_testcase, logdir_local_base)
print "Finishing test case : " + self.testcasename + " On machine " + self.machine
cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
oai.disconnect()
except Exception, e:
error=''
error = error + ' In Class = testCaseThread_generic, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e)
error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo + '\n' + '\n timeout = ' + str(self.timeout)
error = error + traceback.format_exc()
print "Continuing with next test case..."
#sys.exit()
def addsudo (cmd, password=""):
cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
return cmd
def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, password, CleanupAluLteBox,timeout, ExmimoRfStop):
try:
mypassword=password
MachineListFree=[]
while MachineListFree.__len__() == 0 :
MachineListBusy=[]
#first we need to find the list of free machines that we could run our test case
if threadListGeneric.__len__() ==0 :
#This means no thread is started yet
else :
for param in threadListGeneric :
thread_id = param["thread_id"]
machine = param["Machine"]
testcasenameold = param["testcasename"]
threadListNew.append(param)
print "thread_id is alive: testcasename: " + testcasenameold + " on machine "+ machine
if machine not in MachineListBusy:
MachineListBusy.append(machine)
print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
#threadListGeneric.remove(param)
#if machine not in MachineListFree:
# MachineListFree.append(machine)
#Now we check if there is at least one free machine
MachineListFree = MachineList[:]
for machine in MachineListBusy:
if machine in MachineListFree:
MachineListFree.remove(machine)
print "MachineListFree = " + ','.join(MachineListFree)
print "MachineListBusy = " + ','.join(MachineListBusy)
print "MachineList = " + ','.join(MachineList)
machine = MachineListFree[0]
thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, password, timeout, ExmimoRfStop)
param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
thread.start()
threadListNew.append(param)
return threadListNew
except Exception, e:
error=''
error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e)
error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) + '\n'
error = error + traceback.format_exc()
print error
print "Continuing..."
#sys.exit(1)
#Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
threadListGenericNew=[]
for param in threadListGeneric:
thread_id = param["thread_id"]
machine = param["Machine"]
testcasenameold = param["testcasename"]
thread_id.join(timeout)
if thread_id.isAlive() == True:
threadListGenericNew.append(param)
print "thread_id on machine: " + machine + " is still alive: testcasename: " + testcasenameold
print " Exiting now..."
sys.exit(1)
else:
print "thread_id on machine: " + machine + " is stopped: testcasename: " + testcasenameold
#threadListGeneric.remove(param)
return threadListGenericNew
#Function to handle test case class : lte-softmodem
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, password, CleanUpAluLteBox, ExmimoRfStop):
#We ignore the password sent to this function for secuirity reasons for password present in log files
#It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
# your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
mypassword=''
#addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
addpass = 'echo \'' + mypassword + '\' | '
user = getpass.getuser()
testcasename = testcase.get('id')
testcaseclass = testcase.findtext('class',default='')
timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
timeout_cmd = int(float(timeout_cmd))
#Timeout_thread is more than that of cmd to have room for compilation time, etc
timeout_thread = timeout_cmd + 300
nruns = testcase.findtext('nruns',default='')
nruns = int(float(nruns))
tags = testcase.findtext('tags',default='')
eNBMachine = testcase.findtext('eNB',default='')
eNB_config_file = testcase.findtext('eNB_config_file',default='')
eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='True')
eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')
UEMachine = testcase.findtext('UE',default='')
UE_config_file = testcase.findtext('UE_config_file',default='')
UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
UE_main_exec = testcase.findtext('UE_main_exec',default='')
UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='True')
UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
UE_stop_script = testcase.findtext('UE_stop_script','')
EPCMachine = testcase.findtext('EPC',default='')
EPC_config_file = testcase.findtext('EPC_config_file',default='')
EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')
EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')
HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')
EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='True')
EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')
index_eNBMachine = MachineList.index(eNBMachine)
index_UEMachine = MachineList.index(UEMachine)
index_EPCMachine = MachineList.index(EPCMachine)
cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
oai_eNB = openair('localdomain', eNBMachine)
oai_eNB.connect(user, password)
oai_UE = openair('localdomain', UEMachine)
oai_UE.connect(user, password)
oai_EPC = openair('localdomain', EPCMachine)
oai_EPC.connect(user, password)
cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
logdir_UE = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename
logdir_local = os.environ.get('OPENAIR_DIR')
if logdir_local is None:
print "Environment variable OPENAIR_DIR not set correctly"
sys.exit()
#Make the log directory of test case
#cmd = 'mkdir -p ' + logdir_eNB
#result = oai_eNB.send_recv(cmd)
#cmd = 'mkdir -p ' + logdir_UE
#result = oai_UE.send_recv(cmd)
#cmd = 'mkdir -p ' + logdir_EPC
#result = oai_EPC.send_recv(cmd)
print "Updating the config files for ENB/UE/EPC..."
#updating the eNB/UE/EPC configuration file from the test case
#update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
#update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
#update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
test_result=1
test_result_string=''
start_time=time.time()
for run in range(0,nruns):
run_result=1
run_result_string=''
logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
logdir_UE = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
#Make the log directory of test case
cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
result = oai_eNB.send_recv(cmd)
cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' + logdir_UE
result = oai_UE.send_recv(cmd)
cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
result = oai_EPC.send_recv(cmd)
cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
result = os.system(cmd)
logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log'
task_eNB_compile = ' ( uname -a ; date \n'
task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR \n'
task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
if eNB_compile_prog != "":
task_eNB_compile = task_eNB_compile + ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
task_eNB_compile = task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1 '
write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")
task_eNB = ' ( uname -a ; date \n'
task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
task_eNB = task_eNB + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n'
if eNB_pre_exec != "":
task_eNB = task_eNB + ' ( ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
if eNB_main_exec != "":
task_eNB = task_eNB + ' ( ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
if eNB_traffic_exec != "":
task_eNB = task_eNB + ' ( ' + eNB_traffic_exec + ' ' + eNB_traffic_exec_args + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n '
task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
#terminate the eNB test case after timeout_cmd seconds
task_eNB = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
#task_eNB = task_eNB + 'sleep ' + str(timeout_cmd) + ' \n'
task_eNB = task_eNB + 'handle_ctrl_c' + '\n'
task_eNB = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1 '
write_file(logfile_task_eNB, task_eNB, mode="w")
#task_eNB = 'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB
logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'
logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log'
task_UE_compile = ' ( uname -a ; date \n'
task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n'
task_UE_compile = task_UE_compile + 'source oaienv \n'
task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
task_UE_compile = task_UE_compile + 'env |grep OPENAIR \n'
task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
if UE_compile_prog != "":
task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
task_UE_compile = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
write_file(logfile_task_UE_compile, task_UE_compile, mode="w")
task_UE = ' ( uname -a ; date \n'
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
task_UE = task_UE + 'array_exec_pid=()' + '\n'
task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'
task_UE = task_UE + 'source oaienv \n'
task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
task_UE = task_UE + 'env |grep OPENAIR \n'
if UE_pre_exec != "":
task_UE = task_UE + ' ( ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
if UE_main_exec != "":
task_UE = task_UE + ' ( ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword) + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
task_UE = task_UE + 'array_exec_pid+=($!) \n'
task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
if UE_traffic_exec != "":
task_UE = task_UE + ' ( ' + UE_traffic_exec + ' ' + UE_traffic_exec_args + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
task_UE = task_UE + 'array_exec_pid+=($!) \n'
task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
#terminate the UE test case after timeout_cmd seconds
task_UE = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
#task_UE = task_UE + 'sleep ' + str(timeout_cmd) + ' \n'
task_UE = task_UE + 'handle_ctrl_c' + '\n'
task_UE = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
write_file(logfile_task_UE, task_UE, mode="w")
#task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE
logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log'
task_EPC_compile = ' ( uname -a ; date \n'
task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + '\n'
task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
task_EPC_compile = task_EPC_compile + 'source BUILD/TOOLS/build_helper \n'
if EPC_compile_prog != "":
task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args + ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
if HSS_compile_prog != "":
task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
task_EPC_compile = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 '
write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
task_EPC = ' ( uname -a ; date \n'
task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '\n'
task_EPC = task_EPC + 'source BUILD/TOOLS/build_helper \n'
if EPC_pre_exec != "":
task_EPC = task_EPC + ' ( ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
if HSS_main_exec != "":
task_EPC = task_EPC + '(' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS + ' 2>&1 & \n'
task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
if EPC_main_exec != "":
task_EPC = task_EPC + '(' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC + ' 2>&1 & \n'
task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
if EPC_traffic_exec != "":
task_EPC = task_EPC + '(' + EPC_traffic_exec + ' ' + EPC_traffic_exec_args + ' ) > ' + logfile_traffic_EPC + ' 2>&1 & \n'
task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
#terminate the EPC test case after timeout_cmd seconds
task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
#task_EPC = task_EPC + 'sleep ' + str(timeout_cmd) + '\n'
task_EPC = task_EPC + 'handle_ctrl_c' '\n'
task_EPC = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 '
write_file(logfile_task_EPC, task_EPC, mode="w")
#first we compile all the programs
thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password , task_UE_compile, False, timeout_thread)
threads=[]
threads.append(thread_eNB)
threads.append(thread_UE)
threads.append(thread_EPC)
# Start new Threads
thread_eNB.start()
thread_UE.start()
thread_EPC.start()
#Wait for all the compile threads to complete
for t in threads:
t.join()
#Now we execute all the threads
thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password , task_UE, False, timeout_thread)
threads=[]
threads.append(thread_eNB)
threads.append(thread_UE)
threads.append(thread_EPC)
# Start new Threads
thread_eNB.start()
thread_UE.start()
thread_EPC.start()
#Wait for all the compile threads to complete
for t in threads:
t.join()
#Now we get the log files from remote machines on the local machine
cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'
if UE_stop_script != "":
cmd = ' ( uname -a ; date \n'
cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
cmd = cmd + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n'
cmd = cmd + UE_stop_script + '\n'
cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 '
write_file(logfile_UE_stop_script , cmd, mode="w")
thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password , cmd, False, timeout_thread)
thread_UE.start()
thread_UE.join()
print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
#ssh = SSHSession(EPCMachine , username=user, key_file=None, password=password)
#ssh.get_all(logdir_EPC , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
#ssh = SSHSession(eNBMachine , username=user, key_file=None, password=password)
#ssh.get_all(logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
#ssh = SSHSession(UEMachine , username=user, key_file=None, password=password)
#ssh.get_all(logdir_UE , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
#Currently we only perform throughput tests
tput_run_string=''
result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
tput_run_string = tput_run_string + tput_string
result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
tput_run_string = tput_run_string + tput_string
result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
tput_run_string = tput_run_string + tput_string
run_result_string = ' RUN_'+str(run) + ' = PASS'
run_result_string = ' RUN_'+str(run) + ' = FAIL'
run_result_string = run_result_string + tput_run_string
test_result=test_result & run_result
test_result_string=test_result_string + run_result_string
oai_eNB.disconnect()
oai_UE.disconnect()
oai_EPC.disconnect()
#We need to close the new ssh session that was created
#if index_eNBMachine == index_EPCMachine:
# oai_EPC.disconnect()
#Now we finalize the xml file of the test case
end_time=time.time()
duration= end_time - start_time
xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
if test_result ==0:
result='FAIL'
else:
result = 'PASS'
xml="\n<testcase classname=\'"+ testcaseclass + "\' name=\'" + testcasename + "."+tags + "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
write_file(xmlFile, xml, mode="w")
#This function searches if test case is present in list of test cases that need to be executed by user
Rohit Gupta
committed
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
Rohit Gupta
committed
if test_case_exclude != "":
testcase_exclusion_list=test_case_exclude.split()
for entry in testcase_exclusion_list:
if entry.find('+') >=0:
match = re.search(entry, testcasename)
if match:
print "\nSkipping test case as it is found in black list: " + testcasename
return False
else:
match = entry.find(testcasename)
if match >=0:
print "\nSkipping test case as it is found in black list: " + testcasename
return False
Rohit Gupta
committed
return True
else:
testcaselist = testcasegroup.split()
for entry in testcaselist:
if entry.find('+') >=0:
match = re.search(entry, testcasename)
if match:
return True
else:
match = testcasename.find(entry)
if match >=0:
def cleanOldProgramsAllMachines(oai_list, CleanOldProgs, CleanUpAluLteBox, ExmimoRfStop):
for index in oai_list:
cleanOldPrograms(oai_list[index], CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)
#thread1 = myThread(1, "Thread-1", 1)
debug = 0
pw =''
i = 0
dlsim=0
localshell=0
is_compiled = 0
timeout=2000
xmlInputFile="./test_case_list.xml"
NFSResultsDir = '/mnt/sradio'
cleanupOldProgramsScript = '$OPENAIR_DIR/cmake_targets/autotests/tools/remove_old_programs.bash'
cleanUpRemoteMachines=False
logdir = '/tmp/' + 'OAITestFrameWork-' + getpass.getuser() + '/'
logdirOAI5GRepo = logdir + 'openairinterface5g/'
logdirOpenaircnRepo = logdir + 'openair-cn/'
openairdir_local = os.environ.get('OPENAIR_DIR')
if openairdir_local is None:
print "Environment variable OPENAIR_DIR not set correctly"
sys.exit()
locallogdir = openairdir_local + '/cmake_targets/autotests/log/'
#Remove the contents of local log directory
#os.system(' rm -fr ' + locallogdir + '; mkdir -p ' + locallogdir )
flag_remove_logdir=False
i=1
while i < len (sys.argv):
arg=sys.argv[i]
if arg == '-d':
debug = 1
elif arg == '-dd':
debug = 2
elif arg == '-p' :
prompt2 = sys.argv[i+1]
i = i +1
elif arg == '-r':
flag_remove_logdir=True
elif arg == '-w' :
pw = sys.argv[i+1]
elif arg == '-P' :
dlsim = 1
elif arg == '-l' :
localshell = 1
elif arg == '-c' :
is_compiled = 1
elif arg == '-t' :
timeout = sys.argv[i+1]
i = i +1
elif arg == '-g' :
testcasegroup = sys.argv[i+1].replace("\"","")
i = i +1
elif arg == '-c':
cleanUpRemoteMachines=True
elif arg == '-h' :
print "-d: low debug level"
print "-dd: high debug level"
print "-p: set the prompt"
print "-r: Remove the log directory in autotests"
print "-g: Run test cases in a group"
print "-c: Run cleanup scripts on remote machines"
print "-w: set the password for ssh to localhost"
print "-l: use local shell instead of ssh connection"
print "-t: set the time out in second for commands"
sys.exit()
else :
print "Unrecongnized Option: <" + arg + ">. Use -h to see valid options"
sys.exit()
i= i + 1
try:
os.environ["OPENAIR1_DIR"]
except KeyError:
print "Please set the environment variable OPENAIR1_DIR in the .bashrc"
sys.exit(1)
try:
os.environ["OPENAIR2_DIR"]
except KeyError:
print "Please set the environment variable OPENAIR2_DIR in the .bashrc"
sys.exit(1)
try:
os.environ["OPENAIR_TARGETS"]
except KeyError:
print "Please set the environment variable OPENAIR_TARGETS in the .bashrc"
sys.exit(1)
if flag_remove_logdir == True:
print "Removing directory: " + locallogdir
os.system(' rm -fr ' + locallogdir + '; mkdir -p ' + locallogdir )
paramiko_logfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/log/paramiko.log')
res=os.system(' echo > ' + paramiko_logfile)
paramiko.util.log_to_file(paramiko_logfile)
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# get the oai object
host = os.uname()[1]
#oai = openair('localdomain','calisson')
oai_list = {}
#start_time = time.time() # datetime.datetime.now()
user = getpass.getuser()
print "host = " + host
print "user = " + user
pw=getpass.getpass()
#Now we parse the xml file for basic configuration
xmlTree = ET.parse(xmlInputFile)
xmlRoot = xmlTree.getroot()
MachineList = xmlRoot.findtext('MachineList',default='')
NFSResultsShare = xmlRoot.findtext('NFSResultsShare',default='')
GitOpenaircnRepo = xmlRoot.findtext('GitOpenair-cnRepo',default='')
GitOAI5GRepo = xmlRoot.findtext('GitOAI5GRepo',default='')
GitOAI5GRepoBranch = xmlRoot.findtext('GitOAI5GRepoBranch',default='')
GitOpenaircnRepoBranch = xmlRoot.findtext('GitOpenair-cnRepoBranch',default='')
CleanUpOldProgs = xmlRoot.findtext('CleanUpOldProgs',default='')