run_exec_lte-softmodem_tests.py 66.5 KB
Newer Older
1
#! /usr/bin/python
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#******************************************************************************

#    OpenAirInterface 
#    Copyright(c) 1999 - 2014 Eurecom

#    OpenAirInterface is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.


#    OpenAirInterface is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#   You should have received a copy of the GNU General Public License
#   along with OpenAirInterface.The full GNU General Public License is 
#   included in this distribution in the file called "COPYING". If not, 
#   see <http://www.gnu.org/licenses/>.

#  Contact Information
#  OpenAirInterface Admin: openair_admin@eurecom.fr
#  OpenAirInterface Tech : openair_tech@eurecom.fr
#  OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr
  
#  Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

# *******************************************************************************/

# \file test01.py
# \brief test 01 for OAI
# \author Navid Nikaein
# \date 2013 - 2015
# \version 0.1
# @ingroup _test

import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock 
import xml.etree.ElementTree as ET
Rohit Gupta's avatar
Rohit Gupta committed
50
import re
51

52
53
import numpy as np

54
55
56
57
58
import log

from  openair import *

import paramiko
59

60
61
import subprocess
import commands
62
63
64
65
sys.path.append('/opt/ssh')

import ssh
from ssh import SSHSession
66

Rohit Gupta's avatar
Rohit Gupta committed
67
68
69
70
# \brief write a string to a file
# \param filename name of file
# \param string string to write
# \mode file opening mode (default=write)
71
72
73
74
def write_file(filename, string, mode="w"):
   text_file = open(filename, mode)
   text_file.write(string)
   text_file.close()
Rohit Gupta's avatar
Rohit Gupta committed
75
76
77
78
79
80
81
82
 
# \brief function to check if test case passed throughput test
# \param filename name of file which has throughput results (usually from iperf -s ...
# \param min_tput minimum throughput
# \param max_tuput maximum throughput
# \param average average throughput
# \param min_duration minimum duration of throughput
#The throughput values found in file must be higher than values from from arguments 2,3,4,5
83
84
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
85
   
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
   if os.path.exists(filename):
      with open (filename, "r") as myfile:
         data=myfile.read()
      p=re.compile('(\d*.\d*) Mbits/sec')
      array=p.findall(data)
      array = [ float(x) for x in array ]
      duration = array.__len__()
      if duration !=0:
        min_list = min(array) 
        max_list = max(array)
        average_list = np.mean(array)
      else:
        min_list = 0
        max_list = 0
        average_list=0
101
      tput_string=' ( '+ "min=%0.2f"  % min_list + ' Mbps / ' + "max=%0.2f" %  max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' 
102
      if (min_list >= min_tput and  max_list >= max_tput and average_list >= average and duration >= min_duration):
103
        return True , tput_string
104
      else:
105
        return False , tput_string
106
   else: 
107
      return False , tput_string
108

Rohit Gupta's avatar
Rohit Gupta committed
109
# \brief Convert string to float or return None if there is exception    
110
111
112
113
114
115
def try_convert_to_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

Rohit Gupta's avatar
Rohit Gupta committed
116
117
118
# \brief get throughput statistics from log file
# \param search_expr search expression found in test_case_list.xml file
# \param logfile_traffic logfile which has traffic statistics
119
120
def tput_test_search_expr (search_expr, logfile_traffic):
   result=0
121
   tput_string=''
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
   if search_expr !='':
       if search_expr.find('throughput_test')!= -1 :
          p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
          min_tput=p.findall(search_expr)
          if min_tput.__len__()==1:
             min_tput = min_tput[0]
          else:
             min_tput = None

          p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
          max_tput=p.findall(search_expr)
          if max_tput.__len__()==1:
             max_tput = max_tput[0]
          else:
             max_tput = None

          p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
          avg_tput=p.findall(search_expr)
          if avg_tput.__len__()==1:
             avg_tput=avg_tput[0]
          else:
             avg_tput = None

          p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
          duration=p.findall(search_expr)
          if duration.__len__()==1:
             duration = duration[0]
          else:
             duration = None
          
          min_tput = try_convert_to_float(min_tput)
          max_tput = try_convert_to_float(max_tput)
          avg_tput = try_convert_to_float(avg_tput)
          duration = try_convert_to_float(duration)
          
          if (min_tput != None and max_tput != None  and avg_tput != None  and duration != None ):
158
             result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
Rohit Gupta's avatar
Rohit Gupta committed
159
160
161
   else: 
      result=1

162
   return result, tput_string
163
      
Rohit Gupta's avatar
Rohit Gupta committed
164
165
166
167
168
169
170
# \brief function to copy files to/from remote machine
# \param username user with which to make sftp connection
# \param password password of user
# \param hostname host to connect
# \ports port of remote machine on which server is listening
# \paramList This is list of operations as a set {operation: "get/put", localfile: "filename", remotefile: "filename"
# \param logfile Ignored currently and set once at the beginning of program
Rohit Gupta's avatar
Rohit Gupta committed
171
172
173
def sftp_module (username, password, hostname, ports, paramList,logfile): 
   #localD = localfile
   #remoteD = remotefile
174
175
176
177
   #fd, paramiko_logfile  = tempfile.mkstemp()
   #res = os.close(fd )
   #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
   error = ""
Rohit Gupta's avatar
Rohit Gupta committed
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
   #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
   transport = paramiko.Transport(hostname, ports)
   transport.connect(username = username, password = password)
   sftp = paramiko.SFTPClient.from_transport(transport)
   #  index =0 
   for param in paramList:
      try:
        operation = param["operation"] 
        localD = param["localfile"]
        remoteD = param["remotefile"]
        if operation == "put":
          sftp.put(remotepath=remoteD, localpath=localD)
        elif operation == "get":
          sftp.get(remotepath=remoteD, localpath=localD)
        else :
          print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
          print "hostname = " + hostname
          print "ports = " + ports
          print "localfile = " + localD
          print "remotefile = " + remoteD
          print "operation = " + operation
          sys.exit()
      except Exception, e:
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'  
         error = error + traceback.format_exc()
204
         print error
Rohit Gupta's avatar
Rohit Gupta committed
205
206
207

   sftp.close()
   transport.close() 
208
209
210
211
212
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
   res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
   write_file(logfile, error, "a")
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')

Rohit Gupta's avatar
Rohit Gupta committed
213
214
215
# \brief bash script stub put at the end of scripts to terminate it 
# \param timeout_cmd terminate script after timeout_cmd seconds
# \param terminate_missing_procs if True terminate all the processes launched by script if one of them terminates prematurely (due to error)
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def finalize_deploy_script (timeout_cmd, terminate_missing_procs='True'):
  cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
  if terminate_missing_procs == 'True':
    cmd = cmd +  """
    #The code below checks if one the processes launched in background has crashed.
    #If it does, then the code below terminates all the child processes created by this script
    declare -i wakeup_interval=1
    declare -i step=0
    echo \"Array pid =  ${array_exec_pid[@]}\"
    while [ "$step" -lt "$timeout_cmd" ]
      do
       declare -i break_while_loop=0
       #Iterate over each process ID in array_exec_pid
       for i in "${array_exec_pid[@]}"
       do
        numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
        echo "PID = $i, numchild = $numchild"
        if  [ "$numchild" -eq "0" ] ; then
            echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
            break_while_loop=1
            break
        fi
     done
    if  [ "$break_while_loop" -eq "1" ] ; then
             break
    fi
    step=$(( step + wakeup_interval ))
    sleep $wakeup_interval
    done
    echo "Final time step (Duration of test case) = $step "
Rohit Gupta's avatar
Rohit Gupta committed
246
    date
247
248
249
    """
  else:
    #We do not terminate the script if one of the processes has existed prematurely
Rohit Gupta's avatar
Rohit Gupta committed
250
    cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date  \n'
251
252
253
  
  return cmd

Rohit Gupta's avatar
Rohit Gupta committed
254
255
256
257
258
# \brief run python script and update config file params of test case
# \param oai module with already open connection
# \param config_string config string taken from xml file
# \param logdirRepo directory of remote repository
# \param python_script python script location
259
260
261
262
263
264
265
266
267
268
269
270
271
def update_config_file(oai, config_string, logdirRepo, python_script):
  if config_string :
    stringArray = config_string.splitlines()
    cmd=""
    #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
    for string in stringArray:
       #split the string based on space now
       string1=string.split()
       cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + '  ' + string1[1] +  ' '+ string1[2] + '\n'
       #cmd = cmd + 'perl -p -i  -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\'   ' + logdirRepo + '/' +string1[0] + '\n'
    return cmd
    #result = oai.send_recv(cmd)

Rohit Gupta's avatar
Rohit Gupta committed
272
273
274
275
276
277
278
279
280
# \brief thread safe sshsession wrapper due to occasional connection issues with ssh
# \param machine name of machine
# \param username user login for remote machine
# \param key_file file name which has keys to enable passwordless login
# \param password password for remote machine
# \param logdir_remote remote directory
# \param logdir_local_base local directory
# \param operation operation to perform (get_all, put_all) transfers recursively for directories
def SSHSessionWrapper(machine, username, key_file, password, logdir_remote, logdir_local_base, operation):
281
282
283
284
  max_tries = 100
  i=0
  while i <= max_tries:
    i = i +1
285
286
    try:
       ssh = SSHSession(machine , username, key_file, password)
287
       if operation == "get_all":
Rohit Gupta's avatar
Rohit Gupta committed
288
          ssh.get_all(logdir_remote , logdir_local_base)
289
       elif operation == "put_all":
Rohit Gupta's avatar
Rohit Gupta committed
290
          ssh.put_all(logdir_local_base, logdir_remote )
291
292
293
       else:
          print "Error: Uknown operation in SSHSessionWrapper. Exiting now..."
          sys.exit(1)
294
295
296
297
       break 
    except Exception, e:
       error=''
       error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
Rohit Gupta's avatar
Rohit Gupta committed
298
       error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote + '\n logdir_local_base = ' + logdir_local_base 
299
300
       error = error + traceback.format_exc()
       print error
301
       print " Retrying again in 1 seconds"
Rohit Gupta's avatar
Rohit Gupta committed
302
       time.sleep(1)
303
       print "Continuing ..."
304
305
306
       if i ==max_tries:
          print "Fatal Error: Max no of retries reached. Exiting now..."
          sys.exit(1)
307

308
309

 
Rohit Gupta's avatar
Rohit Gupta committed
310
311
312
313
314
# \briefFunction to clean old programs that might be running from earlier execution
# \param oai - parameter for making connection to machine
# \parm programList list of programs that must be terminated before execution of any test case 
# \param CleanUpAluLteBox program to terminate AlU Bell Labs LTE Box
# \param ExmimoRfStop String to stop EXMIMO card (specified in test_case_list.xml)
Rohit Gupta's avatar
Rohit Gupta committed
315
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop):
316
317
318
319
320
321
322
323
324
325
326
  cmd = 'killall -q -r ' + programList
  result = oai.send(cmd, True)
  print "Killing old programs..." + result
  programArray = programList.split()
  programListJoin = '|'.join(programArray)
  cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
  #result = oai.send_recv(cmd)
  #print result
  result = oai.send_expect_false(cmd, 'Match found', False)
  print result
  res=oai.send_recv(CleanUpAluLteBox, True)
327
  res = oai.send_recv(ExmimoRfStop, False)
328

Rohit Gupta's avatar
Rohit Gupta committed
329
330
331
332
333
334
335
336
337
# \brief Class thread to launch a generic command on remote machine
# \param threadID number of thread (for book keeping)
# \param threadname string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param username username with which to login
# \param password password with which to login
# \param cmd command as a string to run on remote machine
# \parma sudo if True sudo is set
# \param timeout timeout of command in seconds 
338
class oaiThread (threading.Thread):
339
    def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
340
341
        threading.Thread.__init__(self)
        self.threadID = threadID
342
343
344
345
        self.threadname = threadname
        self.machine = machine
        self.username = username
        self.password = password
346
347
348
349
        self.cmd = cmd
        self.sudo = sudo
        self.timeout = timeout
    def run(self):
350
351
        try:
          oai = openair('localdomain',self.machine)
352
          oai.connect(self.username, self.password)
353
          print "Starting " + self.threadname + " on machine " + self.machine
354
355
356
357
358
359
360
          result = oai.send_recv(self.cmd, self.sudo, self.timeout)
          print "result = " + result
          print "Exiting " + self.threadname
          oai.disconnect()
        except Exception, e:
           error=''
           error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
361
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + self.timeout + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) +  '\n username = ' + self.username + '\n'  
362
363
364
           error = error + traceback.format_exc()
           print error

365

Rohit Gupta's avatar
Rohit Gupta committed
366
367
368
369
370
371
372
373
374
375
376
# \brief This class runs test cases with class {execution, compilatation}
# \param threadID number of thread (for book keeping)
# \param name string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param testcasename name of test case to run on remote machine
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param user username with which to login
# \param password password with which to login
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
377
class testCaseThread_generic (threading.Thread):
378
   def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop):
Rohit Gupta's avatar
Rohit Gupta committed
379
       threading.Thread.__init__(self)
380
       self.threadID = threadID
Rohit Gupta's avatar
Rohit Gupta committed
381
382
383
384
385
       self.name = name
       self.testcasename = testcasename
       self.timeout = timeout
       self.machine = machine
       self.logdirOAI5GRepo = logdirOAI5GRepo
386
387
388
       self.oldprogramList = oldprogramList
       self.CleanupAluLteBox = CleanupAluLteBox
       self.password=password
Rohit Gupta's avatar
Rohit Gupta committed
389
       self.ExmimoRfStop = ExmimoRfStop
390
       self.user = user
Rohit Gupta's avatar
Rohit Gupta committed
391
392
393
394
395
   def run(self):
     try:
       mypassword=''
       #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
       addpass = 'echo \'' + mypassword + '\' | '
396
       #user = getpass.getuser()
Rohit Gupta's avatar
Rohit Gupta committed
397
       print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) 
398
       oai = openair('localdomain',self.machine)
399
400
       oai.connect(self.user, self.password)
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
401
402
403
       logdir_local = os.environ.get('OPENAIR_DIR')
       logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
       logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
404
405
406
407
408
409
410
411
412
413
414
415
416
       logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
       logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
       logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
       logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
       #print "logdir_local_testcase = " + logdir_local_testcase
       #print "logdir_remote_testcase = " + logdir_remote_testcase
       #if os.path.exists(logdir_local_testcase) == True :
       #    os.removedirs(logdir_local_testcase)
       #os.mkdir(logdir_local_testcase)
       os.system("rm -fr " + logdir_local_testcase )
       os.system("mkdir -p " +  logdir_local_testcase)
       cmd = "mkdir -p " + logdir_remote_testcase
       res = oai.send_recv(cmd, False, self.timeout) 
Rohit Gupta's avatar
Rohit Gupta committed
417
       #print "res = " + res
418
419

       cmd =  "( cd " +  self.logdirOAI5GRepo + " \n "
Rohit Gupta's avatar
Rohit Gupta committed
420
       cmd = cmd + "source oaienv \n"
421
422
423
424
425
426
       cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
       cmd = cmd + " ) >& "   + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase +  "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase 
      
       #print "cmd = " + cmd
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res
Rohit Gupta's avatar
Rohit Gupta committed
427
       #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
428
429
       write_file(logfile_task_testcasename, cmd, mode="w")
       #Now we copy all the remote files
430
431
       #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
       #ssh.get_all(logdir_remote_testcase , logdir_local_base)
432
       SSHSessionWrapper(self.machine, self.user, None, self.password, logdir_remote_testcase, logdir_local_base, "get_all")
433
       print "Finishing test case : " + self.testcasename + " On machine " + self.machine
434
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
435
       #oai.kill(user,mypassword)
436
437
438
       oai.disconnect()
     except Exception, e:
         error=''
439
         error = error + ' In Class = testCaseThread_generic,  function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
440
         error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo +  '\n' + '\n timeout = ' + str(self.timeout)  + '\n user = ' + self.user
441
         error = error + traceback.format_exc()
442
         print error
443
444
         print "Continuing with next test case..."
         #sys.exit()
Rohit Gupta's avatar
Rohit Gupta committed
445
446


447
       
Rohit Gupta's avatar
Rohit Gupta committed
448
449
450
# \bried function to run a command as a sudo
# \param cmd command as a string
# \param password password to be supplied   
451
452
453
def addsudo (cmd, password=""):
  cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
  return cmd
454

Rohit Gupta's avatar
Rohit Gupta committed
455
456
457
458
459
460
461
462
463
464
465
# \brief handler for executing test cases (compilation, execution)
# \param name of testcase
# \param threadListGeneric list of threads which are already running on remote machines
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param MachineList list of all machines on which generic test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
466
def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, user, password, CleanupAluLteBox,timeout, ExmimoRfStop):
467
468
469
  try:
    mypassword=password
    MachineListFree=[]
Rohit Gupta's avatar
Rohit Gupta committed
470
    threadListNew=[]
471
472
    while MachineListFree.__len__() == 0 :
       MachineListBusy=[]
Rohit Gupta's avatar
Rohit Gupta committed
473
474
       MachineListFree=[]
       threadListNew=[]
475
476
477
       #first we need to find the list of free machines that we could run our test case
       if threadListGeneric.__len__() ==0 :
       #This means no thread is started yet
478
          MachineListFree = MachineList[:]
479
480
481
482
       else :
          for param in threadListGeneric :
             thread_id = param["thread_id"]
             machine = param["Machine"]
483
             testcasenameold = param["testcasename"]
484
             thread_id.join(1)
485
             if thread_id.isAlive() == True:
Rohit Gupta's avatar
Rohit Gupta committed
486
487
                threadListNew.append(param)
                print "thread_id is alive: testcasename: " + testcasenameold +  " on machine "+ machine
488
489
                if machine not in MachineListBusy:
                   MachineListBusy.append(machine)
490
             else :
Rohit Gupta's avatar
Rohit Gupta committed
491
492
493
494
                print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
                #threadListGeneric.remove(param)
                #if machine not in MachineListFree:
                #   MachineListFree.append(machine)
495
496
497
498
499
       #Now we check if there is at least one free machine
       MachineListFree = MachineList[:]
       for machine in MachineListBusy:
          if machine in MachineListFree:
            MachineListFree.remove(machine)
Rohit Gupta's avatar
Rohit Gupta committed
500
501
502
       print "MachineListFree = " + ','.join(MachineListFree)
       print "MachineListBusy = " + ','.join(MachineListBusy)
       print "MachineList = " + ','.join(MachineList)
503
    machine = MachineListFree[0]
504
    thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop)
505
506
    param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
    thread.start()
Rohit Gupta's avatar
Rohit Gupta committed
507
508
    threadListNew.append(param)
    return threadListNew
509
  except Exception, e:
510
511
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
512
     error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) +  '\n' + 'user = ' + user
513
514
     error = error + traceback.format_exc()
     print error
515
516
     print "Continuing..."
     #sys.exit(1)
517

Rohit Gupta's avatar
Rohit Gupta committed
518
519
520
# \brief Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
# \param threadListGeneric list of threads which are running on remote machines
# \param timeout time to wait on threads in seconds
521
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
522
   threadListGenericNew=[]
523
524
525
526
527
528
   for param in threadListGeneric:
      thread_id = param["thread_id"]
      machine = param["Machine"]
      testcasenameold = param["testcasename"]
      thread_id.join(timeout)
      if thread_id.isAlive() == True:
529
         threadListGenericNew.append(param)
530
531
532
533
534
         print "thread_id on machine: " + machine + "  is still alive: testcasename: " + testcasenameold
         print " Exiting now..."
         sys.exit(1)
      else:
         print "thread_id on machine: " + machine + "  is stopped: testcasename: " + testcasenameold
535
536
         #threadListGeneric.remove(param)
   return threadListGenericNew
537

Rohit Gupta's avatar
Rohit Gupta committed
538
539
540
541
542
543
544
545
546
547
548

# \brief handler for executing test cases (lte-softmodem)
# \param testcase name of testcase
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed
# \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed
# \param MachineList list of all machines on which test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
549
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop):
550
551
552
553
554
555
  #We ignore the password sent to this function for secuirity reasons for password present in log files
  #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
  # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
  mypassword=''
  #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
  addpass = 'echo \'' + mypassword + '\' | '
556
  #user = getpass.getuser()
557
  testcasename = testcase.get('id')
558
  testcaseclass = testcase.findtext('class',default='')
559
560
561
562
563
564
  timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
  timeout_cmd = int(float(timeout_cmd))
  #Timeout_thread is more than that of cmd to have room for compilation time, etc
  timeout_thread = timeout_cmd + 300 
  nruns = testcase.findtext('nruns',default='')
  nruns = int(float(nruns))
565
  tags = testcase.findtext('tags',default='')
566
567
568
569
570
571
572
573
574
575
576
  eNBMachine = testcase.findtext('eNB',default='')
  eNB_config_file = testcase.findtext('eNB_config_file',default='')
  eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
  eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
  eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
  eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
  eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
  eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
  eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
  eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
  eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='True')
577
  eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')
578
579
580
581
582
583
584
585
586
587
588
589

  UEMachine = testcase.findtext('UE',default='')
  UE_config_file = testcase.findtext('UE_config_file',default='')
  UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
  UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
  UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
  UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
  UE_main_exec = testcase.findtext('UE_main_exec',default='')
  UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
  UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
  UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
  UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='True')
590
  UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
Rohit Gupta's avatar
Rohit Gupta committed
591
  UE_stop_script =  testcase.findtext('UE_stop_script','')
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608

  EPCMachine = testcase.findtext('EPC',default='')
  EPC_config_file = testcase.findtext('EPC_config_file',default='')
  EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
  EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
  HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
  HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
  
  EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
  EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')  
  EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
  EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')  
  HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
  HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')  
  EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
  EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
  EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='True')
609
  EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')
610
611
612
613

  index_eNBMachine = MachineList.index(eNBMachine)
  index_UEMachine = MachineList.index(UEMachine)
  index_EPCMachine = MachineList.index(EPCMachine)
Rohit Gupta's avatar
Rohit Gupta committed
614
  cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
615
616
  oai_eNB = openair('localdomain', eNBMachine)
  oai_eNB.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
617
  res= oai_eNB.send_recv(cmd)
618
619
  oai_UE = openair('localdomain', UEMachine)
  oai_UE.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
620
  res = oai_eNB.send_recv(cmd)
621
622
  oai_EPC = openair('localdomain', EPCMachine)
  oai_EPC.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
623
  res = oai_eNB.send_recv(cmd)
624

625
626
627
  #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
  logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
  logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename
  logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename
  logdir_local = os.environ.get('OPENAIR_DIR')
  if logdir_local is None:
     print "Environment variable OPENAIR_DIR not set correctly"
     sys.exit()
   
  #Make the log directory of test case
  #cmd = 'mkdir -p ' + logdir_eNB
  #result = oai_eNB.send_recv(cmd)
  #cmd = 'mkdir -p ' +  logdir_UE
  #result = oai_UE.send_recv(cmd)
  #cmd = 'mkdir -p ' + logdir_EPC
  #result = oai_EPC.send_recv(cmd)
  
  print "Updating the config files for ENB/UE/EPC..."
  #updating the eNB/UE/EPC configuration file from the test case 
  #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
  #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
  #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
649
650
651
  test_result=1
  test_result_string=''
  start_time=time.time()
652
  for run in range(0,nruns):
653
654
    run_result=1
    run_result_string=''
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
    logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    #Make the log directory of test case
    cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
    result = oai_eNB.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' +  logdir_UE
    result = oai_UE.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
    result = oai_EPC.send_recv(cmd)
    cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
    result = os.system(cmd)
    
    logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
    logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
673
674
    logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
675
676
    logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
    logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
677
    logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
678
679
680
681
682

    task_eNB_compile = ' ( uname -a ; date \n'
    task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR  \n'
    task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
683
    if eNB_compile_prog != "":
Rohit Gupta's avatar
Rohit Gupta committed
684
685
686
687
688
689
690
691
       task_eNB_compile  = task_eNB_compile +  ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
    task_eNB_compile =  task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1  '
    write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")

    task_eNB = ' ( uname -a ; date \n'
    task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB = task_eNB + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

692
    if eNB_pre_exec != "":
693
       task_eNB  = task_eNB +  ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
694
    if eNB_main_exec != "":
695
       task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
696
697
698
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
    if eNB_traffic_exec != "":
699
       task_eNB = task_eNB + ' (date;  ' + eNB_traffic_exec + ' ' + eNB_traffic_exec_args + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n '
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
    #terminate the eNB test case after timeout_cmd seconds
    task_eNB  = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
    #task_eNB  = task_eNB + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_eNB  = task_eNB + 'handle_ctrl_c' + '\n' 
    task_eNB  = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1  '
    write_file(logfile_task_eNB, task_eNB, mode="w")

    #task_eNB =  'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB 
    logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
    logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'    
    logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
    logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
716
717
    logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
718
    logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
719
720
721
722
723
724
725
726
727
728
729
730
731
732

    task_UE_compile = ' ( uname -a ; date \n'
    task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
    task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE_compile = task_UE_compile + 'source oaienv \n'
    task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
    task_UE_compile = task_UE_compile + 'env |grep OPENAIR  \n'
    task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if UE_compile_prog != "":
       task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
    task_UE_compile  = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
    write_file(logfile_task_UE_compile, task_UE_compile, mode="w")

    task_UE = ' ( uname -a ; date \n'
733
734
735
736
737
738
    task_UE = task_UE + 'array_exec_pid=()' + '\n'
    task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE = task_UE + 'source oaienv \n'
    task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
    task_UE = task_UE + 'env |grep OPENAIR  \n'
    if UE_pre_exec != "":
739
       task_UE  = task_UE +  ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
740
    if UE_main_exec != "":
741
       task_UE = task_UE + ' ( date;  ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword)  + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
742
743
744
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
    if UE_traffic_exec != "":
745
       task_UE = task_UE + ' ( date;  ' + UE_traffic_exec + ' ' + UE_traffic_exec_args + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
    #terminate the UE test case after timeout_cmd seconds
    task_UE  = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
    #task_UE  = task_UE + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_UE  = task_UE + 'handle_ctrl_c' + '\n' 
    task_UE  = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
    write_file(logfile_task_UE, task_UE, mode="w")
    #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE

    logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
    logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
    logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
    logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
    logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
    logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
    logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
764
765
    logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
766
    logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
767
768
769

    task_EPC_compile = ' ( uname -a ; date \n'
    task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
770
    task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + ' ; source oaienv \n'
Rohit Gupta's avatar
Rohit Gupta committed
771
772
773
774
775
776
777
778
779
780
    task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
    task_EPC_compile = task_EPC_compile +  'source BUILD/TOOLS/build_helper \n'
    if EPC_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args +  ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
    if HSS_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
    task_EPC_compile  = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' 
    write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
    
    task_EPC = ' ( uname -a ; date \n'
781
    task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
782
    task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n'
783
784
    task_EPC = task_EPC +  'source BUILD/TOOLS/build_helper \n'
    if EPC_pre_exec != "":
785
       task_EPC  = task_EPC +  ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
786
    if HSS_main_exec !=  "":
787
       task_EPC  = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS  +  ' 2>&1   & \n'
788
789
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
790
    if EPC_main_exec !=  "":
791
       task_EPC  = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC  +  ' 2>&1   & \n'
792
793
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
794
    if EPC_traffic_exec !=  "":
795
       task_EPC  = task_EPC + '( date; ' + EPC_traffic_exec + ' ' + EPC_traffic_exec_args + ' ) > ' + logfile_traffic_EPC  +  ' 2>&1   & \n' 
796
797
798
799
800
801
802
803
804
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'  
       task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
    #terminate the EPC test case after timeout_cmd seconds   
    task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
    #task_EPC  = task_EPC + 'sleep ' +  str(timeout_cmd) + '\n'
    task_EPC  = task_EPC + 'handle_ctrl_c' '\n' 
    task_EPC  = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' 
    write_file(logfile_task_EPC, task_EPC, mode="w")
    
Rohit Gupta's avatar
Rohit Gupta committed
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
    #first we compile all the programs
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE_compile, False, timeout_thread) 
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
    # Start new Threads
    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()

    #Now we execute all the threads
822
823
824
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE, False, timeout_thread) 
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839

    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
    # Start new Threads

    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()

    #Wait for all the compile threads to complete
    for t in threads:
       t.join()
    #Now we get the log files from remote machines on the local machine
840
    cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
841

842
843
844
    #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
    #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
    #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
Rohit Gupta's avatar
Rohit Gupta committed
845
846
847
848
849
850
851
852
853
854
855
856
857
858
    logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
    logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'

    if UE_stop_script != "":
      cmd = ' ( uname -a ; date \n'
      cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
      cmd = cmd + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'
      cmd = cmd + UE_stop_script + '\n'
      cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' 
      write_file(logfile_UE_stop_script , cmd, mode="w")
      thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password  , cmd, False, timeout_thread)
      thread_UE.start()
      thread_UE.join()
   
859

860
    print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
861
862
    #ssh = SSHSession(EPCMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_EPC , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
863
    SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
864

865
    print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
866
867
    #ssh = SSHSession(eNBMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
868
    SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
869

870
    print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
871
872
    #ssh = SSHSession(UEMachine , username=user, key_file=None, password=password)
    #ssh.get_all(logdir_UE , logdir_local + '/cmake_targets/autotests/log/'+ testcasename)
873
    SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
874
875


Rohit Gupta's avatar
Rohit Gupta committed
876
    
877
    #Currently we only perform throughput tests
878
879
880
    tput_run_string=''
    result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
    tput_run_string = tput_run_string + tput_string
881
    run_result=run_result&result
882
    result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
883
    run_result=run_result&result
884
885
    tput_run_string = tput_run_string + tput_string
    result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
886
    run_result=run_result&result
887
    tput_run_string = tput_run_string + tput_string
Rohit Gupta's avatar
Rohit Gupta committed
888
    
889
    if run_result == 1:  
Rohit Gupta's avatar
Rohit Gupta committed
890
      run_result_string = ' RUN_'+str(run) + ' = PASS'
891
    else:
Rohit Gupta's avatar
Rohit Gupta committed
892
      run_result_string = ' RUN_'+str(run) + ' = FAIL'
893
894
    
    run_result_string = run_result_string + tput_run_string
895
896
897
898

    test_result=test_result & run_result
    test_result_string=test_result_string + run_result_string

899
900
901
    oai_eNB.disconnect()
    oai_UE.disconnect()
    oai_EPC.disconnect()
902
    #We need to close the new ssh session that was created  
903
904
    #if index_eNBMachine == index_EPCMachine:
    #    oai_EPC.disconnect()
905
906
907
  #Now we finalize the xml file of the test case
  end_time=time.time()
  duration= end_time - start_time
Rohit Gupta's avatar
Rohit Gupta committed
908
  xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
909
910
911
912
  if test_result ==0: 
    result='FAIL'
  else:
    result = 'PASS'
913
  xml="\n<testcase classname=\'"+ testcaseclass +  "\' name=\'" + testcasename + "."+tags +  "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
914
915
  write_file(xmlFile, xml, mode="w")

916

Rohit Gupta's avatar
Rohit Gupta committed
917
918
919
920
# \brief This function searches if test case is present in list of test cases that need to be executed by user
# \param testcasename the test case to search for
# \param testcasegroup list that is passed from the arguments
# \param test_case_exclude list of test cases excluded from execution (specified in test_case_list.xml)
921
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
922
    
923
924
925
926
927
928
929
930
931
932
933
934
935
    if test_case_exclude != "":
       testcase_exclusion_list=test_case_exclude.split()
       for entry in testcase_exclusion_list:
          if entry.find('+') >=0:
            match = re.search(entry, testcasename)
            if match:
               print "\nSkipping test case as it is found in black list: " + testcasename
               return False
          else:
             match = entry.find(testcasename)
             if match >=0:
                print "\nSkipping test case as it is found in black list: " + testcasename
                return False
Rohit Gupta's avatar
Rohit Gupta committed
936
    if testcasegroup == '':
937
938
939
940
941
942
943
944
945
946
947
         return True
    else:
      testcaselist = testcasegroup.split()
      for entry in testcaselist:
        if entry.find('+') >=0:
           match = re.search(entry, testcasename)
           if match:
              return True
        else:
           match = testcasename.find(entry)
           if match >=0:
Rohit Gupta's avatar
Rohit Gupta committed
948
949
             return True
    return False
950

Rohit Gupta's avatar
Rohit Gupta committed
951
952
953
954
955
956
957
# \brief thread that cleans up remote machines from pre-existing test case executions
# \param threadID number of thread (for book keeping)
# \param threadname name of thread (for book keeping)
# \param oai handler that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
958
959
960
961
962
963
964
965
966
967
968
class oaiCleanOldProgramThread (threading.Thread):
    def __init__(self, threadID, threadname, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.threadname = threadname
        self.oai = oai
        self.CleanUpOldProgs = CleanUpOldProgs
        self.CleanUpAluLteBox = CleanUpAluLteBox
        self.ExmimoRfStop = ExmimoRfStop
    def run(self):
        try:
969
          cleanOldPrograms(self.oai, self.CleanUpOldProgs, self.CleanUpAluLteBox, self.ExmimoRfStop)
970
971
972
973
974
975
976
        except Exception, e:
           error=''
           error = error + ' In class oaiCleanOldProgramThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n CleanUpOldProgs = ' + self.CleanUpOldProgs + '\n CleanUpAluLteBox = ' + self.CleanUpAluLteBox + '\n ExmimoRfStop = ' + self.ExmimoRfStop + '\n'  
           error = error + traceback.format_exc()
           print error

Rohit Gupta's avatar
Rohit Gupta committed
977
978
979
980
981
# \brief Run parallel threads in all machines for clean up old execution of test cases
# \param oai_list list of handlers that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
982
def cleanOldProgramsAllMachines(oai_list, CleanOldProgs, CleanUpAluLteBox, ExmimoRfStop):
983
984
985
986
   threadId=0
   threadList=[]
   for oai in oai_list:
      threadName="cleanup_thread_"+str(threadId)
987
      thread=oaiCleanOldProgramThread(threadId, threadName, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop)
988
989
990
991
992
993
994
      threadList.append(thread)
      thread.start()
      threadId = threadId + 1
   for t in threadList:
      t.join()


995
996
997
998
999
debug = 0
pw =''
i = 0
dlsim=0
localshell=0
1000
GitOAI5GRepoBranch=''
1001
1002
1003
GitOAI5GHeadVersion=''
user=''
pw=''
1004
testcasegroup=''
1005
NFSResultsShare=''
1006
cleanUpRemoteMachines=False
1007
1008
1009
1010
openairdir_local = os.environ.get('OPENAIR_DIR')
if openairdir_local is None:
   print "Environment variable OPENAIR_DIR not set correctly"
   sys.exit()
1011
1012
locallogdir = openairdir_local + '/cmake_targets/autotests/log'
MachineList = ''
1013
MachineListGeneric=''
1014
flag_remove_logdir=False
Rohit Gupta's avatar
Rohit Gupta committed
1015
i=1
1016
1017
1018

print "Number of arguments argc = " + str(len(sys.argv))

Rohit Gupta's avatar
Rohit Gupta committed
1019
1020
while i < len (sys.argv):
    arg=sys.argv[i]
Rohit Gupta's avatar
Rohit Gupta committed
1021
    if arg == '-r':
1022
        flag_remove_logdir=True 
Rohit Gupta's avatar
Rohit Gupta committed
1023
1024
1025
    elif arg == '-g' :
        testcasegroup = sys.argv[i+1].replace("\"","")
        i = i +1   
1026
1027
    elif arg == '-c':
        cleanUpRemoteMachines=True
1028
1029
1030
    elif arg == '-5GRepoBranch':
        GitOAI5GRepoBranch = sys.argv[i+1]
        i = i +1
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
    elif arg == '-5GRepoHeadVersion':
        GitOAI5GHeadVersion = sys.argv[i+1]
        #We now find the branch that corresponds to this Git Head Commit
        cmd = "git show-ref --head " + " | grep " + GitOAI5GHeadVersion
        cmd_out = subprocess.check_output ([cmd], shell=True)
        cmd_out=cmd_out.replace("\n","")
        cmd_out = cmd_out.split('/')
        GitOAI5GRepoBranch = cmd_out[-1]
        if GitOAI5GRepoBranch == '':
           print "Error extracting GitBranch from head commit. Exiting now..."
           sys.exit(1)
        i = i +1
    elif arg == '-u':
        user = sys.argv[i+1]
        i = i +1
    elif arg == '-p': 
        pw = sys.argv[i+1]
        i = i +1
1049
1050
1051
    elif arg == '-n': 
        NFSResultsShare = sys.argv[i+1]
        i = i +1
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
    elif arg == '-MachineList':
        MachineList =  sys.argv[i+1]
        MachineList = MachineList.replace("\"","")
        MachineList = MachineList.replace("\'","")
        i = i +1
    elif arg == '-MachineListGeneric':
        MachineListGeneric =  sys.argv[i+1]
        MachineListGeneric = MachineListGeneric.replace("\"","")
        MachineListGeneric = MachineListGeneric.replace("\'","")
        i = i +1
1062
    elif arg == '-h' :
1063
1064
        print "-r:  Remove the log directory in autotests"
        print "-g:  Run test cases in a group"
Rohit Gupta's avatar
Rohit Gupta committed
1065
        print "-c:  Run cleanup scripts on remote machines and exit"
1066
        print "-5GRepoBranch:  Branch for OAI 5G Repository to run tests (overrides the branch in test_case_list.xml)"
1067
1068
1069
        print "-5GRepoHeadVersion:  Head commit on which to run tests (overrides the branch in test_case_list.xml)"
        print "-u:  use the user name passed as argument"
        print "-p:  use the password passed as an argument"
1070
        print "-n:  Set the NFS share passed as an argument"
1071
1072
        print "-MachineList : overrides the MachineList parameter in test_case_list.xml"
        print "-MachineListGeneric : overrides the MachineListGeneric  parameter in test_case_list.xml"
1073
        sys.exit()
Rohit Gupta's avatar
Rohit Gupta committed
1074
1075
1076
    else :
        print "Unrecongnized Option: <" + arg + ">. Use -h to see valid options"
        sys.exit()
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
    i= i + 1     

try:  
   os.environ["OPENAIR1_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR1_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR2_DIR"]
except KeyError: 
   print "Please set the environment variable OPENAIR2_DIR in the .bashrc"
   sys.exit(1)

try:  
   os.environ["OPENAIR_TARGETS"]
except KeyError: 
   print "Please set the environment variable OPENAIR_TARGETS in the .bashrc"
   sys.exit(1)

1097

Rohit Gupta's avatar
Rohit Gupta committed
1098

1099
1100
1101
# get the oai object
host = os.uname()[1]
#oai = openair('localdomain','calisson')
1102
oai_list = []
1103
1104

#start_time = time.time()  # datetime.datetime.now()
1105
1106
if user=='':
  user = getpass.getuser()
1107
if pw=='':
1108
  pw = getpass.getpass()
1109
1110
print "host = " + host 
print "user = " + user
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
xmlInputFile=os.environ.get('OPENAIR_DIR')+"/cmake_targets/autotests/test_case_list.xml"
NFSResultsDir = '/mnt/sradio'
cleanupOldProgramsScript = '$OPENAIR_DIR/cmake_targets/autotests/tools/remove_old_programs.bash'
logdir = '/tmp/' + 'OAITestFrameWork-' + user + '/'
logdirOAI5GRepo = logdir + 'openairinterface5g/'
logdirOpenaircnRepo = logdir + 'openair-cn/'

if flag_remove_logdir == True:
   print "Removing directory: " + locallogdir
   os.system(' rm -fr ' + locallogdir + '; mkdir -p ' +  locallogdir  )

1122
1123
1124
1125
paramiko_logfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/log/paramiko.log')
res=os.system(' echo > ' + paramiko_logfile)
paramiko.util.log_to_file(paramiko_logfile)

1126
#pw=getpass.getpass()
1127
1128
1129
1130
1131
1132
1133

#Now we parse the xml file for basic configuration
xmlTree = ET.parse(xmlInputFile)
xmlRoot = xmlTree.getroot()



1134
1135
if MachineList =='':
   MachineList = xmlRoot.findtext('MachineList',default='')
1136
1137
1138
NFSResultsShare = xmlRoot.findtext('NFSResultsShare',default='')
GitOpenaircnRepo = xmlRoot.findtext('GitOpenair-cnRepo',default='')
GitOAI5GRepo = xmlRoot.findtext('GitOAI5GRepo',default='')
1139
1140
1141
1142

if GitOAI5GRepoBranch == '':
   GitOAI5GRepoBranch = xmlRoot.findtext('GitOAI5GRepoBranch',default='')

1143
1144
1145
GitOpenaircnRepoBranch = xmlRoot.findtext('GitOpenair-cnRepoBranch',default='')
CleanUpOldProgs = xmlRoot.findtext('CleanUpOldProgs',default='')
CleanUpAluLteBox = xmlRoot.findtext('CleanUpAluLteBox',default='')
1146
Timeout_execution = int (xmlRoot.findtext('Timeout_execution'))
1147
1148
if MachineListGeneric == '':
   MachineListGeneric = xmlRoot.findtext('MachineListGeneric',default='')
1149
TestCaseExclusionList = xmlRoot.findtext('TestCaseExclusionList',default='')
Rohit Gupta's avatar
Rohit Gupta committed
1150
ExmimoRfStop = xmlRoot.findtext('ExmimoRfStop',default='')
1151

1152
1153
1154
1155
1156
1157
print "MachineList = " + MachineList
print "GitOpenair-cnRepo = " + GitOpenaircnRepo
print "GitOAI5GRepo = " + GitOAI5GRepo
print "GitOAI5GBranch = " + GitOAI5GRepoBranch
print "GitOpenaircnRepoBranch = " + GitOpenaircnRepoBranch
print "NFSResultsShare = " + NFSResultsShare
1158
1159
1160
1161
1162
1163

if GitOAI5GHeadVersion == '':
  cmd = "git show-ref --heads -s "+ GitOAI5GRepoBranch
  GitOAI5GHeadVersion = subprocess.check_output ([cmd], shell=True)
  GitOAI5GHeadVersion=GitOAI5GHeadVersion.replace("\n","")

1164
1165
print "GitOAI5GHeadVersion = " + GitOAI5GHeadVersion
print "CleanUpOldProgs = " + CleanUpOldProgs
1166
print "Timeout_execution = " + str(Timeout_execution)
1167

1168
1169
1170
1171
if GitOAI5GHeadVersion == '':
  print "Error getting the OAI5GBranch Head version...Exiting"
  sys.exit()

1172
NFSTestsResultsDir = NFSResultsShare + '/'+ GitOAI5GRepoBranch + '/' + GitOAI5GHeadVersion