run_exec_lte-softmodem_tests.py 79.6 KB
Newer Older
1
#! /usr/bin/python
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.0  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
22

Rohit Gupta's avatar
Rohit Gupta committed
23
# \author Rohit Gupta
24
25
26
27
28
29
30
31
32
33
34
35
36
37
# \version 0.1
# @ingroup _test

import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock 
import xml.etree.ElementTree as ET
Rohit Gupta's avatar
Rohit Gupta committed
38
import re
39

40
41
import numpy as np

42
43
44
45
46
import log

from  openair import *

import paramiko
47

48
49
import subprocess
import commands
50
sys.path.append('/opt/ssh')
51
sys.path.append(os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/'))
52

53
from lib_autotest import *
54
55
import ssh
from ssh import SSHSession
Rohit Gupta's avatar
Rohit Gupta committed
56
import argparse
57

Rohit Gupta's avatar
Rohit Gupta committed
58
59
60
61
# \brief write a string to a file
# \param filename name of file
# \param string string to write
# \mode file opening mode (default=write)
62
63
64
65
def write_file(filename, string, mode="w"):
   text_file = open(filename, mode)
   text_file.write(string)
   text_file.close()
Rohit Gupta's avatar
Rohit Gupta committed
66
67
68
69
70
71
72
73
 
# \brief function to check if test case passed throughput test
# \param filename name of file which has throughput results (usually from iperf -s ...
# \param min_tput minimum throughput
# \param max_tuput maximum throughput
# \param average average throughput
# \param min_duration minimum duration of throughput
#The throughput values found in file must be higher than values from from arguments 2,3,4,5
74
75
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
76
   
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
   if os.path.exists(filename):
      with open (filename, "r") as myfile:
         data=myfile.read()
      p=re.compile('(\d*.\d*) Mbits/sec')
      array=p.findall(data)
      array = [ float(x) for x in array ]
      duration = array.__len__()
      if duration !=0:
        min_list = min(array) 
        max_list = max(array)
        average_list = np.mean(array)
      else:
        min_list = 0
        max_list = 0
        average_list=0
92
      tput_string=' ( '+ "min=%0.2f"  % min_list + ' Mbps / ' + "max=%0.2f" %  max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' 
93
      if (min_list >= min_tput and  max_list >= max_tput and average_list >= average and duration >= min_duration):
94
        return True , tput_string
95
      else:
96
        return False , tput_string
97
   else: 
98
      return False , tput_string
99

Rohit Gupta's avatar
Rohit Gupta committed
100
# \brief Convert string to float or return None if there is exception    
101
102
103
104
105
106
def try_convert_to_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

Rohit Gupta's avatar
Rohit Gupta committed
107
108
109
# \brief get throughput statistics from log file
# \param search_expr search expression found in test_case_list.xml file
# \param logfile_traffic logfile which has traffic statistics
110
111
def tput_test_search_expr (search_expr, logfile_traffic):
   result=0
112
   tput_string=''
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
   if search_expr !='':
       if search_expr.find('throughput_test')!= -1 :
          p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
          min_tput=p.findall(search_expr)
          if min_tput.__len__()==1:
             min_tput = min_tput[0]
          else:
             min_tput = None

          p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
          max_tput=p.findall(search_expr)
          if max_tput.__len__()==1:
             max_tput = max_tput[0]
          else:
             max_tput = None

          p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
          avg_tput=p.findall(search_expr)
          if avg_tput.__len__()==1:
             avg_tput=avg_tput[0]
          else:
             avg_tput = None

          p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
          duration=p.findall(search_expr)
          if duration.__len__()==1:
             duration = duration[0]
          else:
             duration = None
          
          min_tput = try_convert_to_float(min_tput)
          max_tput = try_convert_to_float(max_tput)
          avg_tput = try_convert_to_float(avg_tput)
          duration = try_convert_to_float(duration)
          
          if (min_tput != None and max_tput != None  and avg_tput != None  and duration != None ):
149
             result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
Rohit Gupta's avatar
Rohit Gupta committed
150
151
152
   else: 
      result=1

153
   return result, tput_string
154
      
Rohit Gupta's avatar
Rohit Gupta committed
155
156
157
158
159
160
161
# \brief function to copy files to/from remote machine
# \param username user with which to make sftp connection
# \param password password of user
# \param hostname host to connect
# \ports port of remote machine on which server is listening
# \paramList This is list of operations as a set {operation: "get/put", localfile: "filename", remotefile: "filename"
# \param logfile Ignored currently and set once at the beginning of program
Rohit Gupta's avatar
Rohit Gupta committed
162
163
164
def sftp_module (username, password, hostname, ports, paramList,logfile): 
   #localD = localfile
   #remoteD = remotefile
165
166
167
168
   #fd, paramiko_logfile  = tempfile.mkstemp()
   #res = os.close(fd )
   #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
   error = ""
Rohit Gupta's avatar
Rohit Gupta committed
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
   #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
   transport = paramiko.Transport(hostname, ports)
   transport.connect(username = username, password = password)
   sftp = paramiko.SFTPClient.from_transport(transport)
   #  index =0 
   for param in paramList:
      try:
        operation = param["operation"] 
        localD = param["localfile"]
        remoteD = param["remotefile"]
        if operation == "put":
          sftp.put(remotepath=remoteD, localpath=localD)
        elif operation == "get":
          sftp.get(remotepath=remoteD, localpath=localD)
        else :
          print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
          print "hostname = " + hostname
          print "ports = " + ports
          print "localfile = " + localD
          print "remotefile = " + remoteD
          print "operation = " + operation
          sys.exit()
      except Exception, e:
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'  
         error = error + traceback.format_exc()
195
         print error
Rohit Gupta's avatar
Rohit Gupta committed
196
197
198

   sftp.close()
   transport.close() 
199
200
201
202
203
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
   res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
   write_file(logfile, error, "a")
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')

Rohit Gupta's avatar
Rohit Gupta committed
204
205
206
# \brief bash script stub put at the end of scripts to terminate it 
# \param timeout_cmd terminate script after timeout_cmd seconds
# \param terminate_missing_procs if True terminate all the processes launched by script if one of them terminates prematurely (due to error)
207
def finalize_deploy_script (timeout_cmd, terminate_missing_procs='False'):
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
  cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
  if terminate_missing_procs == 'True':
    cmd = cmd +  """
    #The code below checks if one the processes launched in background has crashed.
    #If it does, then the code below terminates all the child processes created by this script
    declare -i wakeup_interval=1
    declare -i step=0
    echo \"Array pid =  ${array_exec_pid[@]}\"
    while [ "$step" -lt "$timeout_cmd" ]
      do
       declare -i break_while_loop=0
       #Iterate over each process ID in array_exec_pid
       for i in "${array_exec_pid[@]}"
       do
        numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
        echo "PID = $i, numchild = $numchild"
        if  [ "$numchild" -eq "0" ] ; then
            echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
            break_while_loop=1
            break
        fi
     done
    if  [ "$break_while_loop" -eq "1" ] ; then
             break
    fi
    step=$(( step + wakeup_interval ))
    sleep $wakeup_interval
    done
    echo "Final time step (Duration of test case) = $step "
Rohit Gupta's avatar
Rohit Gupta committed
237
    date
238
239
240
    """
  else:
    #We do not terminate the script if one of the processes has existed prematurely
Rohit Gupta's avatar
Rohit Gupta committed
241
    cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date  \n'
242
243
244
  
  return cmd

Rohit Gupta's avatar
Rohit Gupta committed
245
246
247
248
249
# \brief run python script and update config file params of test case
# \param oai module with already open connection
# \param config_string config string taken from xml file
# \param logdirRepo directory of remote repository
# \param python_script python script location
250
def update_config_file(oai, config_string, logdirRepo, python_script):
251
  cmd=""
252
253
254
255
256
257
258
259
  if config_string :
    stringArray = config_string.splitlines()
    #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
    for string in stringArray:
       #split the string based on space now
       string1=string.split()
       cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + '  ' + string1[1] +  ' '+ string1[2] + '\n'
       #cmd = cmd + 'perl -p -i  -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\'   ' + logdirRepo + '/' +string1[0] + '\n'
260
261
  return cmd
  #result = oai.send_recv(cmd)
262

Rohit Gupta's avatar
Rohit Gupta committed
263
264
265
266
267
268
269
270
271
# \brief thread safe sshsession wrapper due to occasional connection issues with ssh
# \param machine name of machine
# \param username user login for remote machine
# \param key_file file name which has keys to enable passwordless login
# \param password password for remote machine
# \param logdir_remote remote directory
# \param logdir_local_base local directory
# \param operation operation to perform (get_all, put_all) transfers recursively for directories
def SSHSessionWrapper(machine, username, key_file, password, logdir_remote, logdir_local_base, operation):
272
273
274
275
  max_tries = 100
  i=0
  while i <= max_tries:
    i = i +1
276
277
    try:
       ssh = SSHSession(machine , username, key_file, password)
278
       if operation == "get_all":
Rohit Gupta's avatar
Rohit Gupta committed
279
          ssh.get_all(logdir_remote , logdir_local_base)
280
       elif operation == "put_all":
Rohit Gupta's avatar
Rohit Gupta committed
281
          ssh.put_all(logdir_local_base, logdir_remote )
282
283
284
       else:
          print "Error: Uknown operation in SSHSessionWrapper. Exiting now..."
          sys.exit(1)
285
286
287
288
       break 
    except Exception, e:
       error=''
       error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
Rohit Gupta's avatar
Rohit Gupta committed
289
       error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote + '\n logdir_local_base = ' + logdir_local_base 
290
291
       error = error + traceback.format_exc()
       print error
292
       print " Retrying again in 1 seconds"
Rohit Gupta's avatar
Rohit Gupta committed
293
       time.sleep(1)
294
       print "Continuing ..."
295
296
297
       if i ==max_tries:
          print "Fatal Error: Max no of retries reached. Exiting now..."
          sys.exit(1)
298

299
300

 
Rohit Gupta's avatar
Rohit Gupta committed
301
302
303
304
305
# \briefFunction to clean old programs that might be running from earlier execution
# \param oai - parameter for making connection to machine
# \parm programList list of programs that must be terminated before execution of any test case 
# \param CleanUpAluLteBox program to terminate AlU Bell Labs LTE Box
# \param ExmimoRfStop String to stop EXMIMO card (specified in test_case_list.xml)
306
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo):
307
  cmd = 'sudo -E killall -s INT -q -r ' + programList + ' ; sleep 5 ; sudo -E killall -9 -q -r ' + programList
308
309
310
311
  result = oai.send(cmd, True)
  print "Killing old programs..." + result
  programArray = programList.split()
  programListJoin = '|'.join(programArray)
312
  cmd = " ( date ;echo \"Starting cleaning old programs.. \" ; dmesg|tail ; echo \"Current disk space.. \" ; df -h )>& " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
313
  result=oai.send_recv(cmd)
314
315
316
317
  cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
  #result = oai.send_recv(cmd)
  #print result
  result = oai.send_expect_false(cmd, 'Match found', False)
318
  print "Looking for old programs..." + result
319
  res=oai.send_recv(CleanUpAluLteBox, True)
320
321
322
323
  cmd= " echo \"Starting EXmimoRF Stop... \"  >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
  oai.send_recv(cmd)
  cmd  = "( " + "cd " + logdirOAI5GRepo + " ; source oaienv ;  "  +  ExmimoRfStop + " ) >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync "
  print "cleanoldprograms cmd = " + cmd
324
  res=oai.send_recv(cmd, False, timeout=600)
325
326
327
  cmd= " echo \"Stopping EXmimoRF Stop... \" >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
  oai.send_recv(cmd)

328
329
330
  #res = oai.send_recv(ExmimoRfStop, False)
  cmd = " ( date ;echo \"Finished cleaning old programs.. \" ; dmesg | tail)>> $HOME/.oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
  res=oai.send_recv(cmd)
331

Rohit Gupta's avatar
Rohit Gupta committed
332
333
334
335
336
337
338
339
340
# \brief Class thread to launch a generic command on remote machine
# \param threadID number of thread (for book keeping)
# \param threadname string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param username username with which to login
# \param password password with which to login
# \param cmd command as a string to run on remote machine
# \parma sudo if True sudo is set
# \param timeout timeout of command in seconds 
341
class oaiThread (threading.Thread):
342
    def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
343
344
        threading.Thread.__init__(self)
        self.threadID = threadID
345
346
347
348
        self.threadname = threadname
        self.machine = machine
        self.username = username
        self.password = password
349
350
351
352
        self.cmd = cmd
        self.sudo = sudo
        self.timeout = timeout
    def run(self):
353
354
        try:
          oai = openair('localdomain',self.machine)
355
          oai.connect(self.username, self.password)
356
          print "Starting " + self.threadname + " on machine " + self.machine
357
358
359
360
361
362
363
          result = oai.send_recv(self.cmd, self.sudo, self.timeout)
          print "result = " + result
          print "Exiting " + self.threadname
          oai.disconnect()
        except Exception, e:
           error=''
           error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
364
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + str(self.timeout) + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) +  '\n username = ' + self.username + '\n'  
365
366
367
           error = error + traceback.format_exc()
           print error

368

Rohit Gupta's avatar
Rohit Gupta committed
369
370
371
372
373
374
375
376
377
378
379
# \brief This class runs test cases with class {execution, compilatation}
# \param threadID number of thread (for book keeping)
# \param name string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param testcasename name of test case to run on remote machine
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param user username with which to login
# \param password password with which to login
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
380
class testCaseThread_generic (threading.Thread):
381
   def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop):
Rohit Gupta's avatar
Rohit Gupta committed
382
       threading.Thread.__init__(self)
383
       self.threadID = threadID
Rohit Gupta's avatar
Rohit Gupta committed
384
385
386
387
388
       self.name = name
       self.testcasename = testcasename
       self.timeout = timeout
       self.machine = machine
       self.logdirOAI5GRepo = logdirOAI5GRepo
389
390
391
       self.oldprogramList = oldprogramList
       self.CleanupAluLteBox = CleanupAluLteBox
       self.password=password
Rohit Gupta's avatar
Rohit Gupta committed
392
       self.ExmimoRfStop = ExmimoRfStop
393
       self.user = user
Rohit Gupta's avatar
Rohit Gupta committed
394
395
396
397
398
   def run(self):
     try:
       mypassword=''
       #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
       addpass = 'echo \'' + mypassword + '\' | '
399
       #user = getpass.getuser()
Rohit Gupta's avatar
Rohit Gupta committed
400
       print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) 
401
       oai = openair('localdomain',self.machine)
402
403
       oai.connect(self.user, self.password)
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
404
405
406
       logdir_local = os.environ.get('OPENAIR_DIR')
       logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
       logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
407
408
409
410
411
412
413
414
415
416
417
418
419
       logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
       logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
       logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
       logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
       #print "logdir_local_testcase = " + logdir_local_testcase
       #print "logdir_remote_testcase = " + logdir_remote_testcase
       #if os.path.exists(logdir_local_testcase) == True :
       #    os.removedirs(logdir_local_testcase)
       #os.mkdir(logdir_local_testcase)
       os.system("rm -fr " + logdir_local_testcase )
       os.system("mkdir -p " +  logdir_local_testcase)
       cmd = "mkdir -p " + logdir_remote_testcase
       res = oai.send_recv(cmd, False, self.timeout) 
Rohit Gupta's avatar
Rohit Gupta committed
420
       #print "res = " + res
421
422

       cmd =  "( cd " +  self.logdirOAI5GRepo + " \n "
Rohit Gupta's avatar
Rohit Gupta committed
423
       cmd = cmd + "source oaienv \n"
424
425
426
427
428
429
       cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
       cmd = cmd + " ) >& "   + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase +  "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase 
      
       #print "cmd = " + cmd
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res
Rohit Gupta's avatar
Rohit Gupta committed
430
       #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
431
432
       write_file(logfile_task_testcasename, cmd, mode="w")
       #Now we copy all the remote files
433
434
       #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
       #ssh.get_all(logdir_remote_testcase , logdir_local_base)
435
       SSHSessionWrapper(self.machine, self.user, None, self.password, logdir_remote_testcase, logdir_local_base, "get_all")
436
       print "Finishing test case : " + self.testcasename + " On machine " + self.machine
437
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
438
       #oai.kill(user,mypassword)
439
440
441
       oai.disconnect()
     except Exception, e:
         error=''
442
         error = error + ' In Class = testCaseThread_generic,  function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
443
         error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo +  '\n' + '\n timeout = ' + str(self.timeout)  + '\n user = ' + self.user
444
         error = error + traceback.format_exc()
445
         print error
446
447
         print "Continuing with next test case..."
         #sys.exit()
Rohit Gupta's avatar
Rohit Gupta committed
448
449


450
       
Rohit Gupta's avatar
Rohit Gupta committed
451
452
453
# \bried function to run a command as a sudo
# \param cmd command as a string
# \param password password to be supplied   
454
455
456
def addsudo (cmd, password=""):
  cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
  return cmd
457

Rohit Gupta's avatar
Rohit Gupta committed
458
459
460
461
462
463
464
465
466
467
468
# \brief handler for executing test cases (compilation, execution)
# \param name of testcase
# \param threadListGeneric list of threads which are already running on remote machines
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param MachineList list of all machines on which generic test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
469
def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, user, password, CleanupAluLteBox,timeout, ExmimoRfStop):
470
471
472
  try:
    mypassword=password
    MachineListFree=[]
Rohit Gupta's avatar
Rohit Gupta committed
473
    threadListNew=[]
474
475
    while MachineListFree.__len__() == 0 :
       MachineListBusy=[]
Rohit Gupta's avatar
Rohit Gupta committed
476
477
       MachineListFree=[]
       threadListNew=[]
478
479
480
       #first we need to find the list of free machines that we could run our test case
       if threadListGeneric.__len__() ==0 :
       #This means no thread is started yet
481
          MachineListFree = MachineList[:]
482
483
484
485
       else :
          for param in threadListGeneric :
             thread_id = param["thread_id"]
             machine = param["Machine"]
486
             testcasenameold = param["testcasename"]
487
             thread_id.join(1)
488
             if thread_id.isAlive() == True:
Rohit Gupta's avatar
Rohit Gupta committed
489
490
                threadListNew.append(param)
                print "thread_id is alive: testcasename: " + testcasenameold +  " on machine "+ machine
491
492
                if machine not in MachineListBusy:
                   MachineListBusy.append(machine)
493
             else :
Rohit Gupta's avatar
Rohit Gupta committed
494
495
496
497
                print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
                #threadListGeneric.remove(param)
                #if machine not in MachineListFree:
                #   MachineListFree.append(machine)
498
499
500
501
502
       #Now we check if there is at least one free machine
       MachineListFree = MachineList[:]
       for machine in MachineListBusy:
          if machine in MachineListFree:
            MachineListFree.remove(machine)
Rohit Gupta's avatar
Rohit Gupta committed
503
504
505
       print "MachineListFree = " + ','.join(MachineListFree)
       print "MachineListBusy = " + ','.join(MachineListBusy)
       print "MachineList = " + ','.join(MachineList)
506
    machine = MachineListFree[0]
507
    thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop)
508
509
    param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
    thread.start()
Rohit Gupta's avatar
Rohit Gupta committed
510
511
    threadListNew.append(param)
    return threadListNew
512
  except Exception, e:
513
514
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
515
     error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) +  '\n' + 'user = ' + user
516
517
     error = error + traceback.format_exc()
     print error
518
519
     print "Continuing..."
     #sys.exit(1)
520

Rohit Gupta's avatar
Rohit Gupta committed
521
522
523
# \brief Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
# \param threadListGeneric list of threads which are running on remote machines
# \param timeout time to wait on threads in seconds
524
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
525
   threadListGenericNew=[]
526
527
528
529
530
531
   for param in threadListGeneric:
      thread_id = param["thread_id"]
      machine = param["Machine"]
      testcasenameold = param["testcasename"]
      thread_id.join(timeout)
      if thread_id.isAlive() == True:
532
         threadListGenericNew.append(param)
533
534
535
536
537
         print "thread_id on machine: " + machine + "  is still alive: testcasename: " + testcasenameold
         print " Exiting now..."
         sys.exit(1)
      else:
         print "thread_id on machine: " + machine + "  is stopped: testcasename: " + testcasenameold
538
539
         #threadListGeneric.remove(param)
   return threadListGenericNew
540

Rohit Gupta's avatar
Rohit Gupta committed
541
542
543
544
545
546
547
548
549
550
551

# \brief handler for executing test cases (lte-softmodem)
# \param testcase name of testcase
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed
# \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed
# \param MachineList list of all machines on which test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
552
# \param nruns_lte-softmodem global parameter to override number of runs (nruns) within the test case
553
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, timeout_cmd):
554
555
556
557
558
559
  #We ignore the password sent to this function for secuirity reasons for password present in log files
  #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
  # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
  mypassword=''
  #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
  addpass = 'echo \'' + mypassword + '\' | '
560
  #user = getpass.getuser()
561
  testcasename = testcase.get('id')
562
  testcaseclass = testcase.findtext('class',default='')
563
564
  if timeout_cmd == '':
     timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
565
566
567
  timeout_cmd = int(float(timeout_cmd))
  #Timeout_thread is more than that of cmd to have room for compilation time, etc
  timeout_thread = timeout_cmd + 300 
568
569
570
571
  if nruns_lte_softmodem == '':
    nruns = testcase.findtext('nruns',default='')
  else:
    nruns = nruns_lte_softmodem
572
  nruns = int(float(nruns))
573
  tags = testcase.findtext('tags',default='')
Rohit Gupta's avatar
Rohit Gupta committed
574
575
576
577
578
579
580
581

  RRHMachine = testcase.findtext('RRH',default='')
  RRH_compile_prog = testcase.findtext('RRH_compile_prog',default='')
  RRH_compile_prog_args = testcase.findtext('RRH_compile_prog_args',default='')
  RRH_pre_exec = testcase.findtext('RRH_pre_exec',default='')
  RRH_pre_exec_args = testcase.findtext('RRH_pre_exec_args',default='')
  RRH_main_exec = testcase.findtext('RRH_main_exec',default='')
  RRH_main_exec_args = testcase.findtext('RRH_main_exec_args',default='')
582
  RRH_terminate_missing_procs = testcase.findtext('RRH_terminate_missing_procs',default='False')
Rohit Gupta's avatar
Rohit Gupta committed
583
584


585
586
587
588
589
590
591
592
593
594
  eNBMachine = testcase.findtext('eNB',default='')
  eNB_config_file = testcase.findtext('eNB_config_file',default='')
  eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
  eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
  eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
  eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
  eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
  eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
  eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
  eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
595
  eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='False')
596
  eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')
597
  if re.compile('\w+').match(eNB_search_expr_true) != None:
598
      eNB_search_expr_true = eNB_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's' 
599
600
601
602
603
604
605
606
607
608
609

  UEMachine = testcase.findtext('UE',default='')
  UE_config_file = testcase.findtext('UE_config_file',default='')
  UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
  UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
  UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
  UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
  UE_main_exec = testcase.findtext('UE_main_exec',default='')
  UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
  UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
  UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
610
  UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='False')
611
  UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
Rohit Gupta's avatar
Rohit Gupta committed
612
  UE_stop_script =  testcase.findtext('UE_stop_script','')
613
  if re.compile('\w+').match(UE_search_expr_true) != None:
614
      UE_search_expr_true = UE_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630

  EPCMachine = testcase.findtext('EPC',default='')
  EPC_config_file = testcase.findtext('EPC_config_file',default='')
  EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
  EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
  HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
  HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
  
  EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
  EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')  
  EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
  EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')  
  HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
  HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')  
  EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
  EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
631
  EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='False')
632
  EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')
633
  if re.compile('\w+').match(EPC_search_expr_true) != None:
634
     EPC_search_expr_true = EPC_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
635
636
637
638

  index_eNBMachine = MachineList.index(eNBMachine)
  index_UEMachine = MachineList.index(UEMachine)
  index_EPCMachine = MachineList.index(EPCMachine)
Rohit Gupta's avatar
Rohit Gupta committed
639
  cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
640
641
  oai_eNB = openair('localdomain', eNBMachine)
  oai_eNB.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
642
  res= oai_eNB.send_recv(cmd)
643
644
  oai_UE = openair('localdomain', UEMachine)
  oai_UE.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
645
  res = oai_eNB.send_recv(cmd)
646
647
  oai_EPC = openair('localdomain', EPCMachine)
  oai_EPC.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
648
  res = oai_eNB.send_recv(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
649
650
651
652
653
654
  if RRHMachine != '':
    cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
    index_RRHMachine = MachineList.index(RRHMachine)
    oai_RRH = openair('localdomain', RRHMachine)
    oai_RRH.connect(user, password)
    res= oai_RRH.send_recv(cmd)
655
656
657
  #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
  logdir_local = os.environ.get('OPENAIR_DIR')
  if logdir_local is None:
     print "Environment variable OPENAIR_DIR not set correctly"
     sys.exit()
   
  #Make the log directory of test case
  #cmd = 'mkdir -p ' + logdir_eNB
  #result = oai_eNB.send_recv(cmd)
  #cmd = 'mkdir -p ' +  logdir_UE
  #result = oai_UE.send_recv(cmd)
  #cmd = 'mkdir -p ' + logdir_EPC
  #result = oai_EPC.send_recv(cmd)
  
  print "Updating the config files for ENB/UE/EPC..."
  #updating the eNB/UE/EPC configuration file from the test case 
  #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
  #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
  #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
676
677
678
  test_result=1
  test_result_string=''
  start_time=time.time()
679
  for run in range(0,nruns):
680
681
    run_result=1
    run_result_string=''
682
    logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
Rohit Gupta's avatar
Rohit Gupta committed
683
    logdir_RRH = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
684
685
686
687
    logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    #Make the log directory of test case
Rohit Gupta's avatar
Rohit Gupta committed
688
689
690
    if RRHMachine != '':
      cmd = 'rm -fr ' + logdir_RRH + ' ; mkdir -p ' + logdir_RRH
      result = oai_RRH.send_recv(cmd)
691
692
693
694
695
696
697
698
    cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
    result = oai_eNB.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' +  logdir_UE
    result = oai_UE.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
    result = oai_EPC.send_recv(cmd)
    cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
    result = os.system(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731

    if RRHMachine != '':
       logfile_compile_RRH = logdir_RRH + '/RRH_compile' + '_' + str(run) + '_.log'
       logfile_exec_RRH = logdir_RRH + '/RRH_exec' + '_' + str(run) + '_.log'
       logfile_pre_exec_RRH = logdir_RRH + '/RRH_pre_exec' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile_out = logdir_RRH + '/RRH_task_compile_out' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile = logdir_local_testcase + '/RRH_task_compile' + '_' + str(run) + '_.log'
       logfile_task_RRH_out = logdir_RRH + '/RRH_task_out' + '_' + str(run) + '_.log'
       logfile_task_RRH = logdir_local_testcase + '/RRH_task' + '_' + str(run) + '_.log'
       task_RRH_compile = ' ( uname -a ; date \n'
       task_RRH_compile = task_RRH_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
       task_RRH_compile = task_RRH_compile + 'env |grep OPENAIR  \n'
       if RRH_compile_prog != "":
         task_RRH_compile  = task_RRH_compile +  ' ( ' + RRH_compile_prog + ' '+ RRH_compile_prog_args + ' ) > ' + logfile_compile_RRH + ' 2>&1 \n'
       task_RRH_compile =  task_RRH_compile + ' date ) > ' + logfile_task_RRH_compile_out + ' 2>&1  '
       write_file(logfile_task_RRH_compile, task_RRH_compile, mode="w")

       task_RRH = ' ( uname -a ; date \n'
       task_RRH = task_RRH + ' export OPENAIR_TESTDIR=' + logdir_RRH + '\n'
       task_RRH = task_RRH + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
       task_RRH = task_RRH + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

       if RRH_pre_exec != "":
          task_RRH  = task_RRH +  ' ( date; ' + RRH_pre_exec + ' '+ RRH_pre_exec_args + ' ) > ' + logfile_pre_exec_RRH + ' 2>&1 \n'
       if RRH_main_exec != "":
          task_RRH = task_RRH + ' ( date; ' + addsudo(RRH_main_exec + ' ' + RRH_main_exec_args, mypassword) + ' ) > ' + logfile_exec_RRH + ' 2>&1 & \n'
          task_RRH = task_RRH + 'array_exec_pid+=($!) \n'
          task_RRH = task_RRH + 'echo eNB_main_exec PID = $! \n'
       #terminate the eNB test case after timeout_cmd seconds
       task_RRH  = task_RRH + finalize_deploy_script (timeout_cmd, RRH_terminate_missing_procs) + ' \n'
       task_RRH  = task_RRH + 'handle_ctrl_c' + '\n' 
       task_RRH  = task_RRH + ' ) > ' + logfile_task_RRH_out + ' 2>&1  '
       write_file(logfile_task_RRH, task_RRH, mode="w")
732
733
734
735
736
    
    logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
    logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
737
738
    logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
739
740
    logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
    logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
741
    logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
742
743
    logfile_tshark_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.log'
    logfile_pcap_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap'
744
    logfile_pcap_zip_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap.zip'
Rohit Gupta's avatar
Rohit Gupta committed
745
    logfile_pcap_tmp_eNB = '/tmp/' + '/eNB_tshark' + '_' + str(run) + '_.pcap'
Rohit Gupta's avatar
Rohit Gupta committed
746
747
748
749
750

    task_eNB_compile = ' ( uname -a ; date \n'
    task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR  \n'
    task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
751
    if eNB_compile_prog != "":
Rohit Gupta's avatar
Rohit Gupta committed
752
753
754
755
756
       task_eNB_compile  = task_eNB_compile +  ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
    task_eNB_compile =  task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1  '
    write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")

    task_eNB = ' ( uname -a ; date \n'
757
    task_eNB = task_eNB + ' export OPENAIR_TESTDIR=' + logdir_eNB + '\n'
Rohit Gupta's avatar
Rohit Gupta committed
758
759
760
    task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB = task_eNB + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

761
    if eNB_pre_exec != "":
762
       task_eNB  = task_eNB +  ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
763
    if eNB_main_exec != "":
764
       task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
765
766
767
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
    if eNB_traffic_exec != "":
768
769
       cmd_traffic = eNB_traffic_exec + ' ' + eNB_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
770
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
771
       task_eNB = task_eNB + ' (date;  ' + cmd_traffic + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n'
772
773
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
Rohit Gupta's avatar
Rohit Gupta committed
774

775
    task_eNB = task_eNB + ' (date; sudo rm -f ' + logfile_pcap_tmp_eNB + ' ; sudo -E tshark -i lo -s 65535 -a duration:' + str(timeout_cmd-10)+ ' -w ' + logfile_pcap_tmp_eNB+ ' ; sudo -E chown ' + user + ' ' + logfile_pcap_tmp_eNB + ' ; zip -j -9  ' + logfile_pcap_zip_eNB + ' ' + logfile_pcap_tmp_eNB + '   ) > ' + logfile_tshark_eNB + ' 2>&1 & \n '
Rohit Gupta's avatar
Rohit Gupta committed
776
777
    task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
    task_eNB = task_eNB + 'echo eNB_tshark_exec PID = $! \n'
778
779
780
781
782
783
784
785
786
787
788
789
790
791
    #terminate the eNB test case after timeout_cmd seconds
    task_eNB  = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
    #task_eNB  = task_eNB + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_eNB  = task_eNB + 'handle_ctrl_c' + '\n' 
    task_eNB  = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1  '
    write_file(logfile_task_eNB, task_eNB, mode="w")

    #task_eNB =  'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB 
    logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
    logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'    
    logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
    logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
792
793
    logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
794
    logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
795
796
797
798
799
800
801
802
803
804
805
806
807
808

    task_UE_compile = ' ( uname -a ; date \n'
    task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
    task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE_compile = task_UE_compile + 'source oaienv \n'
    task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
    task_UE_compile = task_UE_compile + 'env |grep OPENAIR  \n'
    task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if UE_compile_prog != "":
       task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
    task_UE_compile  = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
    write_file(logfile_task_UE_compile, task_UE_compile, mode="w")

    task_UE = ' ( uname -a ; date \n'
809
810
811
812
813
814
    task_UE = task_UE + 'array_exec_pid=()' + '\n'
    task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE = task_UE + 'source oaienv \n'
    task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
    task_UE = task_UE + 'env |grep OPENAIR  \n'
    if UE_pre_exec != "":
815
       task_UE  = task_UE +  ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
816
    if UE_main_exec != "":
817
       task_UE = task_UE + ' ( date;  ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword)  + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
818
819
820
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
    if UE_traffic_exec != "":
821
822
       cmd_traffic = UE_traffic_exec + ' ' + UE_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
823
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
824
       task_UE = task_UE + ' ( date;  ' + cmd_traffic + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
    #terminate the UE test case after timeout_cmd seconds
    task_UE  = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
    #task_UE  = task_UE + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_UE  = task_UE + 'handle_ctrl_c' + '\n' 
    task_UE  = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
    write_file(logfile_task_UE, task_UE, mode="w")
    #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE

    logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
    logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
    logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
    logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
    logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
    logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
    logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
843
844
    logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
845
    logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
846
847
848

    task_EPC_compile = ' ( uname -a ; date \n'
    task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
849
    task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + ' ; source oaienv \n'
Rohit Gupta's avatar
Rohit Gupta committed
850
851
852
853
854
855
856
857
858
859
    task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
    task_EPC_compile = task_EPC_compile +  'source BUILD/TOOLS/build_helper \n'
    if EPC_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args +  ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
    if HSS_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
    task_EPC_compile  = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' 
    write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
    
    task_EPC = ' ( uname -a ; date \n'
860
    task_EPC = task_EPC + ' export OPENAIRCN_TESTDIR=' + logdir_EPC + '\n'
861
    task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
862
    task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n'
863
864
    task_EPC = task_EPC +  'source BUILD/TOOLS/build_helper \n'
    if EPC_pre_exec != "":
865
       task_EPC  = task_EPC +  ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
866
    if HSS_main_exec !=  "":
867
       task_EPC  = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS  +  ' 2>&1   & \n'
868
869
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
870
    if EPC_main_exec !=  "":
871
       task_EPC  = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC  +  ' 2>&1   & \n'
872
873
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
874
    if EPC_traffic_exec !=  "":
875
876
       cmd_traffic = EPC_traffic_exec + ' ' + EPC_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
877
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
878
       task_EPC  = task_EPC + '( date; ' + cmd_traffic + ' ) > ' + logfile_traffic_EPC  +  ' 2>&1   & \n' 
879
880
881
882
883
884
885
886
887
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'  
       task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
    #terminate the EPC test case after timeout_cmd seconds   
    task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
    #task_EPC  = task_EPC + 'sleep ' +  str(timeout_cmd) + '\n'
    task_EPC  = task_EPC + 'handle_ctrl_c' '\n' 
    task_EPC  = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' 
    write_file(logfile_task_EPC, task_EPC, mode="w")
    
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
    #first we compile all the programs but only for run_0
    if run == 0:
       thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
       thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
       thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE_compile, False, timeout_thread) 
       if RRHMachine != '':
          thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH_compile, False, timeout_thread) 
       threads=[]
       threads.append(thread_eNB)
       threads.append(thread_UE)
       threads.append(thread_EPC)
       if RRHMachine != '':
         threads.append(thread_RRH)
       # Start new Threads
       thread_eNB.start()
       thread_UE.start()
       thread_EPC.start()
       if RRHMachine != '':
         thread_RRH.start()
       #Wait for all the compile threads to complete
       for t in threads:
         t.join()
Rohit Gupta's avatar
Rohit Gupta committed
910
911

    #Now we execute all the threads
912
913
914
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE, False, timeout_thread) 
Rohit Gupta's avatar
Rohit Gupta committed
915
916
    if RRHMachine != '':
        thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH, False, timeout_thread) 
917
918
919
920
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
Rohit Gupta's avatar
Rohit Gupta committed
921
922
    if RRHMachine != '':
        threads.append(thread_RRH)
923
924
925
926
927
    # Start new Threads

    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
Rohit Gupta's avatar
Rohit Gupta committed
928
929
    if RRHMachine != '':
        thread_RRH.start()
930
931
932
933
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()
    #Now we get the log files from remote machines on the local machine
Rohit Gupta's avatar
Rohit Gupta committed
934
    if RRHMachine != '':
935
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC, oai_RRH] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC, logdir_RRH], logdirOAI5GRepo)
Rohit Gupta's avatar
Rohit Gupta committed
936
    else:
937
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC], logdirOAI5GRepo)       
Rohit Gupta's avatar
Rohit Gupta committed
938
939
940
941
942
943
944
945
946
947
948
949
950
951
    logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
    logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'

    if UE_stop_script != "":
      cmd = ' ( uname -a ; date \n'
      cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
      cmd = cmd + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'
      cmd = cmd + UE_stop_script + '\n'
      cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' 
      write_file(logfile_UE_stop_script , cmd, mode="w")
      thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password  , cmd, False, timeout_thread)
      thread_UE.start()
      thread_UE.join()
   
Rohit Gupta's avatar
Rohit Gupta committed
952
953
954
955
956
957
958
959
960
961
962
963
964
    #Now we change the permissions of the logfiles to avoid some of them being with root permissions
    cmd = 'sudo -E chown -R ' + user + ' ' + logdir_eNB
    res= oai_eNB.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_eNB + "> in eNB machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_UE
    res= oai_UE.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_UE + "> in UE machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_EPC
    res= oai_EPC.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_EPC + "> in EPC machine..." + res

Rohit Gupta's avatar
Rohit Gupta committed
965
966
967
968
    if RRHMachine != '':
       cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_RRH
       res= oai_RRH.send_recv(cmd)
       print "Changing permissions of logdir <" + logdir_RRH + "> in RRH machine..." + res
969

970
    print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
971
    SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
972

973
    print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
974
    SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
975

976
    print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
977
    SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
978

Rohit Gupta's avatar
Rohit Gupta committed
979
980
981
    if RRHMachine != '':
       print "Copying files from RRHMachine : " + RRHMachine + "logdir_RRH = " + logdir_RRH
       SSHSessionWrapper(RRHMachine, user, None, password, logdir_RRH, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
982

Rohit Gupta's avatar
Rohit Gupta committed
983
    
984
    #Currently we only perform throughput tests
985
986
987
    tput_run_string=''
    result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
    tput_run_string = tput_run_string + tput_string
988
    run_result=run_result&result
989
    result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
990
    run_result=run_result&result
991
992
    tput_run_string = tput_run_string + tput_string
    result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
993
    run_result=run_result&result
994
    tput_run_string = tput_run_string + tput_string
Rohit Gupta's avatar
Rohit Gupta committed
995
    
996
    if run_result == 1:  
Rohit Gupta's avatar
Rohit Gupta committed
997
      run_result_string = ' RUN_'+str(run) + ' = PASS'
998
    else:
Rohit Gupta's avatar
Rohit Gupta committed
999
      run_result_string = ' RUN_'+str(run) + ' = FAIL'
1000
1001
1002
1003

    #If there is assertion, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"assertion\" " + logdir_local_testcase + " | cat " 
    cmd_out = subprocess.check_output ([cmd], shell=True)
1004
    if len(cmd_out) !=0 :
1005
1006
1007
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Assert)'

1008
1009
1010
    #If there is thread busy error, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"thread busy\" " + logdir_local_testcase + " | cat "
    cmd_out = subprocess.check_output ([cmd], shell=True)
1011
    if len(cmd_out) !=0:
1012
1013
1014
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(Thread_Busy)'

1015
1016
1017
1018
1019
1020
1021
    #If there is Segmentation fault, we mark the test case as failure as most likely eNB crashed
    cmd = "grep -ilr \"segmentation fault\" " + logdir_local_testcase + " | cat "
    cmd_out = subprocess.check_output ([cmd], shell=True)
    if len(cmd_out) !=0:
      run_result=0
      run_result_string = ' RUN_'+str(run) + ' = FAIL(SEGFAULT)'

1022
    run_result_string = run_result_string + tput_run_string
1023
1024
1025
1026

    test_result=test_result & run_result
    test_result_string=test_result_string + run_result_string

1027
1028
1029
    oai_eNB.disconnect()
    oai_UE.disconnect()
    oai_EPC.disconnect()
1030
    #We need to close the new ssh session that was created  
1031
1032
    #if index_eNBMachine == index_EPCMachine:
    #    oai_EPC.disconnect()
1033
1034
1035
  #Now we finalize the xml file of the test case
  end_time=time.time()
  duration= end_time - start_time
Rohit Gupta's avatar
Rohit Gupta committed
1036
  xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml'
1037
1038
1039
1040
  if test_result ==0: 
    result='FAIL'
  else:
    result = 'PASS'
1041
  xml="\n<testcase classname=\'"+ testcaseclass +  "\' name=\'" + testcasename + "."+tags +  "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n"
1042
1043
  write_file(xmlFile, xml, mode="w")

1044

Rohit Gupta's avatar
Rohit Gupta committed
1045
1046
1047
1048
# \brief This function searches if test case is present in list of test cases that need to be executed by user
# \param testcasename the test case to search for
# \param testcasegroup list that is passed from the arguments
# \param test_case_exclude list of test cases excluded from execution (specified in test_case_list.xml)
1049
def search_test_case_group(testcasename, testcasegroup, test_case_exclude):
1050
    
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
    if test_case_exclude != "":
       testcase_exclusion_list=test_case_exclude.split()
       for entry in testcase_exclusion_list:
          if entry.find('+') >=0:
            match = re.search(entry, testcasename)
            if match:
               print "\nSkipping test case as it is found in black list: " + testcasename
               return False
          else:
             match = entry.find(testcasename)
             if match >=0:
                print "\nSkipping test case as it is found in black list: " + testcasename
                return False
Rohit Gupta's avatar
Rohit Gupta committed
1064
    if testcasegroup == '':
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
         return True
    else:
      testcaselist = testcasegroup.split()
      for entry in testcaselist:
        if entry.find('+') >=0:
           match = re.search(entry, testcasename)
           if match:
              return True
        else:
           match = testcasename.find(entry)
           if match >=0:
Rohit Gupta's avatar
Rohit Gupta committed
1076
1077
             return True
    return False
1078

Rohit Gupta's avatar
Rohit Gupta committed
1079
1080
1081
1082
1083
1084
1085
# \brief thread that cleans up remote machines from pre-existing test case executions
# \param threadID number of thread (for book keeping)
# \param threadname name of thread (for book keeping)
# \param oai handler that can be used to execute programs on remote machines
# \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml)
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
1086
class oaiCleanOldProgramThread (threading.Thread):