run_exec_lte-softmodem_tests.py 124 KB
Newer Older
1
#! /usr/bin/python
2
3
4
5
6
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
Cedric Roux's avatar
Cedric Roux committed
7
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
22

23
# \author Rohit Gupta - Benoit ROBERT (benoit.robert@syrtem.com)
24
25
26
# \version 0.1
# @ingroup _test

27
28
29
30
# \Changelog
# 2016-11-18 : 
#   - Add progess bar during test execution update_progress()

31
32
33
34
35
36
37
38
39
40
41
import tempfile
import threading
import sys
import traceback
import wave
import os
import time
import datetime
import getpass
import math #from time import clock 
import xml.etree.ElementTree as ET
Rohit Gupta's avatar
Rohit Gupta committed
42
import re
43

44
45
#from dict2xml import dict2xml as xmlify

46
47
from colorama import Fore, Back, Style

48
49
import numpy as np

50
51
52
53
54
import log

from  openair import *

import paramiko
55

56
57
import subprocess
import commands
58
sys.path.append('/opt/ssh')
59
sys.path.append(os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/'))
60

61
from lib_autotest import *
62
63
import ssh
from ssh import SSHSession
Rohit Gupta's avatar
Rohit Gupta committed
64
import argparse
65

66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

# update_progress() : Displays or updates a console progress bar
## Accepts a float between 0 and 1. Any int will be converted to a float.
## A value under 0 represents a 'halt'.
## A value at 1 or bigger represents 100%
def update_progress(progress, prefix_string):
    barLength = 20 # Modify this to change the length of the progress bar
    status = ""
    #print "progress = "+ str(progress)
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress >= 1:
        progress = 1
        status = "Done...\r\n"
    block = int(round(barLength*progress))
    text = "\r"+prefix_string+" [{0}] {1}% {2}".format( "="*block + " "*(barLength-block), progress*100, status)
    sys.stdout.write(text)
    sys.stdout.flush()


def exit_prog(exit_val):
  print(Style.RESET_ALL),
  sys.exit(exit_val)

Rohit Gupta's avatar
Rohit Gupta committed
96
97
98
99
# \brief write a string to a file
# \param filename name of file
# \param string string to write
# \mode file opening mode (default=write)
100
101
102
103
def write_file(filename, string, mode="w"):
   text_file = open(filename, mode)
   text_file.write(string)
   text_file.close()
Rohit Gupta's avatar
Rohit Gupta committed
104
105
106
107
108
109
110
111
 
# \brief function to check if test case passed throughput test
# \param filename name of file which has throughput results (usually from iperf -s ...
# \param min_tput minimum throughput
# \param max_tuput maximum throughput
# \param average average throughput
# \param min_duration minimum duration of throughput
#The throughput values found in file must be higher than values from from arguments 2,3,4,5
112
113
#The function returns True if throughput conditions are saisfied else it returns fails
def tput_test(filename, min_tput, max_tput, average, min_duration):
114
   
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
   if os.path.exists(filename):
      with open (filename, "r") as myfile:
         data=myfile.read()
      p=re.compile('(\d*.\d*) Mbits/sec')
      array=p.findall(data)
      array = [ float(x) for x in array ]
      duration = array.__len__()
      if duration !=0:
        min_list = min(array) 
        max_list = max(array)
        average_list = np.mean(array)
      else:
        min_list = 0
        max_list = 0
        average_list=0
130
      tput_string=' ( '+ "min=%0.2f"  % min_list + ' Mbps / ' + "max=%0.2f" %  max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' 
131
      if (min_list >= min_tput and  max_list >= max_tput and average_list >= average and duration >= min_duration):
132
        return True , tput_string
133
      else:
134
        return False , tput_string
135
   else: 
136
      return False , tput_string
137

Rohit Gupta's avatar
Rohit Gupta committed
138
# \brief Convert string to float or return None if there is exception    
139
140
141
142
143
144
def try_convert_to_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

Rohit Gupta's avatar
Rohit Gupta committed
145
146
147
# \brief get throughput statistics from log file
# \param search_expr search expression found in test_case_list.xml file
# \param logfile_traffic logfile which has traffic statistics
148
149
def tput_test_search_expr (search_expr, logfile_traffic):
   result=0
150
   tput_string=''
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
   if search_expr !='':
       if search_expr.find('throughput_test')!= -1 :
          p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec')
          min_tput=p.findall(search_expr)
          if min_tput.__len__()==1:
             min_tput = min_tput[0]
          else:
             min_tput = None

          p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec')
          max_tput=p.findall(search_expr)
          if max_tput.__len__()==1:
             max_tput = max_tput[0]
          else:
             max_tput = None

          p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec')
          avg_tput=p.findall(search_expr)
          if avg_tput.__len__()==1:
             avg_tput=avg_tput[0]
          else:
             avg_tput = None

          p= re.compile('duration\s*=\s*(\d*.\d*)\s*s')
          duration=p.findall(search_expr)
          if duration.__len__()==1:
             duration = duration[0]
          else:
             duration = None
          
          min_tput = try_convert_to_float(min_tput)
          max_tput = try_convert_to_float(max_tput)
          avg_tput = try_convert_to_float(avg_tput)
          duration = try_convert_to_float(duration)
          
          if (min_tput != None and max_tput != None  and avg_tput != None  and duration != None ):
187
             result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration)
Rohit Gupta's avatar
Rohit Gupta committed
188
189
190
   else: 
      result=1

191
   return result, tput_string
192
      
Rohit Gupta's avatar
Rohit Gupta committed
193
194
195
196
197
198
199
# \brief function to copy files to/from remote machine
# \param username user with which to make sftp connection
# \param password password of user
# \param hostname host to connect
# \ports port of remote machine on which server is listening
# \paramList This is list of operations as a set {operation: "get/put", localfile: "filename", remotefile: "filename"
# \param logfile Ignored currently and set once at the beginning of program
Rohit Gupta's avatar
Rohit Gupta committed
200
201
202
def sftp_module (username, password, hostname, ports, paramList,logfile): 
   #localD = localfile
   #remoteD = remotefile
203
204
205
206
   #fd, paramiko_logfile  = tempfile.mkstemp()
   #res = os.close(fd )
   #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless
   error = ""
Rohit Gupta's avatar
Rohit Gupta committed
207
   #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure
208
   transport = paramiko.Transport((hostname, ports))
Rohit Gupta's avatar
Rohit Gupta committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
   transport.connect(username = username, password = password)
   sftp = paramiko.SFTPClient.from_transport(transport)
   #  index =0 
   for param in paramList:
      try:
        operation = param["operation"] 
        localD = param["localfile"]
        remoteD = param["remotefile"]
        if operation == "put":
          sftp.put(remotepath=remoteD, localpath=localD)
        elif operation == "get":
          sftp.get(remotepath=remoteD, localpath=localD)
        else :
          print "sftp_module: unidentified operation:<" + operation + "> Exiting now"
          print "hostname = " + hostname
          print "ports = " + ports
          print "localfile = " + localD
          print "remotefile = " + remoteD
          print "operation = " + operation
          sys.exit()
      except Exception, e:
         error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
         error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n'  
         error = error + traceback.format_exc()
233
         print error
Rohit Gupta's avatar
Rohit Gupta committed
234
235
236

   sftp.close()
   transport.close() 
237
238
239
240
241
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ')
   res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n')
   write_file(logfile, error, "a")
   res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n')

Rohit Gupta's avatar
Rohit Gupta committed
242
243
244
# \brief bash script stub put at the end of scripts to terminate it 
# \param timeout_cmd terminate script after timeout_cmd seconds
# \param terminate_missing_procs if True terminate all the processes launched by script if one of them terminates prematurely (due to error)
245
def finalize_deploy_script (timeout_cmd, terminate_missing_procs='False'):
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
  cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n'
  if terminate_missing_procs == 'True':
    cmd = cmd +  """
    #The code below checks if one the processes launched in background has crashed.
    #If it does, then the code below terminates all the child processes created by this script
    declare -i wakeup_interval=1
    declare -i step=0
    echo \"Array pid =  ${array_exec_pid[@]}\"
    while [ "$step" -lt "$timeout_cmd" ]
      do
       declare -i break_while_loop=0
       #Iterate over each process ID in array_exec_pid
       for i in "${array_exec_pid[@]}"
       do
        numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w`
        echo "PID = $i, numchild = $numchild"
        if  [ "$numchild" -eq "0" ] ; then
            echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes "
            break_while_loop=1
            break
        fi
     done
    if  [ "$break_while_loop" -eq "1" ] ; then
             break
    fi
    step=$(( step + wakeup_interval ))
    sleep $wakeup_interval
    done
    echo "Final time step (Duration of test case) = $step "
Rohit Gupta's avatar
Rohit Gupta committed
275
    date
276
277
278
    """
  else:
    #We do not terminate the script if one of the processes has existed prematurely
Rohit Gupta's avatar
Rohit Gupta committed
279
    cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date  \n'
280
281
282
  
  return cmd

Rohit Gupta's avatar
Rohit Gupta committed
283
284
285
286
287
# \brief run python script and update config file params of test case
# \param oai module with already open connection
# \param config_string config string taken from xml file
# \param logdirRepo directory of remote repository
# \param python_script python script location
288
def update_config_file(oai, config_string, logdirRepo, python_script):
289
  cmd=""
290
291
292
293
294
295
296
297
  if config_string :
    stringArray = config_string.splitlines()
    #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py'
    for string in stringArray:
       #split the string based on space now
       string1=string.split()
       cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + '  ' + string1[1] +  ' '+ string1[2] + '\n'
       #cmd = cmd + 'perl -p -i  -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\'   ' + logdirRepo + '/' +string1[0] + '\n'
298
299
  return cmd
  #result = oai.send_recv(cmd)
300

Rohit Gupta's avatar
Rohit Gupta committed
301
302
303
304
305
306
307
308
309
# \brief thread safe sshsession wrapper due to occasional connection issues with ssh
# \param machine name of machine
# \param username user login for remote machine
# \param key_file file name which has keys to enable passwordless login
# \param password password for remote machine
# \param logdir_remote remote directory
# \param logdir_local_base local directory
# \param operation operation to perform (get_all, put_all) transfers recursively for directories
def SSHSessionWrapper(machine, username, key_file, password, logdir_remote, logdir_local_base, operation):
310
  max_tries = 10
311
312
313
  i=0
  while i <= max_tries:
    i = i +1
314
315
    try:
       ssh = SSHSession(machine , username, key_file, password)
316
       if operation == "get_all":
Rohit Gupta's avatar
Rohit Gupta committed
317
          ssh.get_all(logdir_remote , logdir_local_base)
318
       elif operation == "put_all":
Rohit Gupta's avatar
Rohit Gupta committed
319
          ssh.put_all(logdir_local_base, logdir_remote )
320
321
322
       else:
          print "Error: Uknown operation in SSHSessionWrapper. Exiting now..."
          sys.exit(1)
323
324
325
326
       break 
    except Exception, e:
       error=''
       error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
Rohit Gupta's avatar
Rohit Gupta committed
327
       error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote + '\n logdir_local_base = ' + logdir_local_base 
328
329
       error = error + traceback.format_exc()
       print error
330
       print " Retrying again in 1 seconds"
Rohit Gupta's avatar
Rohit Gupta committed
331
       time.sleep(1)
332
       print "Continuing ..."
333
334
335
       if i ==max_tries:
          print "Fatal Error: Max no of retries reached. Exiting now..."
          sys.exit(1)
336

337
338

 
Rohit Gupta's avatar
Rohit Gupta committed
339
340
341
342
343
# \briefFunction to clean old programs that might be running from earlier execution
# \param oai - parameter for making connection to machine
# \parm programList list of programs that must be terminated before execution of any test case 
# \param CleanUpAluLteBox program to terminate AlU Bell Labs LTE Box
# \param ExmimoRfStop String to stop EXMIMO card (specified in test_case_list.xml)
344
def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo):
345
  cmd = 'killall -9 ' + programList
346
  result = oai.send(cmd, True)
347
348
349
350
#  print "\t\t > "+cmd
#  print "\t\t < "+result

#  print "Killing old programs on ..." + result
351
352
  programArray = programList.split()
  programListJoin = '|'.join(programArray)
353
  cmd = " ( date ;echo \"Starting cleaning old programs.. \" ; dmesg|tail ; echo \"Current disk space.. \" ; df -h )>& " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
354
  result=oai.send_recv(cmd)
355
356
357
#  print "\t\t > "+cmd
#  print "\t\t < "+result

358
359
360
361
  cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\''
  #result = oai.send_recv(cmd)
  #print result
  result = oai.send_expect_false(cmd, 'Match found', False)
362
363
364
365
#  print "\t\t > "+cmd
#  print "\t\t < "+result

#  print "Looking for old programs..." + result
366
  res=oai.send_recv(CleanUpAluLteBox, True)
367
368
#  print "\t\t > "+CleanUpAluLteBox
#  print "\t\t < "+res
369
  cmd= " echo \"Starting EXmimoRF Stop... \"  >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
370
371
372
  result = oai.send_recv(cmd)
#  print "\t\t > "+cmd
#  print "\t\t < "+result
373
  cmd  = "( " + "cd " + logdirOAI5GRepo + " ; source oaienv ;  "  +  ExmimoRfStop + " ) >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync "
374
#  print "cleanoldprograms cmd = " + cmd
375
  res=oai.send_recv(cmd, False, timeout=600)
376
377
378
#  print "\t\t > "+cmd
#  print "\t\t < "+res

379
  cmd= " echo \"Stopping EXmimoRF Stop... \" >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1  ; sync ";
380
381
382
  result = oai.send_recv(cmd)
#  print "\t\t > "+cmd
#  print "\t\t < "+result
383

384
385
386
  #res = oai.send_recv(ExmimoRfStop, False)
  cmd = " ( date ;echo \"Finished cleaning old programs.. \" ; dmesg | tail)>> $HOME/.oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync"
  res=oai.send_recv(cmd)
387
388
#  print "\t\t > "+cmd
#  print "\t\t < "+res
389

Rohit Gupta's avatar
Rohit Gupta committed
390
391
392
393
394
395
396
397
398
# \brief Class thread to launch a generic command on remote machine
# \param threadID number of thread (for book keeping)
# \param threadname string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param username username with which to login
# \param password password with which to login
# \param cmd command as a string to run on remote machine
# \parma sudo if True sudo is set
# \param timeout timeout of command in seconds 
399
class oaiThread (threading.Thread):
400
    def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout):
401
402
        threading.Thread.__init__(self)
        self.threadID = threadID
403
404
405
406
        self.threadname = threadname
        self.machine = machine
        self.username = username
        self.password = password
407
408
409
410
        self.cmd = cmd
        self.sudo = sudo
        self.timeout = timeout
    def run(self):
411
412
        try:
          oai = openair('localdomain',self.machine)
413
          oai.connect(self.username, self.password)
414
          # print "Starting " + self.threadname + " on machine " + self.machine
415
          result = oai.send_recv(self.cmd, self.sudo, self.timeout)
416
417
          #print "result = " + result
          #print "Exiting " + self.threadname
418
419
420
421
          oai.disconnect()
        except Exception, e:
           error=''
           error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
422
           error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + str(self.timeout) + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) +  '\n username = ' + self.username + '\n'  
423
424
425
           error = error + traceback.format_exc()
           print error

426

Rohit Gupta's avatar
Rohit Gupta committed
427
428
429
430
431
432
433
434
435
436
437
# \brief This class runs test cases with class {execution, compilatation}
# \param threadID number of thread (for book keeping)
# \param name string of threadname (for book keeping)
# \param machine machine name on which to run the command
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param testcasename name of test case to run on remote machine
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param user username with which to login
# \param password password with which to login
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
438
class testCaseThread_generic (threading.Thread):
439
   def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop):
Rohit Gupta's avatar
Rohit Gupta committed
440
       threading.Thread.__init__(self)
441
       self.threadID = threadID
Rohit Gupta's avatar
Rohit Gupta committed
442
443
444
445
446
       self.name = name
       self.testcasename = testcasename
       self.timeout = timeout
       self.machine = machine
       self.logdirOAI5GRepo = logdirOAI5GRepo
447
448
449
       self.oldprogramList = oldprogramList
       self.CleanupAluLteBox = CleanupAluLteBox
       self.password=password
Rohit Gupta's avatar
Rohit Gupta committed
450
       self.ExmimoRfStop = ExmimoRfStop
451
       self.user = user
Rohit Gupta's avatar
Rohit Gupta committed
452
453
454
455
456
   def run(self):
     try:
       mypassword=''
       #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
       addpass = 'echo \'' + mypassword + '\' | '
457
       #user = getpass.getuser()
Rohit Gupta's avatar
Rohit Gupta committed
458
       print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) 
459
       oai = openair('localdomain',self.machine)
460
461
       oai.connect(self.user, self.password)
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
462
463
464
       logdir_local = os.environ.get('OPENAIR_DIR')
       logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename
       logdir_local_base = logdir_local +'/cmake_targets/autotests/log/'
465
466
467
468
469
470
471
472
473
474
475
476
477
       logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename
       logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/'
       logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log'
       logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log'
       #print "logdir_local_testcase = " + logdir_local_testcase
       #print "logdir_remote_testcase = " + logdir_remote_testcase
       #if os.path.exists(logdir_local_testcase) == True :
       #    os.removedirs(logdir_local_testcase)
       #os.mkdir(logdir_local_testcase)
       os.system("rm -fr " + logdir_local_testcase )
       os.system("mkdir -p " +  logdir_local_testcase)
       cmd = "mkdir -p " + logdir_remote_testcase
       res = oai.send_recv(cmd, False, self.timeout) 
Rohit Gupta's avatar
Rohit Gupta committed
478
       #print "res = " + res
479
480

       cmd =  "( cd " +  self.logdirOAI5GRepo + " \n "
Rohit Gupta's avatar
Rohit Gupta committed
481
       cmd = cmd + "source oaienv \n"
482
483
484
485
486
487
       cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'"
       cmd = cmd + " ) >& "   + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase +  "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase 
      
       #print "cmd = " + cmd
       res = oai.send_recv(cmd, False, self.timeout) 
       #print "res = " + res
Rohit Gupta's avatar
Rohit Gupta committed
488
       #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res
489
490
       write_file(logfile_task_testcasename, cmd, mode="w")
       #Now we copy all the remote files
491
492
       #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password)
       #ssh.get_all(logdir_remote_testcase , logdir_local_base)
493
       SSHSessionWrapper(self.machine, self.user, None, self.password, logdir_remote_testcase, logdir_local_base, "get_all")
494
       print "Finishing test case : " + self.testcasename + " On machine " + self.machine
495
       #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop)
496
       #oai.kill(user,mypassword)
497
498
499
       oai.disconnect()
     except Exception, e:
         error=''
500
         error = error + ' In Class = testCaseThread_generic,  function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
501
         error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo +  '\n' + '\n timeout = ' + str(self.timeout)  + '\n user = ' + self.user
502
         error = error + traceback.format_exc()
503
         print error
504
505
         print "Continuing with next test case..."
         #sys.exit()
Rohit Gupta's avatar
Rohit Gupta committed
506
507


508
       
Rohit Gupta's avatar
Rohit Gupta committed
509
510
511
# \bried function to run a command as a sudo
# \param cmd command as a string
# \param password password to be supplied   
512
513
514
def addsudo (cmd, password=""):
  cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' '
  return cmd
515

Rohit Gupta's avatar
Rohit Gupta committed
516
517
518
519
520
521
522
523
524
525
526
# \brief handler for executing test cases (compilation, execution)
# \param name of testcase
# \param threadListGeneric list of threads which are already running on remote machines
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed
# \param MachineList list of all machines on which generic test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param timeout timeout of command in seconds
# \param ExmimoRfStop command to stop EXMIMO Card
527
def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, user, password, CleanupAluLteBox,timeout, ExmimoRfStop):
528
529
530
  try:
    mypassword=password
    MachineListFree=[]
Rohit Gupta's avatar
Rohit Gupta committed
531
    threadListNew=[]
532
533
    while MachineListFree.__len__() == 0 :
       MachineListBusy=[]
Rohit Gupta's avatar
Rohit Gupta committed
534
535
       MachineListFree=[]
       threadListNew=[]
536
537
538
       #first we need to find the list of free machines that we could run our test case
       if threadListGeneric.__len__() ==0 :
       #This means no thread is started yet
539
          MachineListFree = MachineList[:]
540
541
542
543
       else :
          for param in threadListGeneric :
             thread_id = param["thread_id"]
             machine = param["Machine"]
544
             testcasenameold = param["testcasename"]
545
             thread_id.join(1)
546
             if thread_id.isAlive() == True:
Rohit Gupta's avatar
Rohit Gupta committed
547
548
                threadListNew.append(param)
                print "thread_id is alive: testcasename: " + testcasenameold +  " on machine "+ machine
549
550
                if machine not in MachineListBusy:
                   MachineListBusy.append(machine)
551
             else :
Rohit Gupta's avatar
Rohit Gupta committed
552
553
554
555
                print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine
                #threadListGeneric.remove(param)
                #if machine not in MachineListFree:
                #   MachineListFree.append(machine)
556
557
558
559
560
       #Now we check if there is at least one free machine
       MachineListFree = MachineList[:]
       for machine in MachineListBusy:
          if machine in MachineListFree:
            MachineListFree.remove(machine)
Rohit Gupta's avatar
Rohit Gupta committed
561
562
563
       print "MachineListFree = " + ','.join(MachineListFree)
       print "MachineListBusy = " + ','.join(MachineListBusy)
       print "MachineList = " + ','.join(MachineList)
564
    machine = MachineListFree[0]
565
    thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop)
566
567
    param={"thread_id":thread, "Machine":machine, "testcasename":testcasename}
    thread.start()
Rohit Gupta's avatar
Rohit Gupta committed
568
569
    threadListNew.append(param)
    return threadListNew
570
  except Exception, e:
571
572
     error=''
     error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: '  + str(e.__class__) + " : " + str( e)
573
     error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) +  '\n' + 'user = ' + user
574
575
     error = error + traceback.format_exc()
     print error
576
577
     print "Continuing..."
     #sys.exit(1)
578

Rohit Gupta's avatar
Rohit Gupta committed
579
580
581
# \brief Blocking wait for all threads related to generic testcase execution, class (compilation and execution)
# \param threadListGeneric list of threads which are running on remote machines
# \param timeout time to wait on threads in seconds
582
def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1):
583
   threadListGenericNew=[]
584
585
586
587
588
589
   for param in threadListGeneric:
      thread_id = param["thread_id"]
      machine = param["Machine"]
      testcasenameold = param["testcasename"]
      thread_id.join(timeout)
      if thread_id.isAlive() == True:
590
         threadListGenericNew.append(param)
591
592
593
594
595
         print "thread_id on machine: " + machine + "  is still alive: testcasename: " + testcasenameold
         print " Exiting now..."
         sys.exit(1)
      else:
         print "thread_id on machine: " + machine + "  is stopped: testcasename: " + testcasenameold
596
597
         #threadListGeneric.remove(param)
   return threadListGenericNew
598

Rohit Gupta's avatar
Rohit Gupta committed
599
600
601
602
603
604
605
606
607
608
609

# \brief handler for executing test cases (lte-softmodem)
# \param testcase name of testcase
# \param oldprogramList list of programs which must be terminated before running a test case
# \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed
# \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed
# \param MachineList list of all machines on which test cases can be run
# \param user username with which to login
# \param password password with which to login
# \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml)
# \param ExmimoRfStop command to stop EXMIMO Card
610
# \param nruns_lte-softmodem global parameter to override number of runs (nruns) within the test case
611
def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem,  GitOAI5GRepoBranch,  GitOpenaircnRepoBranch,timeout_cmd):
612
613
614
615
616
617
  #We ignore the password sent to this function for secuirity reasons for password present in log files
  #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt
  # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL
  mypassword=''
  #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E '
  addpass = 'echo \'' + mypassword + '\' | '
618
  #user = getpass.getuser()
619
  testcasename = testcase.get('id')
620
  testcaseclass = testcase.findtext('class',default='')
621
622
  if timeout_cmd == '':
     timeout_cmd = testcase.findtext('TimeOut_cmd',default='')
623
624
625
  timeout_cmd = int(float(timeout_cmd))
  #Timeout_thread is more than that of cmd to have room for compilation time, etc
  timeout_thread = timeout_cmd + 300 
626
627
628
629
  if nruns_lte_softmodem == '':
    nruns = testcase.findtext('nruns',default='')
  else:
    nruns = nruns_lte_softmodem
630
  nruns = int(float(nruns))
631
  tags = testcase.findtext('tags',default='')
Rohit Gupta's avatar
Rohit Gupta committed
632
633

  RRHMachine = testcase.findtext('RRH',default='')
Rohit Gupta's avatar
Rohit Gupta committed
634
  RRH_config_file = testcase.findtext('RRH_config_file',default='')
Rohit Gupta's avatar
Rohit Gupta committed
635
636
637
638
639
640
  RRH_compile_prog = testcase.findtext('RRH_compile_prog',default='')
  RRH_compile_prog_args = testcase.findtext('RRH_compile_prog_args',default='')
  RRH_pre_exec = testcase.findtext('RRH_pre_exec',default='')
  RRH_pre_exec_args = testcase.findtext('RRH_pre_exec_args',default='')
  RRH_main_exec = testcase.findtext('RRH_main_exec',default='')
  RRH_main_exec_args = testcase.findtext('RRH_main_exec_args',default='')
641
  RRH_terminate_missing_procs = testcase.findtext('RRH_terminate_missing_procs',default='False')
642
  RRH_branch = testcase.findtext('RRH_branch',default=GitOAI5GRepoBranch)
Rohit Gupta's avatar
Rohit Gupta committed
643

644
645
646
647
648
649
650
651
652
653
  eNBMachine = testcase.findtext('eNB',default='')
  eNB_config_file = testcase.findtext('eNB_config_file',default='')
  eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='')
  eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='')
  eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='')
  eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='')
  eNB_main_exec = testcase.findtext('eNB_main_exec',default='')
  eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='')
  eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='')
  eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='')
654
  eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='False')
655
  eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','')
656
  if re.compile('\w+').match(eNB_search_expr_true) != None:
657
      eNB_search_expr_true = eNB_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's' 
658
  eNB_branch = testcase.findtext('eNB_branch',default=GitOAI5GRepoBranch)
659
660
661
662
663
664
665
666
667
668
669

  UEMachine = testcase.findtext('UE',default='')
  UE_config_file = testcase.findtext('UE_config_file',default='')
  UE_compile_prog = testcase.findtext('UE_compile_prog',default='')
  UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='')
  UE_pre_exec = testcase.findtext('UE_pre_exec',default='')
  UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='')
  UE_main_exec = testcase.findtext('UE_main_exec',default='')
  UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='')
  UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='')
  UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='')
670
  UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='False')
671
  UE_search_expr_true = testcase.findtext('UE_search_expr_true','')
Rohit Gupta's avatar
Rohit Gupta committed
672
  UE_stop_script =  testcase.findtext('UE_stop_script','')
673
  if re.compile('\w+').match(UE_search_expr_true) != None:
674
      UE_search_expr_true = UE_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
675
  UE_branch = testcase.findtext('UE_branch',default=GitOAI5GRepoBranch)
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691

  EPCMachine = testcase.findtext('EPC',default='')
  EPC_config_file = testcase.findtext('EPC_config_file',default='')
  EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='')
  EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='')
  HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='')
  HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='')
  
  EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='')
  EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='')  
  EPC_main_exec= testcase.findtext('EPC_main_exec',default='')
  EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='')  
  HSS_main_exec= testcase.findtext('HSS_main_exec',default='')
  HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='')  
  EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='')
  EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='')
692
  EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='False')
693
  EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','')
694
695
  EPC_branch = testcase.findtext('EPC_branch',default=GitOpenaircnRepoBranch)

696
  if re.compile('\w+').match(EPC_search_expr_true) != None:
697
     EPC_search_expr_true = EPC_search_expr_true + '  duration=' + str(timeout_cmd-90) + 's'
698
699
700
701

  index_eNBMachine = MachineList.index(eNBMachine)
  index_UEMachine = MachineList.index(UEMachine)
  index_EPCMachine = MachineList.index(EPCMachine)
Rohit Gupta's avatar
Rohit Gupta committed
702
  cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
703
704
  oai_eNB = openair('localdomain', eNBMachine)
  oai_eNB.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
705
  res= oai_eNB.send_recv(cmd)
706
707
  oai_UE = openair('localdomain', UEMachine)
  oai_UE.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
708
  res = oai_eNB.send_recv(cmd)
709
710
  oai_EPC = openair('localdomain', EPCMachine)
  oai_EPC.connect(user, password)
Rohit Gupta's avatar
Rohit Gupta committed
711
  res = oai_eNB.send_recv(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
712
713
714
715
716
717
  if RRHMachine != '':
    cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR'
    index_RRHMachine = MachineList.index(RRHMachine)
    oai_RRH = openair('localdomain', RRHMachine)
    oai_RRH.connect(user, password)
    res= oai_RRH.send_recv(cmd)
718
719
720
  #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
  #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop)
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
  logdir_local = os.environ.get('OPENAIR_DIR')
  if logdir_local is None:
     print "Environment variable OPENAIR_DIR not set correctly"
     sys.exit()
   
  #Make the log directory of test case
  #cmd = 'mkdir -p ' + logdir_eNB
  #result = oai_eNB.send_recv(cmd)
  #cmd = 'mkdir -p ' +  logdir_UE
  #result = oai_UE.send_recv(cmd)
  #cmd = 'mkdir -p ' + logdir_EPC
  #result = oai_EPC.send_recv(cmd)
  
  print "Updating the config files for ENB/UE/EPC..."
  #updating the eNB/UE/EPC configuration file from the test case 
  #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo)
  #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo)
  #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo)
739
740
741
  test_result=1
  test_result_string=''
  start_time=time.time()
742
  for run in range(0,nruns):
743
744
    run_result=1
    run_result_string=''
745
    logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
Rohit Gupta's avatar
Rohit Gupta committed
746
    logdir_RRH = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
747
748
749
750
    logdir_UE =  logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run)
    logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run)
    #Make the log directory of test case
Rohit Gupta's avatar
Rohit Gupta committed
751
    if RRHMachine != '':
752
      cmd = 'rm -fr ' + logdir_RRH + ' ; mkdir -p ' + logdir_RRH 
Rohit Gupta's avatar
Rohit Gupta committed
753
      result = oai_RRH.send_recv(cmd)
754
755
756
757
758
759
760
761
    cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB
    result = oai_eNB.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' +  logdir_UE
    result = oai_UE.send_recv(cmd)
    cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC
    result = oai_EPC.send_recv(cmd)
    cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase
    result = os.system(cmd)
Rohit Gupta's avatar
Rohit Gupta committed
762
763
764
765
766
767
768
769
770
771

    if RRHMachine != '':
       logfile_compile_RRH = logdir_RRH + '/RRH_compile' + '_' + str(run) + '_.log'
       logfile_exec_RRH = logdir_RRH + '/RRH_exec' + '_' + str(run) + '_.log'
       logfile_pre_exec_RRH = logdir_RRH + '/RRH_pre_exec' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile_out = logdir_RRH + '/RRH_task_compile_out' + '_' + str(run) + '_.log'
       logfile_task_RRH_compile = logdir_local_testcase + '/RRH_task_compile' + '_' + str(run) + '_.log'
       logfile_task_RRH_out = logdir_RRH + '/RRH_task_out' + '_' + str(run) + '_.log'
       logfile_task_RRH = logdir_local_testcase + '/RRH_task' + '_' + str(run) + '_.log'
       task_RRH_compile = ' ( uname -a ; date \n'
772
773
       task_RRH_compile = task_RRH_compile + 'cd ' + logdirOAI5GRepo + '; git reset --hard HEAD ; git checkout ' + RRH_branch + ' ; source oaienv   \n' 
       task_RRH_compile = task_RRH_compile + ' source cmake_targets/tools/build_helper \n'
Rohit Gupta's avatar
Rohit Gupta committed
774
       task_RRH_compile = task_RRH_compile + 'env |grep OPENAIR  \n'
Rohit Gupta's avatar
Rohit Gupta committed
775
       task_RRH_compile = task_RRH_compile + update_config_file(oai_RRH, RRH_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
Rohit Gupta's avatar
Rohit Gupta committed
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
       if RRH_compile_prog != "":
         task_RRH_compile  = task_RRH_compile +  ' ( ' + RRH_compile_prog + ' '+ RRH_compile_prog_args + ' ) > ' + logfile_compile_RRH + ' 2>&1 \n'
       task_RRH_compile =  task_RRH_compile + ' date ) > ' + logfile_task_RRH_compile_out + ' 2>&1  '
       write_file(logfile_task_RRH_compile, task_RRH_compile, mode="w")

       task_RRH = ' ( uname -a ; date \n'
       task_RRH = task_RRH + ' export OPENAIR_TESTDIR=' + logdir_RRH + '\n'
       task_RRH = task_RRH + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
       task_RRH = task_RRH + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

       if RRH_pre_exec != "":
          task_RRH  = task_RRH +  ' ( date; ' + RRH_pre_exec + ' '+ RRH_pre_exec_args + ' ) > ' + logfile_pre_exec_RRH + ' 2>&1 \n'
       if RRH_main_exec != "":
          task_RRH = task_RRH + ' ( date; ' + addsudo(RRH_main_exec + ' ' + RRH_main_exec_args, mypassword) + ' ) > ' + logfile_exec_RRH + ' 2>&1 & \n'
          task_RRH = task_RRH + 'array_exec_pid+=($!) \n'
          task_RRH = task_RRH + 'echo eNB_main_exec PID = $! \n'
       #terminate the eNB test case after timeout_cmd seconds
       task_RRH  = task_RRH + finalize_deploy_script (timeout_cmd, RRH_terminate_missing_procs) + ' \n'
       task_RRH  = task_RRH + 'handle_ctrl_c' + '\n' 
       task_RRH  = task_RRH + ' ) > ' + logfile_task_RRH_out + ' 2>&1  '
       write_file(logfile_task_RRH, task_RRH, mode="w")
797
798
799
800
801
    
    logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log'
    logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
802
803
    logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log'
804
805
    logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log'
    logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
806
    logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
807
808
    logfile_tshark_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.log'
    logfile_pcap_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap'
809
    logfile_pcap_zip_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap.zip'
Rohit Gupta's avatar
Rohit Gupta committed
810
    logfile_pcap_tmp_eNB = '/tmp/' + '/eNB_tshark' + '_' + str(run) + '_.pcap'
Rohit Gupta's avatar
Rohit Gupta committed
811
812

    task_eNB_compile = ' ( uname -a ; date \n'
813
814
    task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + '; git reset --hard HEAD ; git checkout ' + eNB_branch + ' ; source oaienv   \n' 
    task_eNB_compile = task_eNB_compile + ' source cmake_targets/tools/build_helper \n'
Rohit Gupta's avatar
Rohit Gupta committed
815
816
    task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR  \n'
    task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
817
    if eNB_compile_prog != "":
Rohit Gupta's avatar
Rohit Gupta committed
818
819
820
821
822
       task_eNB_compile  = task_eNB_compile +  ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n'
    task_eNB_compile =  task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1  '
    write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w")

    task_eNB = ' ( uname -a ; date \n'
823
    task_eNB = task_eNB + ' export OPENAIR_TESTDIR=' + logdir_eNB + '\n'
Rohit Gupta's avatar
Rohit Gupta committed
824
825
826
    task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
    task_eNB = task_eNB + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'

827
    if eNB_pre_exec != "":
828
       task_eNB  = task_eNB +  ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n'
829
    if eNB_main_exec != "":
830
       task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n'
831
832
833
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n'
    if eNB_traffic_exec != "":
834
835
       cmd_traffic = eNB_traffic_exec + ' ' + eNB_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
836
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
837
       task_eNB = task_eNB + ' (date;  ' + cmd_traffic + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n'
838
839
       task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
       task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n'
Rohit Gupta's avatar
Rohit Gupta committed
840

841
    task_eNB = task_eNB + ' (date; sudo rm -f ' + logfile_pcap_tmp_eNB + ' ; sudo -E tshark -i lo -s 65535 -a duration:' + str(timeout_cmd-10)+ ' -w ' + logfile_pcap_tmp_eNB+ ' ; sudo -E chown ' + user + ' ' + logfile_pcap_tmp_eNB + ' ; zip -j -9  ' + logfile_pcap_zip_eNB + ' ' + logfile_pcap_tmp_eNB + '   ) > ' + logfile_tshark_eNB + ' 2>&1 & \n '
Rohit Gupta's avatar
Rohit Gupta committed
842
843
    task_eNB = task_eNB + 'array_exec_pid+=($!) \n'
    task_eNB = task_eNB + 'echo eNB_tshark_exec PID = $! \n'
844
845
846
847
848
849
850
851
852
853
854
855
856
857
    #terminate the eNB test case after timeout_cmd seconds
    task_eNB  = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n'
    #task_eNB  = task_eNB + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_eNB  = task_eNB + 'handle_ctrl_c' + '\n' 
    task_eNB  = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1  '
    write_file(logfile_task_eNB, task_eNB, mode="w")

    #task_eNB =  'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB 
    logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log'
    logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log'
    logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log'    
    logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log'
    logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
858
859
    logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
860
    logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
861
862
863

    task_UE_compile = ' ( uname -a ; date \n'
    task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n'
864
    task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '; git reset --hard HEAD ; git checkout ' + UE_branch + ' ; source oaienv   \n' 
Rohit Gupta's avatar
Rohit Gupta committed
865
866
867
868
869
870
871
872
873
    task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n'
    task_UE_compile = task_UE_compile + 'env |grep OPENAIR  \n'
    task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n'
    if UE_compile_prog != "":
       task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n'
    task_UE_compile  = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 '
    write_file(logfile_task_UE_compile, task_UE_compile, mode="w")

    task_UE = ' ( uname -a ; date \n'
874
875
876
877
878
879
    task_UE = task_UE + 'array_exec_pid=()' + '\n'
    task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n'  
    task_UE = task_UE + 'source oaienv \n'
    task_UE = task_UE + 'source cmake_targets/tools/build_helper \n'
    task_UE = task_UE + 'env |grep OPENAIR  \n'
    if UE_pre_exec != "":
880
       task_UE  = task_UE +  ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n'
881
    if UE_main_exec != "":
882
       task_UE = task_UE + ' ( date;  ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword)  + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n'
883
884
885
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_main_exec PID = $! \n'
    if UE_traffic_exec != "":
886
887
       cmd_traffic = UE_traffic_exec + ' ' + UE_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
888
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
889
       task_UE = task_UE + ' ( date;  ' + cmd_traffic + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n'
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
       task_UE = task_UE + 'array_exec_pid+=($!) \n'
       task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n'
    #terminate the UE test case after timeout_cmd seconds
    task_UE  = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n'
    #task_UE  = task_UE + 'sleep ' +  str(timeout_cmd) + ' \n'
    task_UE  = task_UE + 'handle_ctrl_c' + '\n' 
    task_UE  = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 '
    write_file(logfile_task_UE, task_UE, mode="w")
    #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE

    logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log'
    logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log'
    logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log'
    logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log'
    logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log'
    logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log'
    logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log'
    logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
908
909
    logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log'
    logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log'
Rohit Gupta's avatar
Rohit Gupta committed
910
    logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' 
Rohit Gupta's avatar
Rohit Gupta committed
911
912
913

    task_EPC_compile = ' ( uname -a ; date \n'
    task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n'
914
    task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + '; git reset --hard HEAD ; git checkout ' + EPC_branch + ' ; source oaienv   \n' 
Rohit Gupta's avatar
Rohit Gupta committed
915
916
917
918
919
920
921
922
923
924
    task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n'
    task_EPC_compile = task_EPC_compile +  'source BUILD/TOOLS/build_helper \n'
    if EPC_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args +  ' ) > ' + logfile_compile_EPC + ' 2>&1 \n'
    if HSS_compile_prog != "":
       task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n'
    task_EPC_compile  = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' 
    write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w")
    
    task_EPC = ' ( uname -a ; date \n'
925
    task_EPC = task_EPC + ' export OPENAIRCN_TESTDIR=' + logdir_EPC + '\n'
926
    task_EPC = task_EPC + 'array_exec_pid=()' + '\n'
927
    task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n'
928
929
    task_EPC = task_EPC +  'source BUILD/TOOLS/build_helper \n'
    if EPC_pre_exec != "":
930
       task_EPC  = task_EPC +  ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n'
931
    if HSS_main_exec !=  "":
932
       task_EPC  = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS  +  ' 2>&1   & \n'
933
934
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n'
935
    if EPC_main_exec !=  "":
936
       task_EPC  = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC  +  ' 2>&1   & \n'
937
938
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'
       task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n'
939
    if EPC_traffic_exec !=  "":
940
941
       cmd_traffic = EPC_traffic_exec + ' ' + EPC_traffic_exec_args
       if cmd_traffic.find('-c') >= 0:
942
          cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60)
943
       task_EPC  = task_EPC + '( date; ' + cmd_traffic + ' ) > ' + logfile_traffic_EPC  +  ' 2>&1   & \n' 
944
945
946
947
948
949
950
951
952
       task_EPC = task_EPC + 'array_exec_pid+=($!) \n'  
       task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n'
    #terminate the EPC test case after timeout_cmd seconds   
    task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n'
    #task_EPC  = task_EPC + 'sleep ' +  str(timeout_cmd) + '\n'
    task_EPC  = task_EPC + 'handle_ctrl_c' '\n' 
    task_EPC  = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' 
    write_file(logfile_task_EPC, task_EPC, mode="w")
    
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
    #first we compile all the programs but only for run_0
    if run == 0:
       thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread)
       thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread)
       thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE_compile, False, timeout_thread) 
       if RRHMachine != '':
          thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH_compile, False, timeout_thread) 
       threads=[]
       threads.append(thread_eNB)
       threads.append(thread_UE)
       threads.append(thread_EPC)
       if RRHMachine != '':
         threads.append(thread_RRH)
       # Start new Threads
       thread_eNB.start()
       thread_UE.start()
       thread_EPC.start()
       if RRHMachine != '':
         thread_RRH.start()
       #Wait for all the compile threads to complete
       for t in threads:
         t.join()
Rohit Gupta's avatar
Rohit Gupta committed
975
976

    #Now we execute all the threads
977
978
979
    thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread)
    thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread)
    thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password  , task_UE, False, timeout_thread) 
Rohit Gupta's avatar
Rohit Gupta committed
980
981
    if RRHMachine != '':
        thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password  , task_RRH, False, timeout_thread) 
982
983
984
985
    threads=[]
    threads.append(thread_eNB)
    threads.append(thread_UE)
    threads.append(thread_EPC)
Rohit Gupta's avatar
Rohit Gupta committed
986
987
    if RRHMachine != '':
        threads.append(thread_RRH)
988
989
990
991
992
    # Start new Threads

    thread_eNB.start()
    thread_UE.start()
    thread_EPC.start()
Rohit Gupta's avatar
Rohit Gupta committed
993
994
    if RRHMachine != '':
        thread_RRH.start()
995
996
997
998
    #Wait for all the compile threads to complete
    for t in threads:
       t.join()
    #Now we get the log files from remote machines on the local machine
Rohit Gupta's avatar
Rohit Gupta committed
999
    if RRHMachine != '':
1000
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC, oai_RRH] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC, logdir_RRH], logdirOAI5GRepo)
Rohit Gupta's avatar
Rohit Gupta committed
1001
    else:
1002
       cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC], logdirOAI5GRepo)       
Rohit Gupta's avatar
Rohit Gupta committed
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
    logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log'
    logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log'

    if UE_stop_script != "":
      cmd = ' ( uname -a ; date \n'
      cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n'
      cmd = cmd + 'env |grep OPENAIR  \n' + 'array_exec_pid=() \n'
      cmd = cmd + UE_stop_script + '\n'
      cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' 
      write_file(logfile_UE_stop_script , cmd, mode="w")
      thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password  , cmd, False, timeout_thread)
      thread_UE.start()
      thread_UE.join()
   
Rohit Gupta's avatar
Rohit Gupta committed
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
    #Now we change the permissions of the logfiles to avoid some of them being with root permissions
    cmd = 'sudo -E chown -R ' + user + ' ' + logdir_eNB
    res= oai_eNB.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_eNB + "> in eNB machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_UE
    res= oai_UE.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_UE + "> in UE machine..." + res

    cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_EPC
    res= oai_EPC.send_recv(cmd)
    print "Changing permissions of logdir <" + logdir_EPC + "> in EPC machine..." + res

Rohit Gupta's avatar
Rohit Gupta committed
1030
1031
1032
1033
    if RRHMachine != '':
       cmd = 'sudo -E chown -R ' + user + ' ' +  logdir_RRH
       res= oai_RRH.send_recv(cmd)
       print "Changing permissions of logdir <" + logdir_RRH + "> in RRH machine..." + res
1034

1035
    print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC
1036
    SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
1037

1038
    print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB
1039
    SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
1040

1041
    print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE
1042
    SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
1043

Rohit Gupta's avatar
Rohit Gupta committed
1044
1045
1046
    if RRHMachine != '':
       print "Copying files from RRHMachine : " + RRHMachine + "logdir_RRH = " + logdir_RRH
       SSHSessionWrapper(RRHMachine, user, None, password, logdir_RRH, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all")
1047

Rohit Gupta's avatar
Rohit Gupta committed
1048
    
1049
    #Currently we only perform throughput tests
1050
1051
1052
    tput_run_string=''
    result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out)
    tput_run_string = tput_run_string + tput_string
1053
    run_result=run_result&result
1054
    result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out)
1055
    run_result=run_result&result
1056
1057
    tput_run_string = tput_run_string + tput_string
    result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out)
1058
    run_result=run_result&result
1059
    tput_run_string = tput_run_string + tput_string
Rohit Gupta's avatar
Rohit Gupta committed
1060
    
1061
    if run_result == 1: