run_exec_autotests.bash 18.4 KB
Newer Older
1 2
#!/bin/bash

3 4
if [ -s $OPENAIR_DIR/cmake_targets/tools/build_helper ] ; then
   source $OPENAIR_DIR/cmake_targets/tools/build_helper
5 6 7 8 9
else
   echo "Error: no file in the file tree: is OPENAIR_DIR variable set?"
   exit 1
fi

10 11
trap handle_ctrl_c INT

12
source $OPENAIR_DIR/cmake_targets/tools/test_helper
guptar's avatar
guptar committed
13

14

15
#SUDO="sudo -E "
guptar's avatar
guptar committed
16
tdir=$OPENAIR_DIR/cmake_targets/autotests
17 18
rm -fr $tdir/bin 
mkdir -p $tdir/bin
19
results_file="$tdir/log/results_autotests.xml"
guptar's avatar
guptar committed
20 21 22 23 24 25 26 27

updated=$(svn st -q $OPENAIR_DIR)
if [ "$updated" != "" ] ; then
	echo "some files are not in svn:\n $updated"
fi

cd $tdir 

28 29 30 31 32 33 34 35 36 37 38 39
#\param $1 -> name of test case
#\param $2 -> name of compilation program
#\param $3 -> arguments for compilation program
#\param $4 -> name of pre-executable to install kernel modules, etc
#\param $5 -> arguments of pre-executable
#\param $6 -> name of executable
#\param $7 -> arguments for running the program
#\param $8 -> search expression
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault) 
#\param $10 -> number of runs
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
40
#\param $13 -> output of compilation program that needs to be found for test case to pass
41
#\param $14 -> tags to help identify the test case for readability in output xml file
42
function test_compile() {
43 44 45

    xUnit_start
    test_case_name=$1
46 47
    log_dir=$tdir/log/$test_case_name
    log_file=$log_dir/test.$1.log.txt
48 49 50 51 52 53 54 55 56 57
    compile_prog=$2
    compile_args=$3
    pre_exec_file=$4
    pre_exec_args=$5
    exec_args=$7
    search_expr_array=("${!8}")
    search_expr_negative=$9
    nruns=${10}
    pre_compile_prog=${11}
    class=${12}
58
    compile_prog_out=${13}
59
    tags=${14}
60 61 62
    xmlfile_testcase=$log_dir/test.$1.xml
    #build_dir=$tdir/$1/build
    #exec_file=$build_dir/$6
63
    
64 65 66 67
    #compile_prog_out=`eval "echo $compile_prog_out"`
    #echo "compile_prog_out = $compile_prog_out"
    read -a compile_prog_out_array <<< "$compile_prog_out"

68 69
    #Temporary log file where execution log is stored.
    temp_exec_log=$log_dir/temp_log.txt
70 71
    rm -fr $log_dir
    mkdir -p $log_dir
72 73 74 75 76 77 78 79 80 81 82 83 84 85



    #echo "log_dir = $log_dir"
    #echo "log_file = $log_file"
    #echo "exec_file = $exec_file"
    #echo "exec_args = $exec_args"
    #echo "search_expr = $search_expr"
    #echo "pre_exec_file = $pre_exec_file"
    #echo "nruns = $nruns"
    echo "class = $class"
    
    compile_prog_array=()
    read -a compile_prog_array <<<"$compile_prog"
86
    
87 88
    #tags_array=()
    #read -a tags_array <<<"$tags"
89
    
90 91 92 93
    #pre_compile_prog_array=()
    #readarray -t pre_compile_prog_array <<< "$pre_compile_prog"
    result=1
    result_string=""
94 95
    for (( run_index=1; run_index <= $nruns; run_index++ ))
    do
96 97 98 99
       
    #tags_array_index=0
    #for pre_compile_prog_array_index in "${pre_compile_prog_array[@]}"  
    #do
100
    
101 102 103 104 105 106 107 108
    #for compile_prog_array_index in "${compile_prog_array[@]}"  
    #do
       echo "Compiling test case $test_case_name.${tags} Log file = $log_file"  
       date=`date`
       echo "<COMPILATION LOG file=$test_case_name.${tags} , Run = $run_index>, Date = $date " >> $log_file
       #rm -fr $build_dir
       #mkdir -p $build_dir
       cd $log_dir
109
       {   
110
          uname -a
111 112 113 114 115 116 117
          compile_log_dir=`eval echo \"$OPENAIR_DIR/cmake_targets/log/\"`
          echo "Removing compilation log files in $compile_log_dir"
          rm -frv $compile_log_dir
          echo "Executing $compile_prog $compile_prog_args ...."
          eval $compile_prog  $compile_prog_args
          echo "Copying compilation log files to test case log directory: $log_dir"
          cp -fvr $OPENAIR_DIR/cmake_targets/log/ $log_dir/compile_log
118 119 120
       }>> $log_file 2>&1
       echo "</COMPILATION LOG>" >> $log_file 2>&1
       if [ "$class" == "compilation" ]; then
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
         for compile_prog_out_index in ${compile_prog_out_array[@]}
            do 
                if  [ -s "$compile_prog_out_index" ]; then
                    let "result = result&1"
                    
                    echo_success "$test_case_name.${tags} RUN = $run_index $compile_prog_out_index = compiled"
                    
                else
                    let "result = result&0"
                            
                    echo_error "$test_case_name.${tags} RUN = $run_index $compile_prog_out_index failed"
                    
                 fi
            done #end of for loop compile_prog_out_index
         if [ "$result" == "1" ]; then
            result_string=$result_string" Run_$run_index = PASS"
         else   
            result_string=$result_string" Run_$run_index = FAIL"           
139
         fi
140

141
       fi
142 143 144
       #let "tags_array_index++"
    #done # End of for loop compile_prog_array
    #done # End of for loop (pre_compile_prog_array_index)
145
    done #End of for loop (run_index)
146 147 148 149 150 151 152 153 154 155
    
   
    #If for for some reason upper for loop does not execute, we fail the test case completely
    if [ "$result_string" == "" ]; then
      result=0
    fi
    if [ "$result" == "1" ]; then
      echo_success "$test_case_name.${tags} PASSED"
      xUnit_success "compilation" "$test_case_name.$tags" "PASS" "$result_string" "$xmlfile_testcase"
    else             
Rohit Gupta's avatar
Rohit Gupta committed
156 157
      echo_error "$test_case_name.${tags} FAILED"
      xUnit_fail "compilation" "$test_case_name.$tags" "FAIL" "$result_string" "$xmlfile_testcase"
158
    fi
159 160 161
}


guptar's avatar
guptar committed
162 163 164 165 166 167 168 169 170

#\param $1 -> name of test case
#\param $2 -> name of compilation program
#\param $3 -> arguments for compilation program
#\param $4 -> name of pre-executable to install kernel modules, etc
#\param $5 -> arguments of pre-executable
#\param $6 -> name of executable
#\param $7 -> arguments for running the program
#\param $8 -> search expression
171 172
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault) 
#\param $10 -> number of runs
173 174
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
175 176
#\param $13 -> output of compilation program that needs to be found for test case to pass
#\param $14 -> tags to help identify the test case for readability in output xml file
177
#\param $15 => password for the user to run certain commands as sudo
178
#\param $16 => test config file params to be modified
guptar's avatar
guptar committed
179

180
function test_compile_and_run() {
guptar's avatar
guptar committed
181
    xUnit_start
182
    test_case_name=$1
183 184
    log_dir=$tdir/log/$test_case_name
    log_file=$log_dir/test.$1.log.txt
guptar's avatar
guptar committed
185 186 187 188
    compile_prog=$2
    compile_args=$3
    pre_exec_file=$4
    pre_exec_args=$5
189
    main_exec=$6
guptar's avatar
guptar committed
190
    exec_args=$7
guptar's avatar
guptar committed
191
    search_expr_array=("${!8}")
192 193
    search_expr_negative=$9
    nruns=${10}
194 195
    pre_compile_prog=${11}
    class=${12}
196 197
    compile_prog_out=${13}
    tags=${14}
198
    mypassword=${15}
199 200
    test_config_file=${16}

guptar's avatar
guptar committed
201
    build_dir=$tdir/$1/build
202 203
    #exec_file=$build_dir/$6
    xmlfile_testcase=$log_dir/test.$1.xml
guptar's avatar
guptar committed
204 205
    #Temporary log file where execution log is stored.
    temp_exec_log=$log_dir/temp_log.txt
206
    export OPENAIR_LOGDIR=$log_dir
207 208 209 210 211 212 213
    rm -fr $log_dir
    mkdir -p $log_dir
    
    rm -fr $OPENAIR_DIR/cmake_targets/log

    echo "" > $temp_exec_log
    echo "" > $log_file
214 215 216
    #echo "log_dir = $log_dir"
    #echo "log_file = $log_file"
    #echo "exec_file = $exec_file"
217
    #echo "exec_args = $exec_args"
218 219 220
    #echo "search_expr = $search_expr"
    #echo "pre_exec_file = $pre_exec_file"
    #echo "nruns = $nruns"
221
    echo "class = $class"
guptar's avatar
guptar committed
222
    
223 224
    #compile_prog_array=()
    #read -a compile_prog_array <<<"$compile_prog"
225 226 227 228

    #test_config_file=`eval "echo \"$test_config_file\" "`

    #echo "test_config_file = $test_config_file"
229
  
230 231 232 233 234
    tags_array=()
    read -a tags_array <<<"$tags"
    
    main_exec_args_array=()
    readarray -t main_exec_args_array <<< "$exec_args"
235 236 237 238 239 240
    
    
    #for search_expr in "${compile_prog_array[@]}"  
    #do
       echo "Compiling test case $test_case_name Log file = $log_file"  
       echo "<COMPILATION LOG file=$log_file>" >> $log_file
241

242 243
       #rm -fr $build_dir
       #mkdir -p $build_dir
244

245
       cd $log_dir
246
       {   
247
          uname -a
248 249 250 251 252 253 254 255
          echo "Executing $pre_compile_prog"
          eval $pre_compile_prog
 
          if [ "$test_config_file" != "" ]; then
            echo "Modifying test_config_file parameters..."
            echo "$test_config_file" |xargs -L 1 $OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py 
          fi
          echo "Executing $compile_prog $compile_args"
256 257 258
          eval "$compile_prog $compile_args"
          echo "Copying compilation log files to test case log directory: $log_dir"
          cp -fvr $OPENAIR_DIR/cmake_targets/log/ $log_dir/compile_log
259 260
       }>> $log_file 2>&1
       echo "</COMPILATION LOG>" >> $log_file 2>&1
261
    #done
262
    
263 264
    #process the test case if it is that of execution
    if [ "$class" == "execution" ]; then
265 266 267
      tags_array_index=0
      for main_exec_args_array_index in "${main_exec_args_array[@]}"  
      do
268 269 270 271
        global_result=1
        result_string=""
        
       for (( run_index=1; run_index <= $nruns; run_index++ ))
272
        do
273 274
          temp_exec_log=$log_dir/test.$test_case_name.${tags_array[$tags_array_index]}.run_$run_index
          echo "" > $temp_exec_log
guptar's avatar
guptar committed
275

276 277 278 279 280
          echo "Executing test case $test_case_name.${tags_array[$tags_array_index]}, Run Index = $run_index, Execution Log file = $temp_exec_log"

          echo "-----------------------------------------------------------------------------" >> $temp_exec_log  2>&1
          echo "<EXECUTION LOG Test Case = $test_case_name.${tags_array[$tags_array_index]}, Run = $run_index >" >> $temp_exec_log  2>&1
           
281
          if [ -n "$pre_exec_file" ]; then
282 283
            {  echo " Executing $pre_exec_file $pre_exec_args " 
               eval " echo '$mypassword' |sudo -S -E $pre_exec_file $pre_exec_args " ; }>> $temp_exec_log  2>&1
284

285
          fi
286 287
          echo "Executing $main_exec $main_exec_args_array_index "
          echo "Executing $main_exec $main_exec_args_array_index " >> $temp_exec_log
288
          { uname -a ; eval "$main_exec $main_exec_args_array_index" ;} >> $temp_exec_log  2>&1
289

290
          echo "</EXECUTION LOG Test Case = $test_case_name.${tags_array[$tags_array_index]},  Run = $run_index >" >> $temp_exec_log  2>&1
291
          cat $temp_exec_log >> $log_file  2>&1
292
          
guptar's avatar
guptar committed
293
    
294 295 296
          result=1
          for search_expr in "${search_expr_array[@]}"
          do
guptar's avatar
guptar committed
297
     
298
            search_result=`grep -E "$search_expr" $temp_exec_log`
guptar's avatar
guptar committed
299

300 301
            #echo "search_expr  =   $search_expr"
            #echo "search_result = $search_result"
guptar's avatar
guptar committed
302

303 304 305 306 307 308
            if [ -z "$search_result" ]; then
              let "result = result & 0"
            else
              let "result = result & 1"
            fi
          done
309 310 311 312 313 314 315
          
          #If we find a negative search result then there is crash of program and test case is failed even if above condition is true
          search_result=`grep -iE "$search_expr_negative" $temp_exec_log`
          if [ -n "$search_result" ]; then
            result=0
          fi
          let "global_result = global_result & result"
guptar's avatar
guptar committed
316

317
          #echo "result = $result"
318 319 320
          
          #this is a result of this run
          #test_case_result=""
321
          if [ "$result" -eq "0" ]; then
322 323
            result_string=$result_string" Run_$run_index =FAIL"
            echo_error "$test_case_name.${tags_array[$tags_array_index]} RUN = $run_index Result = FAIL"
324
          fi
guptar's avatar
guptar committed
325

326
          if [ "$result" -eq "1" ]; then
327 328
            result_string=$result_string" Run_$run_index =PASS"
            echo_success "$test_case_name.${tags_array[$tags_array_index]} RUN = $run_index Result = PASS"
329
          fi
guptar's avatar
guptar committed
330

331
          
332 333 334 335 336 337
        done #End of for loop (nindex)

       echo " Result String = $result_string" 

       if [ "$result_string" == "" ]; then
           echo_error "execution $test_case_name.$compile_prog.${tags_array[$tags_array_index]} Run_Result = \"$result_string\"  Result = FAIL"
Rohit Gupta's avatar
Rohit Gupta committed
338
	   xUnit_fail "execution" "$test_case_name.${tags_array[$tags_array_index]}" "FAIL" "$result_string" "$xmlfile_testcase"
339 340 341 342 343 344 345 346 347 348 349 350
       else
        if [ "$global_result" == "0" ]; then
           echo_error "execution $test_case_name.${tags_array[$tags_array_index]} Run_Result = \"$result_string\" Result =  FAIL"
           xUnit_fail "execution" "$test_case_name.${tags_array[$tags_array_index]}" "FAIL" "$result_string" "$xmlfile_testcase"
        fi

        if [ "$global_result" == "1" ]; then
            echo_success "execution $test_case_name.${tags_array[$tags_array_index]} Run_Result = \"$result_string\"  Result = PASS "
	    xUnit_success "execution" "$test_case_name.${tags_array[$tags_array_index]}" "PASS" "$result_string"   "$xmlfile_testcase"     
        fi  
       fi

351
       let "tags_array_index++"
352
     done 
353
   fi
354
   rm -fr $build_dir
guptar's avatar
guptar committed
355 356
}

357 358 359
dbin=$OPENAIR_DIR/cmake_targets/autotests/bin
dlog=$OPENAIR_DIR/cmake_targets/autotests/log

guptar's avatar
guptar committed
360

361
function print_help() {
362 363 364 365 366 367 368 369
 echo_info '
This program runs automated test case system for OpenAirInterface
You should have ubuntu 14.xx, updated, and the Linux kernel >= 3.14
Options
-h | --help
   This help
-g | --run-group
   Run test cases in a group. For example, ./run_exec_autotests "0101* 010102"
Rohit Gupta's avatar
Rohit Gupta committed
370 371
-p
   Use password for logging
372 373
'
}
374

375
function main () {
376
RUN_GROUP=0
Rohit Gupta's avatar
Rohit Gupta committed
377 378
SET_PASSWORD=0
passwd=""
379 380 381
test_case_group=""
test_case_group_array=()
test_case_array=()
382
echo_info "Note that the user should be sudoer for executing certain commands, for example loading kernel modules"
Rohit Gupta's avatar
Rohit Gupta committed
383

384 385 386 387 388 389

until [ -z "$1" ]
  do
    case "$1" in
       -g | --run-group)
            RUN_GROUP=1
390 391
            test_case_group=$2
            echo_info "Will execute test cases only in group $test_case_group"
392
            shift 2;;
Rohit Gupta's avatar
Rohit Gupta committed
393 394 395 396
        -p)
            SET_PASSWORD=1
            passwd=$2
            shift 2;;
397 398 399 400 401 402 403 404 405
        -h | --help)
            print_help
            exit 1;;
	*)
	    print_help
            echo_fatal "Unknown option $1"
            break;;
   esac
  done
406

Rohit Gupta's avatar
Rohit Gupta committed
407 408 409 410 411 412 413
if [ "$SET_PASSWORD" == "1" ]; then
   mypassword=$passwd
else
   read -s -p "Enter Password: " mypassword
fi

tmpfile=`mktemp`
414
echo \'$passwd\' | $SUDO echo $HOME >& $tmpfile
Rohit Gupta's avatar
Rohit Gupta committed
415
tstsudo=`cat $tmpfile`
Rohit Gupta's avatar
Rohit Gupta committed
416
if [ "$tstsudo" != "$HOME" ]; then
417
  echo "$USER might not have sudo privileges. Exiting" 
Rohit Gupta's avatar
Rohit Gupta committed
418
  exit
419 420
else
  echo "$USER has sudo privileges"
Rohit Gupta's avatar
Rohit Gupta committed
421 422 423
fi
echo "tstsudo = $tstsudo"
rm -fr $tmpfile
424

425 426
xml_conf="$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml"

427
test_case_list=`xmlstarlet sel -T -t -m /testCaseList/testCase -s A:N:- "@id" -v "@id" -n $xml_conf`
428 429 430 431 432

echo "test_case_list = $test_case_list"

readarray -t test_case_array <<<"$test_case_list"

433 434
read -a test_case_group_array <<< "$test_case_group"
 
435 436
for search_expr in "${test_case_array[@]}"
  do
437 438 439 440 441 442 443 444 445 446 447 448 449 450
    flag_run_test_case=0
    # search if this test case needs to be executed
    if [ "$RUN_GROUP" -eq "1" ]; then
       for search_group in "${test_case_group_array[@]}"
       do  
          if [[ $search_expr == $search_group ]];then
             flag_run_test_case=1
             echo_info "Test case $search_expr match found in group"
             break
          fi
       done
    else
       flag_run_test_case=1
    fi
451 452


453 454 455 456 457
    #We skip this test case if it is not in the group list
    if [ "$flag_run_test_case" -ne "1" ]; then
      continue
    fi

458
    name=$search_expr
459 460 461 462 463 464 465 466 467 468 469 470 471 472
    class=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/class" $xml_conf`
    desc=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/desc" $xml_conf`
    pre_compile_prog=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_compile_prog" $xml_conf`
    compile_prog=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog" $xml_conf`
    compile_prog_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog_args" $xml_conf`
    pre_exec=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_exec" $xml_conf`
    pre_exec_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_exec_args" $xml_conf`
    main_exec=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/main_exec" $xml_conf`
    main_exec_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/main_exec_args" $xml_conf`
    search_expr_true=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/search_expr_true" $xml_conf`
    search_expr_false=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/search_expr_false" $xml_conf`
    nruns=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/nruns" $xml_conf`
    compile_prog_out=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog_out" $xml_conf`
    tags=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/tags" $xml_conf`
473
    test_config_file=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/test_config_file" $xml_conf`
474

475 476 477 478 479
    echo "class = $class"
    echo "name = $name"
    echo "Description = $desc"
    echo "pre_compile_prog = $pre_compile_prog"
    echo "compile_prog = $compile_prog"
480 481
    echo "compile_prog_args = $compile_prog_args"
    echo "compile_prog_out = $compile_prog_out"
482 483 484 485 486 487 488 489 490
    echo "pre_exec = $pre_exec"
    echo "pre_exec_args = $pre_exec_args"
    echo "main_exec = $main_exec"
    echo "main_exec_args = $main_exec_args"
    echo "search_expr_true = $search_expr_true"
    echo "search_expr_false = $search_expr_false"
    echo "nruns = $nruns"

    #eval $pre_exec
491
    compile_prog_out=`eval echo \"$compile_prog_out\"`
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509

    search_array_true=()

    IFS=\"                  #set the shell's field separator
    set -f                  #don't try to glob 
    #set -- $search_expr_true             #split on $IFS
    for i in $search_expr_true
      do echo "i = $i"
        if [ -n "$i" ] && [ "$i" != " " ]; then
          search_array_true+=("$i")
          #echo "inside i = \"$i\" "
        fi 
      done
    unset IFS 

    #echo "arg1 = ${search_array_true[0]}"
    #echo " arg2 = ${search_array_true[1]}"
    if [ "$class" == "compilation" ]; then
510 511
        test_compile "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags"
    elif  [ "$class" == "execution" ]; then
512
        $SUDO killall -q oaisim_nos1
513
        test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" "$test_config_file"
514
    else
515
        echo "Unexpected class of test case...Skipping the test case $name ...."
516 517 518 519 520 521 522 523
    fi

    done
    
    

}

524 525
uname -a

526
main "$@"
527 528 529 530 531 532 533 534 535

xUnit_write "$results_file"

echo "Test Results are written to $results_file"

exit