Commit 83023e4e authored by Rohit Gupta's avatar Rohit Gupta
Browse files

fixed bugs in autotest framework

parent c120a63d
This diff is collapsed.
......@@ -11,8 +11,9 @@ source $OPENAIR_DIR/cmake_targets/tools/test_helper
#SUDO="sudo -E "
tdir=$OPENAIR_DIR/cmake_targets/autotests
rm -fr $tdir/bin $tdir/log
mkdir -p $tdir/bin $tdir/log
results_file="$tdir/log/execution_autotests.xml"
results_file="$tdir/log/results_autotests.xml"
updated=$(svn st -q $OPENAIR_DIR)
if [ "$updated" != "" ] ; then
......@@ -21,6 +22,92 @@ fi
cd $tdir
#\param $1 -> name of test case
#\param $2 -> name of compilation program
#\param $3 -> arguments for compilation program
#\param $4 -> name of pre-executable to install kernel modules, etc
#\param $5 -> arguments of pre-executable
#\param $6 -> name of executable
#\param $7 -> arguments for running the program
#\param $8 -> search expression
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault)
#\param $10 -> number of runs
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
test_compile() {
xUnit_start
test_case_name=$1
log_dir=$tdir/log
log_file=$tdir/log/test.$1.txt
compile_prog=$2
compile_args=$3
pre_exec_file=$4
pre_exec_args=$5
exec_args=$7
search_expr_array=("${!8}")
search_expr_negative=$9
nruns=${10}
pre_compile_prog=${11}
class=${12}
build_dir=$tdir/$1/build
exec_file=$build_dir/$6
#Temporary log file where execution log is stored.
temp_exec_log=$log_dir/temp_log.txt
#echo "log_dir = $log_dir"
#echo "log_file = $log_file"
#echo "exec_file = $exec_file"
#echo "exec_args = $exec_args"
#echo "search_expr = $search_expr"
#echo "pre_exec_file = $pre_exec_file"
#echo "nruns = $nruns"
echo "class = $class"
compile_prog_array=()
read -a compile_prog_array <<<"$compile_prog"
pre_compile_prog_array=()
readarray -t pre_compile_prog_array <<< "$pre_compile_prog"
for (( run_index=1; run_index <= $nruns; run_index++ ))
do
for pre_compile_prog_array_index in "${pre_compile_prog_array[@]}"
do
for compile_prog_array_index in "${compile_prog_array[@]}"
do
echo "Compiling test case $test_case_name.$compile_prog_array_index Log file = $log_file"
echo "<COMPILATION LOG file=$compile_prog_array_index , Run = $run_index>" >> $log_file
rm -fr $build_dir
mkdir -p $build_dir
cd $build_dir
{
eval $pre_compile_prog_array_index
cmake ..
#rm -fv $exec_file
make -j`nproc` $compile_prog_array_index $compile_args
}>> $log_file 2>&1
echo "</COMPILATION LOG>" >> $log_file 2>&1
if [ "$class" == "compilation" ]; then
if [ -s "$compile_prog_array_index" ] ; then
echo_success "$test_case_name $compile_prog_array_index compiled"
xUnit_success "compilation" "$test_case_name.$compile_prog_array_index" "PASS" "$run_index"
else
echo_error "$test_case_name $exec_prog compilation failed"
xUnit_fail "compilation" "$test_case_name.$compile_prog_array_index" "FAIL" "$run_index"
fi
fi
done # End of for loop compile_prog_array
done # End of for loop (pre_compile_prog_array_index)
done #End of for loop (run_index)
}
#\param $1 -> name of test case
#\param $2 -> name of compilation program
......@@ -32,10 +119,12 @@ cd $tdir
#\param $8 -> search expression
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault)
#\param $10 -> number of runs
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
test_compile_and_run() {
xUnit_start
test_case_name=$1.$2
test_case_name=$1
log_dir=$tdir/log
log_file=$tdir/log/test.$1.txt
compile_prog=$2
......@@ -46,101 +135,128 @@ test_compile_and_run() {
search_expr_array=("${!8}")
search_expr_negative=$9
nruns=${10}
pre_compile_prog=${11}
class=${12}
build_dir=$tdir/$1/build
exec_file=$build_dir/$6
#Temporary log file where execution log is stored.
temp_exec_log=$log_dir/temp_log.txt
echo "Compiling test case $test_case_name. Log file = $log_file"
rm -fr $build_dir
mkdir -p $build_dir
#echo "log_dir = $log_dir"
#echo "log_file = $log_file"
#echo "exec_file = $exec_file"
#echo "args = $args"
#echo "exec_args = $exec_args"
#echo "search_expr = $search_expr"
#echo "pre_exec_file = $pre_exec_file"
#echo "nruns = $nruns"
echo "class = $class"
compile_prog_array=()
read -a compile_prog_array <<<"$compile_prog"
for search_expr in "${compile_prog_array[@]}"
do
echo "Compiling test case $test_case_name.$search_expr Log file = $log_file"
echo "<COMPILATION LOG file=$search_expr>" >> $log_file
rm -fr $build_dir
mkdir -p $build_dir
cd $build_dir
{
eval $pre_compile_prog
cmake ..
#rm -fv $exec_file
make -j`nproc` $search_expr $compile_args
}>> $log_file 2>&1
echo "</COMPILATION LOG>" >> $log_file 2>&1
if [ "$class" == "compilation" ]; then
echo "compilation 1"
#TODO: The number of runs for compilation test cases is always 1
if [ -s "$search_expr" ] ; then
echo "compilation 1 pass"
echo_success "$test_case_name $search_expr compiled"
xUnit_success "compilation" "$test_name.$search_expr" "PASS" 1
else
echo "compilation 1 fail"
echo_error "$test_case_name $exec_prog compilation failed"
xUnit_fail "compilation" "$test_name.$search_expr" "FAIL" 1
fi
fi
done
echo "<COMPILATION LOG>" > $log_file
cd $build_dir
{
cmake ..
#rm -fv $exec_file
make -j`nproc` $compile_prog
}>> $log_file 2>&1
echo "</COMPILATION LOG>" >> $log_file 2>&1
for (( run_index=1; run_index <= $nruns; run_index++ ))
do
#process the test case if it is that of execution
if [ "$class" == "execution" ]; then
for (( run_index=1; run_index <= $nruns; run_index++ ))
do
echo "Executing test case $test_case_name, Run Index = $run_index, Log file = $log_file"
echo "Executing test case $test_case_name, Run Index = $run_index, Log file = $log_file"
echo "-----------------------------------------------------------------------------" >> $log_file 2>&1
echo "<EXECUTION LOG Run = $run_index >" >> $log_file 2>&1
echo "-----------------------------------------------------------------------------" >> $log_file 2>&1
echo "<EXECUTION LOG Run = $run_index >" >> $log_file 2>&1
if [ -n "$pre_exec_file" ]; then
{ source $pre_exec_file $pre_exec_args; } >> $log_file 2>&1
fi
if [ -n "$pre_exec_file" ]; then
{ eval "source $pre_exec_file $pre_exec_args"; } >> $log_file 2>&1
fi
{ $exec_file $exec_args ;} > $temp_exec_log 2>&1
{ eval "$exec_file $exec_args" ;} > $temp_exec_log 2>&1
cat $temp_exec_log >> $log_file 2>&1
echo "</EXECUTION LOG Run = $run_index >" >> $log_file 2>&1
cat $temp_exec_log >> $log_file 2>&1
echo "</EXECUTION LOG Run = $run_index >" >> $log_file 2>&1
result=1
for search_expr in "${search_expr_array[@]}"
do
result=1
for search_expr in "${search_expr_array[@]}"
do
search_result=`grep -E "$search_expr" $temp_exec_log`
search_result=`grep -E "$search_expr" $temp_exec_log`
#echo "search_expr = $search_expr"
#echo "search_result = $search_result"
if [ -z "$search_result" ]; then
let "result = result & 0"
else
let "result = result & 1"
fi
done
if [ -z "$search_result" ]; then
let "result = result & 0"
else
let "result = result & 1"
fi
done
#echo "result = $result"
#echo "result = $result"
test_case_result=""
if [ "$result" -eq "0" ]; then
test_case_result="FAIL"
fi
test_case_result=""
if [ "$result" -eq "0" ]; then
test_case_result="FAIL"
fi
if [ "$result" -eq "1" ]; then
test_case_result="PASS"
fi
if [ "$result" -eq "1" ]; then
test_case_result="PASS"
fi
#If we find a negative search result then there is crash of program and test case is failed even if above condition is true
#If we find a negative search result then there is crash of program and test case is failed even if above condition is true
search_result=`grep -iE "$search_expr_negative" $temp_exec_log`
if [ -n "$search_result" ]; then
test_case_result="FAIL"
fi
search_result=`grep -iE "$search_expr_negative" $temp_exec_log`
if [ -n "$search_result" ]; then
test_case_result="FAIL"
fi
if [ "$test_case_result" == "FAIL" ]; then
echo_error "execution $test_case_name FAIL $run_index"
xUnit_fail "execution" "$test_case_name" "FAIL" "$run_index"
fi
if [ "$test_case_result" == "PASS" ]; then
echo_success "execution $test_case_name PASS $run_index"
xUnit_success "execution" "$test_case_name" "PASS" "$run_index"
fi
if [ "$test_case_result" == "FAIL" ]; then
echo_error "execution $test_case_name.$compile_prog FAIL $run_index"
xUnit_fail "execution" "$test_case_name.$compile_prog" "FAIL" "$run_index"
fi
# End of for loop
done
if [ "$test_case_result" == "PASS" ]; then
echo_success "execution $test_case_name.$compile_prog PASS $run_index"
xUnit_success "execution" "$test_case_name.$compile_prog" "PASS" "$run_index"
fi
# End of for loop
done
fi
}
dbin=$OPENAIR_DIR/cmake_targets/autotests/bin
......@@ -169,6 +285,88 @@ else
fi
}
main () {
xml_conf="$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml"
test_case_list=`xmlstarlet sel -T -t -m /xml/testCaseList/testCase -s D:N:- "@id" -v "@id" -n $xml_conf`
echo "test_case_list = $test_case_list"
readarray -t test_case_array <<<"$test_case_list"
for search_expr in "${test_case_array[@]}"
do
name=$search_expr
class=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/class" $xml_conf`
desc=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/desc" $xml_conf`
pre_compile_prog=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/pre_compile_prog" $xml_conf`
compile_prog=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/compile_prog" $xml_conf`
compile_prog_args=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/compile_prog_args" $xml_conf`
pre_exec=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/pre_exec" $xml_conf`
pre_exec_args=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/pre_exec_args" $xml_conf`
main_exec=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/main_exec" $xml_conf`
main_exec_args=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/main_exec_args" $xml_conf`
search_expr_true=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/search_expr_true" $xml_conf`
search_expr_false=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/search_expr_false" $xml_conf`
nruns=`xmlstarlet sel -t -v "/xml/testCaseList/testCase[@id='$search_expr']/nruns" $xml_conf`
echo "class = $class"
echo "name = $name"
echo "Description = $desc"
echo "pre_compile_prog = $pre_compile_prog"
echo "compile_prog = $compile_prog"
echo "pre_exec = $pre_exec"
echo "pre_exec_args = $pre_exec_args"
echo "main_exec = $main_exec"
echo "main_exec_args = $main_exec_args"
echo "search_expr_true = $search_expr_true"
echo "search_expr_false = $search_expr_false"
echo "nruns = $nruns"
#eval $pre_exec
search_array_true=()
IFS=\" #set the shell's field separator
set -f #don't try to glob
#set -- $search_expr_true #split on $IFS
for i in $search_expr_true
do echo "i = $i"
if [ -n "$i" ] && [ "$i" != " " ]; then
search_array_true+=("$i")
#echo "inside i = \"$i\" "
fi
done
unset IFS
#echo "arg1 = ${search_array_true[0]}"
#echo " arg2 = ${search_array_true[1]}"
if [ "$class" == "compilation" ]; then
test_compile "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class"
elif "$class" == "execution" ]; then
test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class"
else
echo "Unexpected class of test case...Exiting...."
fi
done
}
main
xUnit_write "$results_file"
echo "Test Results are written to $results_file"
exit
#$1 -> name of test case
#$2 -> name of compilation program
#$3 -> arguments for compilation program
......@@ -179,19 +377,22 @@ fi
#$8 -> search expression ARRAY which needs to be found
#$9 -> search expression which should NOT be found (for ex. segmentation fault)
#$10 -> number of runs
#$11 -> pre compile program
#oaisim tests
search_array=("Received RRCConnectionReconfigurationComplete from UE 0")
test_compile_and_run 010200 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
test_compile_and_run 010200 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -A AWGN -b1 -u1 -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
search_array=("Received RRCConnectionReconfigurationComplete from UE 0")
test_compile_and_run 010201 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -a -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
test_compile_and_run 010201 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -A AWGN -b1 -u1 -a -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
search_array=("Received RRCConnectionReconfigurationComplete from UE 0" "Received RRCConnectionReconfigurationComplete from UE 1" "Received RRCConnectionReconfigurationComplete from UE 2")
test_compile_and_run 010202 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u3 -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
test_compile_and_run 010202 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -A AWGN -b1 -u3 -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
search_array=("Received RRCConnectionReconfigurationComplete from UE 0" "Received RRCConnectionReconfigurationComplete from UE 1" "Received RRCConnectionReconfigurationComplete from UE 2")
test_compile_and_run 010203 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u3 -a -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
test_compile_and_run 010203 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O $OPENAIR_TARGETS/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -A AWGN -b1 -u3 -a -n100" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
#PHY unitary simulations for secuirity tests
search_array=("finished with 0 errors")
......@@ -224,8 +425,7 @@ test_compile_and_run 010308 "test_secu_knas_encrypt_eia2" "" "" "" "test_secu_kn
search_array=("finished with 0 errors")
test_compile_and_run 010309 "test_kdf" "" "" "" "test_kdf" " --verbose" "search_array[@]" "segmentation fault|assertion|exiting|fatal" 3
#TODO: Add test cases for 10,20 MHz
#TODO: Test and compile seperately for Rel8/Rel10
#test_compile_and_run 0200 "oaisim_nos1" "" "$OPENAIR_DIR/cmake_targets/tools/init_nas_nos1" "" "oaisim_nos1" " -O /home/calisson/rohit/oai_snav/taets/PROJECTS/GENERIC-LTE-EPC/CONF/enb.band7.generic.oaisim.local_no_mme.conf -b1 -u1 -a " "RRC_CONN" 3
......@@ -240,3 +440,5 @@ xUnit_write "$results_file"
echo "Test Results are written to $results_file"
......@@ -254,7 +254,8 @@ check_install_oai_software() {
openssh-server \
openssl \
python \
subversion
subversion \
xmlstarlet
$SUDO update-alternatives --set liblapack.so /usr/lib/atlas-base/atlas/liblapack.so
if [ `lsb_release -rs` = '12.04' ] ; then
install_nettle_from_source
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment