Merge branch 'develop_integration_wk01' into 'develop'

Develop integration wk01

See merge request !74

.gitlab-ci.yml

 job1:
   script:
-   - echo $PWD
-   - echo $USER
-   - echo $OAI_USER
-   - echo $OAI_EXTRA_ARGS
-   - echo $NFS_SHARE_DIR
-   - echo $EXTERNAL_SHARE_DIR
-   - echo $SHELL
-   - echo $OAI_TEST_CASE_GROUP
-   - echo $MACHINELIST
-   - echo $MACHINELISTGENERIC
-   - git rev-parse --abbrev-ref HEAD
-   - git_repo=`git config --get remote.origin.url`
-   - git_head=`git rev-parse HEAD`
-   - echo $git_head
-   - tmp=`git show-ref --head | grep $git_head`
-   - tmp=(${tmp///// })
-   - git_branch=${tmp[@]:(-1)}
-   - echo $git_branch
-   - source oaienv
-   - echo $OPENAIR_DIR
-   - NFS_TEST_RESULTS_DIR=$NFS_SHARE_DIR/$git_branch/$git_head
-   - EXTERNAL_SHARE_RESULTS_DIR=$EXTERNAL_SHARE_DIR/$git_branch/$git_head
-   - echo $NFS_TEST_RESULTS_DIR
-   - echo $EXTERNAL_SHARE_RESULTS_DIR
-   - echo $NRUNS_LTE_SOFTMODEM
-   - echo $TIMEOUT_CMD
-   - mkdir -p $OPENAIR_DIR/cmake_targets/autotests/log
-   - python $OPENAIR_DIR/cmake_targets/autotests/run_exec_lte-softmodem_tests.py -c -5GRepo $git_repo -MachineList "$MACHINELIST" -MachineListGeneric "$MACHINELISTGENERIC"  -5GRepoHeadVersion $git_head -n $NFS_SHARE_DIR -u $OAI_USER -p $OAI_PASS  $OAI_EXTRA_ARGS -g "$OAI_TEST_CASE_GROUP">& $OPENAIR_DIR/cmake_targets/autotests/python_autotest_cleanup.log
-   - python $OPENAIR_DIR/cmake_targets/autotests/run_exec_lte-softmodem_tests.py -r -5GRepo $git_repo -MachineList "$MACHINELIST" -MachineListGeneric "$MACHINELISTGENERIC" -5GRepoHeadVersion $git_head -n $NFS_SHARE_DIR -u $OAI_USER -p $OAI_PASS `echo $OAI_EXTRA_ARGS` -g "$OAI_TEST_CASE_GROUP" --nrun_lte_softmodem $NRUNS_LTE_SOFTMODEM --timeout_cmd $TIMEOUT_CMD >& $OPENAIR_DIR/cmake_targets/autotests/python_autotest.log
-   - mv $OPENAIR_DIR/cmake_targets/autotests/python_autotest.log $OPENAIR_DIR/cmake_targets/autotests/log/python_autotest.log
-   - mv $OPENAIR_DIR/cmake_targets/autotests/python_autotest_cleanup.log $OPENAIR_DIR/cmake_targets/autotests/log/python_autotest_cleanup.log
-   - sshpass -p "$OAI_PASS" rsync -az -e "ssh -o StrictHostKeyChecking=no "  --rsync-path="mkdir -p $NFS_TEST_RESULTS_DIR && rsync" $OPENAIR_DIR/cmake_targets/autotests/log $OAI_USER@localhost:$NFS_TEST_RESULTS_DIR
-   - sshpass -p "$OAI_PASS" rsync -az -e "ssh -o StrictHostKeyChecking=no "  --rsync-path="mkdir -p $EXTERNAL_SHARE_DIR && rsync" $OPENAIR_DIR/cmake_targets/autotests/log $OAI_USER@localhost:$EXTERNAL_SHARE_DIR
-   - cat $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml
+    - date
+    - pwd
+    - echo $OAI_USER
+    - echo $OAI_PASS
+    - echo $OAI_TEST_CASE_GROUP
+    - echo $MACHINELIST
+    - echo $MACHINELISTGENERIC
+    - echo $RESULT_DIR
+    - echo $NRUNS_LTE_SOFTMODEM
+    - source oaienv
+    - rm -rf cmake_targets/autotests/log
+    - mkdir cmake_targets/autotests/log
+    - cd cmake_targets/autotests/v2
+    - python main.py
+    - date
   only:
-   - triggers
+    - triggers

cmake_targets/CMakeLists.txt

@@ -1691,7 +1691,7 @@ endif()
 # Atlas is required by some packages, but not found in pkg-config
 if(EXISTS "/usr/include/atlas/cblas.h")
   include_directories("/usr/include/atlas")
-  list(APPEND ATLAS_LIBRARIES lapack cblas atlas)
+  list(APPEND ATLAS_LIBRARIES cblas atlas lapack)
 else()
   message("No Blas/Atlas libs found, some targets will fail")
 endif()

cmake_targets/autotests/test_case_list.xml

@@ -13,7 +13,7 @@
  <Timeout_execution>36000</Timeout_execution>
  <TestCaseExclusionList>010141 0102+ 010301 010303 010304 010305 0104+ 015508 015511 015520 015523 015518 015519 015520 015521 015522 015523 015602 015605 015818 015819 015820 015821 015822 015823 016102 016105 016502 016505 017002 017005 018002 018005 018502 018505  025514 025517 025520 025523 025518 025519 025520 025521 025522 025523</TestCaseExclusionList>
  <nruns_lte-softmodem>3</nruns_lte-softmodem>
- <MachineListGeneric>amerique mozart hutch starsky stevens calisson superserver </MachineListGeneric>
+ <MachineListGeneric>nano mozart hutch starsky stevens calisson superserver </MachineListGeneric>
      <testCase id="010101" >
      <class>compilation</class>
      <desc>Build oaisim.Rel8</desc>

cmake_targets/autotests/v2/actions/alu_epc.bash

+sudo /opt/ltebox/tools/stop_ltebox || true
+sudo /opt/ltebox/tools/start_ltebox

cmake_targets/autotests/v2/actions/alu_epc_stop.bash

+sudo /opt/ltebox/tools/stop_ltebox

cmake_targets/autotests/v2/actions/alu_hss.bash

+sudo /opt/ltebox/tools/stop_ltebox || true
+sudo /opt/hss_sim0609/starthss_real

cmake_targets/autotests/v2/actions/bandrich.txt

+The configuration for the bandrich in the test setup is a bit hackish.
+
+Here come some notes explaining what we do here.
+
+To get /dev/bandrich.data and /dev/bandrich.control (so that you don't need
+to look for which /dev/ttyUSBx are used by your dongle, and always use
+the same files, no matter what), add a udev file:
+
+    /etc/udev/rules.d/bandrich.rules
+
+containing lines:
+
+    SUBSYSTEM=="tty", ENV{ID_VENDOR_ID}=="1a8d", ENV{ID_MODEL_ID}=="100d", ENV{ID_SERIAL_SHORT}=="357473040068155", ENV{ID_USB_INTERFACE_NUM}=="00", SYMLINK+="bandrich.data", MODE="0666"
+    SUBSYSTEM=="tty", ENV{ID_VENDOR_ID}=="1a8d", ENV{ID_MODEL_ID}=="100d", ENV{ID_SERIAL_SHORT}=="357473040068155", ENV{ID_USB_INTERFACE_NUM}=="02", SYMLINK+="bandrich.control", MODE="0666"
+
+Change vendor_id/model_id/serial/interface num to match yours.
+Use lsusb -v to find values.
+
+At first we used /dev/ttyUSB2 linked to /dev/bandrich for both data (in
+the wvdial configuration file) and control (in the python code, opening
+/dev/bandrich the standard unix way and read and write into it, with no
+special libusb commands).
+
+But it turned out that using /dev/ttyUSB2 for data gives bad throughput
+results. We tested downlink UDP at 15Mb/s but the maximum throughput for
+a 5MHz cell was around 13, and we had a lot of packets lost at the iperf's
+UDP level. Radio was looking fine (all packets acked correctly), so it was
+not a radio problem. The dongle in a windows machine was working fine,
+15Mbs/s. Using the wvdial configuration file from test setup v1 gave also
+good results. The only difference was the use of /dev/ttyUSB0 instead of
+/dev/ttyUSB2. Using /dev/ttyUSB0 solved the throughput issue.
+
+But using /dev/ttyUSB0 for control in the pyton code did not work. The
+output was incorrect.
+
+So we decided to have /dev/bandrich.data linked to /dev/ttyUSB0 for data
+(wvdial) and /dev/bandrich.control linked to /dev/ttyUSB2 for control
+(in python code).
+
+It may not be the correct way to go, but it seems to do the trick so far.
+
+Your device may need some other solution.
+
+Here, we get, as result of lsusb -v:
+
+    [SNIP]
+    Bus 003 Device 009: ID 1a8d:100d BandRich, Inc. 4G LTE adapter
+    Device Descriptor:
+      bLength                18
+      bDescriptorType         1
+      bcdUSB               2.00
+      bDeviceClass            0 (Defined at Interface level)
+      bDeviceSubClass         0 
+      bDeviceProtocol         0 
+      bMaxPacketSize0        64
+      idVendor           0x1a8d BandRich, Inc.
+      idProduct          0x100d 4G LTE adapter
+      bcdDevice            0.00
+      iManufacturer          10 BandRich, Inc.
+      iProduct                9 BandLuxe HSPA-LTE Adapter
+      iSerial                11 357473040068155
+    [SNIP]
+
+You can also run:
+
+    udevadm monitor
+
+and unplug/replug the dongle. It will print some information.
+
+The command:
+
+    udevadm info --export-db
+
+is also important to get the right identifier to put in ENV{}. (It also
+gives the correct value.)
+
+Here is extracted what we have for our dongle:
+
+    P: /devices/pci0000:00/0000:00:1a.0/usb3/3-1/3-1.2/3-1.2:1.2/ttyUSB2/tty/ttyUSB2
+    N: ttyUSB2
+    S: bandrich
+    S: serial/by-id/usb-BandRich__Inc._BandLuxe_HSPA-LTE_Adapter_357473040068155-if02-port0
+    S: serial/by-path/pci-0000:00:1a.0-usb-0:1.2:1.2-port0
+    E: DEVLINKS=/dev/bandrich /dev/serial/by-id/usb-BandRich__Inc._BandLuxe_HSPA-LTE_Adapter_357473040068155-if02-port0 /dev/serial/by-path/pci-0000:00:1a.0-usb-0:1.2:1.2-port0
+    E: DEVNAME=/dev/ttyUSB2
+    E: DEVPATH=/devices/pci0000:00/0000:00:1a.0/usb3/3-1/3-1.2/3-1.2:1.2/ttyUSB2/tty/ttyUSB2
+    E: ID_BUS=usb
+    E: ID_MM_CANDIDATE=1
+    E: ID_MODEL=BandLuxe_HSPA-LTE_Adapter
+    E: ID_MODEL_ENC=BandLuxe\x20HSPA-LTE\x20Adapter
+    E: ID_MODEL_FROM_DATABASE=4G LTE adapter
+    E: ID_MODEL_ID=100d
+    E: ID_PATH=pci-0000:00:1a.0-usb-0:1.2:1.2
+    E: ID_PATH_TAG=pci-0000_00_1a_0-usb-0_1_2_1_2
+    E: ID_REVISION=0000
+    E: ID_SERIAL=BandRich__Inc._BandLuxe_HSPA-LTE_Adapter_357473040068155
+    E: ID_SERIAL_SHORT=357473040068155
+    E: ID_TYPE=generic
+    E: ID_USB_DRIVER=option
+    E: ID_USB_INTERFACES=:ffffff:020600:0a0000:080650:
+    E: ID_USB_INTERFACE_NUM=02
+    E: ID_VENDOR=BandRich__Inc.
+    E: ID_VENDOR_ENC=BandRich\x2c\x20Inc.
+    E: ID_VENDOR_FROM_DATABASE=BandRich, Inc.
+    E: ID_VENDOR_ID=1a8d
+    E: MAJOR=188
+    E: MINOR=2
+    E: SUBSYSTEM=tty
+    E: USEC_INITIALIZED=672068596
+
+You can also run:
+
+    udevadm info -a -p /sys/bus/usb-serial/devices/ttyUSB0
+    udevadm info -a -p /sys/bus/usb-serial/devices/ttyUSB2
+
+Note: after creating the udev files, you need to unplug/replug your
+dongle for /dev/bandrich.data and /dev/bandrich.control to appear.
+
+Note: the mode 0666 is for everyone to access the dongle (no need for
+      root privileges). If you prefer you can set it to 0600 (only root)
+      or 0660 (root and group).
+
+Then you need to configure pppd, to get correct 'route' information when
+you run wvdial.
+
+The file /etc/ppp/peers/wvdial should have the following content:
+
+    noauth
+    name wvdial
+    usepeerdns
+    defaultroute
+    replacedefaultroute
+
+The file wvdial.bandrich.conf has been created by copying some information
+found on the Internet. Its content may not be fully correct. Adapt to your
+situation. It seems to work here.

cmake_targets/autotests/v2/actions/client_tcp.bash

+echo $SERVER_IP
+timeout -s 9 20s iperf -c $SERVER_IP -i1

cmake_targets/autotests/v2/actions/client_udp.bash

+echo $SERVER_IP
+echo $UDP_BANDWIDTH
+timeout -s 9 20s iperf -c $SERVER_IP -i1 -u -b $UDP_BANDWIDTH

cmake_targets/autotests/v2/actions/clone_repository.bash

+sudo rm -rf /tmp/oai_test_setup
+mkdir /tmp/oai_test_setup
+cd /tmp/oai_test_setup
+git clone $REPOSITORY_URL oai
+cd oai
+git checkout $COMMIT_ID

cmake_targets/autotests/v2/actions/compilation.bash

+cd /tmp/oai_test_setup/oai
+source oaienv
+cd cmake_targets
+rm -rf log
+mkdir -p log
+echo $BUILD_ARGUMENTS
+./build_oai $BUILD_ARGUMENTS
+echo $BUILD_OUTPUT
+ls $BUILD_OUTPUT

cmake_targets/autotests/v2/actions/execution.bash

+echo $EXEC $EXEC_ARGS
+$EXEC $EXEC_ARGS

cmake_targets/autotests/v2/actions/execution_compile.bash

+cd /tmp/oai_test_setup/oai
+source oaienv
+cd cmake_targets
+rm -rf log
+mkdir -p log
+bash -c "$PRE_BUILD"
+$BUILD_PROG $BUILD_ARGUMENTS
+bash -c "$PRE_EXEC"

cmake_targets/autotests/v2/actions/modem.py

+import sys, os, select, re, time
+
+def quit(r):
+    sys.stdout.flush()
+    os._exit(r)
+
+class ModemResponse:
+    def __init__(self, retcode, retstring):
+        self.ret = retcode
+        self.data = retstring
+
+class Modem:
+    def open(self):
+        self.i = os.open(self.devname, os.O_RDONLY)
+        self.o = os.open(self.devname, os.O_WRONLY)
+        #clear output of modem, if any is pending (not sure of this)
+        while True:
+            (ri, ro, re) = select.select([self.i], [], [self.i], 0)
+            if len(ri) == 0:
+                break
+            l = os.read(self.i, 65536)
+            print "WARNING: modem had unread data: '" + \
+                  l.replace('\r', '\\r') + "'"
+
+    def __init__(self, devname):
+        self.devname = devname
+        self.i = -1
+        self.o = -1
+        self.open()
+
+    def send(self, s):
+        print "DEBUG: SEND TO MODEM: '" + s + "'"
+        os.write(self.o, s+"\r")
+
+    def recv(self):
+        return os.read(self.i, 65536)
+
+    def wait(self):
+        ok         = '\r\nOK\r\n'
+        error      = '\r\nERROR\r\n'
+        cme_error  = '\r\nCME ERROR:[^\r]*\r\n'
+        no_carrier = '\r\nNO CARRIER\r\n'
+        l = ''
+        while True:
+            l = l + self.recv()
+            print "DEBUG: CURRENT MODEM RESPONSE: '" + \
+                  l.replace('\r','\\r').replace('\n','\\n') + "'"
+
+            #AT returned 'things' are "\r\nXXXX\r\n", look for that.
+            #Check if last one matches 'ok', 'error' or 'cme_error'.
+            #(Hopefully this is enough and no other reply is possible.)
+            #This code accepts invalid responses from modem, ie. all
+            #that does not fit in the 'findall' is thrashed away, maybe
+            #we want to do something in this case?
+
+            res = re.findall('\r\n[^\r]*\r\n', l)
+            if len(res) == 0:
+                print "DEBUG: NO MATCH: wait for more input from modem"
+                continue
+            last_res = res[len(res)-1]
+            print "DEBUG: CURRENT LAST LINE: '" + \
+                  last_res.replace('\r','\\r').replace('\n','\\n')+"'"
+            if re.match(ok, last_res) != None:
+                return ModemResponse(True, l)
+            if (   re.match(error,      last_res) != None or
+                   re.match(cme_error,  last_res) != None or
+                   re.match(no_carrier, last_res) != None):
+                return ModemResponse(False, l)
+        #TODO purge?
+        #re.purge()
+
+    def modem_reset_cycle(self):
+        #close all
+        os.close(self.i)
+        os.close(self.o)
+        self.i = -1
+        self.o = -1
+
+        print "DEBUG: RESET CYCLE: wait for modem to go away"
+        #wait for file descriptor to go away
+        while True:
+            try:
+                test = os.open(self.devname, os.O_RDONLY)
+                os.close(test)
+                time.sleep(0.1)
+            except BaseException, e:
+                break
+        print "DEBUG: RESET CYCLE: modem has gone away"
+
+        print "DEBUG: RESET CYCLE: wait for modem to come back"
+        #wait for file descriptor to be back, try to open it over and over
+        #TODO: use inotify here? (it's not in basic python as it seems)
+        while True:
+            try:
+                test = os.open(self.devname, os.O_RDONLY)
+                os.close(test)
+                break
+            except BaseException, e:
+                time.sleep(0.1)
+        print "DEBUG: RESET CYCLE: modem is back"
+
+        #back to business
+        self.open()

cmake_targets/autotests/v2/actions/run_enb.bash

+#enable control+C reception (to be refined if it does not work)
+stty isig intr ^C
+
+cd /tmp/oai_test_setup/oai
+source oaienv
+cd cmake_targets/lte_build_oai/build
+ulimit -c unlimited
+sudo rm -f core
+#sudo -E ./lte-softmodem -O $OPENAIR_DIR/cmake_targets/autotests/v2/config/enb.band7.tm1.usrpb210.conf
+sudo -E ./lte-softmodem -O /tmp/enb.conf

cmake_targets/autotests/v2/actions/server_tcp.bash

+stty isig intr ^C
+
+#timeout -s 9 20s iperf -s -i1
+iperf -s -i1

cmake_targets/autotests/v2/actions/server_udp.bash

+stty isig intr ^C
+
+#timeout -s 9 20s iperf -s -i1 -u
+iperf -s -i1 -u

cmake_targets/autotests/v2/actions/start_bandrich.bash

+#enable control+C reception (to be refined if it does not work)
+stty isig intr ^C
+
+#If /dev/bandrich is not present when we start the test, what to do?
+#The following commented lines are an attempt at solving this.
+#This is not satisfying, so we rather do nothing.
+
+##the UE got stuck once, I had to run usb_modeswitch by hand.
+##So at this point, if /dev/bandrich does not exist, maybe the UE is
+##stuck for whatever reason. Let's forcefully run usb_modeswitch.
+#if [ ! -e /dev/bandrich ]; then sudo usb_modeswitch -v 1a8d -p 1000 -I -W -K; true; fi
+#
+##wait for /dev/bandrich (TODO: use inotify?)
+##may fail if the bandrich is in a bad state
+#while [ ! -e /dev/bandrich ]; do sleep 1; done
+
+cd /tmp/oai_test_setup/oai
+source oaienv
+cd cmake_targets/autotests/v2/actions
+python start_bandrich.py
+
+sudo wvdial -C wvdial.bandrich.conf || true
+
+python stop_bandrich.py

cmake_targets/autotests/v2/actions/start_bandrich.py

+import time
+
+from modem import quit, Modem
+
+try:
+    modem = Modem("/dev/bandrich.control")
+
+    #test that modem is there
+    print "INFO: check modem's presence"
+    modem.send('AT')
+    if modem.wait().ret != True:
+        print "ERROR: no modem?"
+        quit(1)
+
+    #activate the modem, be brutal and reset it too!
+    print "INFO: reset and activate the modem"
+    modem.send('AT+CFUN=1,1')
+    if modem.wait().ret != True:
+        print "ERROR: failed asking modem for activation"
+        quit(1)
+
+    #modem has gone! wait for it to pop up again
+    modem.modem_reset_cycle()
+
+    #wait for modem to be connected
+    #timeout after one minute
+    print "INFO: wait for modem to be connected (timeout: one minute)"
+    start_time = time.time()
+    while True:
+        modem.send('AT+CGATT?')
+        r = modem.wait()
+        if r.ret != True:
+            print "ERROR: failed checking attachment status of modem"
+            quit(1)
+        if "+CGATT: 1" in r.data:
+            break
+        if not "CGATT: 0" in r.data:
+            print "ERROR: bad data when checking attachment status of modem"
+            quit(1)
+        time.sleep(0.1)
+        if time.time() > start_time + 60:
+            print "ERROR: modem not connected after one minute, close modem"
+            modem.send('AT+CFUN=4')
+            r = modem.wait()
+            if r.ret != True:
+                print "ERROR: closing modem failed"
+            quit(1)
+
+    print "INFO: modem is connected"
+
+except BaseException, e:
+    print "ERROR: " + str(e)
+    quit(1)
+
+quit(0)

cmake_targets/autotests/v2/actions/stop_bandrich.py

+import time
+
+from modem import quit, Modem
+
+try:
+    modem = Modem("/dev/bandrich.control")
+
+    #test that modem is there
+    print "INFO: check modem's presence"
+    modem.send('AT')
+    r = modem.wait()
+    if r.ret != True and "NO CARRIER" not in r.data:
+        print "ERROR: no modem?"
+        quit(1)
+    if "NO CARRIER" in r.data:
+        print "WARNING: 'NO CARRIER' detected, not sure if handled correctly"
+
+    #deactivate the modem
+    print "INFO: deactivate the modem"
+    modem.send('AT+CFUN=4')
+    if modem.wait().ret != True:
+        print "ERROR: failed asking modem for deactivation"
+        quit(1)
+
+except BaseException, e:
+    print "ERROR: " + str(e)
+    quit(1)
+
+quit(0)

cmake_targets/autotests/v2/actions/wvdial.bandrich.conf

+[Dialer Defaults]
+Modem = /dev/bandrich.data
+ISDN = off
+Modem Type = Analog Modem
+Baud = 9600
+Init = ATZ
+Init2 = AT+CPIN?
+Init3 = AT+CGREG?
+Init4 = AT+COPS?
+Init5 = AT+CSQ
+Init6 =
+Init7 = AT+CGATT=1
+Init8 = 
+Init9 = 
+Phone = *99***1#
+Phone1 = 
+Phone2 = 
+Phone3 = 
+Phone4 = 
+Dial Prefix = 
+Dial Attempts = 1
+Dial Command = ATM1L3DT
+Ask Password = off
+Password = ''
+Username = ImaginLab
+Auto Reconnect = off
+Abort on Busy = off
+Carrier Check = on
+Check Def Route = on
+Abort on No Dialtone = on
+Stupid Mode = on
+Idle Seconds = 0
+Auto DNS = on
+;Minimize = off
+;Dock = off
+;Do NOT edit this file by hand!

cmake_targets/autotests/v2/alu_test.py

+import threading, os, re
+
+from utils import quickshell, log, TestFailed, do_tests
+from task import Task, WAITLOG_SUCCESS
+
+class alu_test:
+    def __init__(self,
+                 epc, enb, ue,
+                 openair,
+                 user, password,
+                 log_subdir,
+                 env):
+        self.epc_machine  = epc
+        self.enb_machine  = enb
+        self.ue_machine   = ue
+        self.openair_dir  = openair
+        self.oai_user     = user
+        self.oai_password = password
+        self.env          = env
+
+        self.task_hss = None
+        self.task_enb = None
+        self.task_ue  = None
+
+        self.logdir = openair + '/cmake_targets/autotests/log/' + log_subdir
+        quickshell('mkdir -p ' + self.logdir)
+
+        #event object used to wait for several tasks at once
+        self.event = threading.Event()
+
+    ##########################################################################
+