lte-softmodem.c

/*******************************************************************************
    OpenAirInterface
    Copyright(c) 1999 - 2014 Eurecom

    OpenAirInterface is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.


    OpenAirInterface is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with OpenAirInterface.The full GNU General Public License is
    included in this distribution in the file called "COPYING". If not,
    see <http://www.gnu.org/licenses/>.

   Contact Information
   OpenAirInterface Admin: openair_admin@eurecom.fr
   OpenAirInterface Tech : openair_tech@eurecom.fr
   OpenAirInterface Dev  : openair4g-devel@eurecom.fr

   Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

*******************************************************************************/

/*! \file lte-softmodem.c
 * \brief main program to control HW and scheduling
 * \author R. Knopp, F. Kaltenberger, Navid Nikaein
 * \date 2012
 * \version 0.1
 * \company Eurecom
 * \email: knopp@eurecom.fr,florian.kaltenberger@eurecom.fr, navid.nikaein@eurecom.fr
 * \note
 * \warning
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sched.h>
#include <linux/sched.h>
#include <signal.h>
#include <execinfo.h>
#include <getopt.h>
#include <syscall.h>

#include "rt_wrapper.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all

#include "assertions.h"

#ifdef EMOS
#include <gps.h>
struct gps_fix_t dummy_gps_data;
#endif

#include "PHY/types.h"

#include "PHY/defs.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#ifdef EXMIMO
#include "openair0_lib.h"
#else
#include "../../ARCH/COMMON/common_lib.h"
#endif

//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#include "PHY/vars.h"
#include "MAC_INTERFACE/vars.h"
//#include "SCHED/defs.h"
#include "SCHED/vars.h"
#include "LAYER2/MAC/vars.h"

#include "../../SIMU/USER/init_lte.h"

#ifdef EMOS
#include "SCHED/phy_procedures_emos.h"
#endif

#ifdef OPENAIR2
#include "LAYER2/MAC/defs.h"
#include "LAYER2/MAC/vars.h"
#include "LAYER2/MAC/proto.h"
#include "RRC/LITE/vars.h"
#include "PHY_INTERFACE/vars.h"
#endif

#ifdef SMBV
#include "PHY/TOOLS/smbv.h"
unsigned short config_frames[4] = {2,9,11,13};
#endif
#include "UTIL/LOG/log_extern.h"
#include "UTIL/OTG/otg_tx.h"
#include "UTIL/OTG/otg_externs.h"
#include "UTIL/MATH/oml.h"
#include "UTIL/LOG/vcd_signal_dumper.h"
#include "UTIL/OPT/opt.h"
#include "enb_config.h"
//#include "PHY/TOOLS/time_meas.h"

#ifndef OPENAIR2
#include "UTIL/OTG/otg_vars.h"
#endif

#if defined(ENABLE_ITTI)
# include "intertask_interface_init.h"
# include "create_tasks.h"
# if defined(ENABLE_USE_MME)
#   include "s1ap_eNB.h"
#ifdef NAS_NETLINK
#   include "SIMULATION/ETH_TRANSPORT/proto.h"
#endif
# endif
#endif

#ifdef XFORMS
#include "PHY/TOOLS/lte_phy_scope.h"
#include "stats.h"
#endif

#define FRAME_PERIOD    100000000ULL
#define DAQ_PERIOD      66667ULL

//#define DEBUG_THREADS 1

//#define USRP_DEBUG 1

struct timing_info_t {
  //unsigned int frame, hw_slot, last_slot, next_slot;
  RTIME time_min, time_max, time_avg, time_last, time_now;
  //unsigned int mbox0, mbox1, mbox2, mbox_target;
  unsigned int n_samples;
} timing_info;



openair0_config_t openair0_cfg[MAX_CARDS];

int32_t **rxdata;
int32_t **txdata;
int setup_ue_buffers(PHY_VARS_UE **phy_vars_ue, openair0_config_t *openair0_cfg, openair0_rf_map rf_map[MAX_NUM_CCs]);
int setup_eNB_buffers(PHY_VARS_eNB **phy_vars_eNB, openair0_config_t *openair0_cfg, openair0_rf_map rf_map[MAX_NUM_CCs]);

void fill_ue_band_info(void);
#ifdef XFORMS
// current status is that every UE has a DL scope for a SINGLE eNB (eNB_id=0)
// at eNB 0, an UL scope for every UE
FD_lte_phy_scope_ue  *form_ue[NUMBER_OF_UE_MAX];
FD_lte_phy_scope_enb *form_enb[NUMBER_OF_UE_MAX];
FD_stats_form                  *form_stats=NULL,*form_stats_l2=NULL;
char title[255];
unsigned char                   scope_enb_num_ue = 1;
#endif //XFORMS

#ifdef RTAI

static long                      main_eNB_thread;
static long                      main_ue_thread;

#else
pthread_t                       main_eNB_thread;
pthread_t                       main_ue_thread;

pthread_attr_t                  attr_dlsch_threads;
pthread_attr_t                  attr_UE_thread;

#ifndef LOWLATENCY
struct sched_param              sched_param_dlsch;
#endif
#endif

pthread_cond_t sync_cond;
pthread_mutex_t sync_mutex;
int sync_var=-1;




struct sched_param              sched_param_UE_thread;

pthread_attr_t                  attr_eNB_proc_tx[MAX_NUM_CCs][NUM_ENB_THREADS];
pthread_attr_t                  attr_eNB_proc_rx[MAX_NUM_CCs][NUM_ENB_THREADS];
#ifndef LOWLATENCY
struct sched_param              sched_param_eNB_proc_tx[MAX_NUM_CCs][NUM_ENB_THREADS];
struct sched_param              sched_param_eNB_proc_rx[MAX_NUM_CCs][NUM_ENB_THREADS];
#endif
#ifdef XFORMS
static pthread_t                forms_thread; //xforms
#endif
#ifdef EMOS
static pthread_t                thread3; //emos
static pthread_t                thread4; //GPS
#endif

openair0_device openair0;


/*
  static int instance_cnt=-1; //0 means worker is busy, -1 means its free
  int instance_cnt_ptr_kern,*instance_cnt_ptr_user;
  int pci_interface_ptr_kern;
*/
//extern unsigned int bigphys_top;
//extern unsigned int mem_base;

int                             card = 0;


#if defined(ENABLE_ITTI)
static volatile int             start_eNB = 0;
static volatile int             start_UE = 0;
#endif
volatile int                    oai_exit = 0;

//static int                      time_offset[4] = {-138,-138,-138,-138};
//static int                      time_offset[4] = {-145,-145,-145,-145};
static int                      time_offset[4] = {0,0,0,0};


static char                     UE_flag=0;
//static uint8_t                  eNB_id=0,UE_id=0;


uint32_t                 downlink_frequency[MAX_NUM_CCs][4];
int32_t                  uplink_frequency_offset[MAX_NUM_CCs][4];

openair0_rf_map rf_map[MAX_NUM_CCs];

static char                    *conf_config_file_name = NULL;
#if defined(ENABLE_ITTI)
static char                    *itti_dump_file = NULL;
#endif

int UE_scan = 1;
runmode_t mode = normal_txrx;


#ifdef EXMIMO
double tx_gain[MAX_NUM_CCs][4] = {{20,20,0,0},{20,20,0,0}};
double rx_gain[MAX_NUM_CCs][4] = {{20,20,0,0},{20,20,0,0}};
// these are for EXMIMO2 target only
/*
  static unsigned int             rxg_max[4] =    {133,133,133,133};
  static unsigned int             rxg_med[4] =    {127,127,127,127};
  static unsigned int             rxg_byp[4] =    {120,120,120,120};
*/
// these are for EXMIMO2 card 39
unsigned int             rxg_max[4] =    {128,128,128,126};
unsigned int             rxg_med[4] =    {122,123,123,120};
unsigned int             rxg_byp[4] =    {116,117,116,116};
unsigned int             nf_max[4] =    {7,9,16,12};
unsigned int             nf_med[4] =    {12,13,22,17};
unsigned int             nf_byp[4] =    {15,20,29,23};
rx_gain_t                rx_gain_mode[MAX_NUM_CCs][4] = {{max_gain,max_gain,max_gain,max_gain},{max_gain,max_gain,max_gain,max_gain}};
#else
double tx_gain[MAX_NUM_CCs][4] = {{20,0,0,0}};
double rx_gain[MAX_NUM_CCs][4] = {{110,0,0,0}};
#endif

double sample_rate=30.72e6;
double bw = 14e6;

static int                      tx_max_power[MAX_NUM_CCs]; /* =  {0,0}*/;

#ifndef EXMIMO
char ref[128] = "internal";
char channels[128] = "0";

unsigned int samples_per_frame = 307200;
unsigned int tx_forward_nsamps;
int tx_delay;

#endif

int                      rx_input_level_dBm;
static int                      online_log_messages=0;
#ifdef XFORMS
extern int                      otg_enabled;
static char                     do_forms=0;
#else
int                             otg_enabled;
#endif
//int                             number_of_cards =   1;
#ifdef EXMIMO
static int                      mbox_bounds[20] =   {8,16,24,30,38,46,54,60,68,76,84,90,98,106,114,120,128,136,144, 0}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
//static int                      mbox_bounds[20] =   {6,14,22,28,36,44,52,58,66,74,82,88,96,104,112,118,126,134,142, 148}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
#endif

static LTE_DL_FRAME_PARMS      *frame_parms[MAX_NUM_CCs];

int multi_thread=1;
uint32_t target_dl_mcs = 28; //maximum allowed mcs
uint32_t target_ul_mcs = 10;
uint8_t exit_missed_slots=1;
uint64_t num_missed_slots=0; // counter for the number of missed slots


time_stats_t softmodem_stats_mt; // main thread
time_stats_t softmodem_stats_hw; //  hw acquisition
time_stats_t softmodem_stats_tx_sf[10]; // total tx time
time_stats_t softmodem_stats_rx_sf[10]; // total rx time
void reset_opp_meas(void);
void print_opp_meas(void);
int transmission_mode=1;


int16_t           glog_level         = LOG_INFO;
int16_t           glog_verbosity     = LOG_MED;
int16_t           hw_log_level       = LOG_INFO;
int16_t           hw_log_verbosity   = LOG_MED;
int16_t           phy_log_level      = LOG_INFO;
int16_t           phy_log_verbosity  = LOG_MED;
int16_t           mac_log_level      = LOG_INFO;
int16_t           mac_log_verbosity  = LOG_MED;
int16_t           rlc_log_level      = LOG_INFO;
int16_t           rlc_log_verbosity  = LOG_MED;
int16_t           pdcp_log_level     = LOG_INFO;
int16_t           pdcp_log_verbosity = LOG_MED;
int16_t           rrc_log_level      = LOG_INFO;
int16_t           rrc_log_verbosity  = LOG_MED;
int16_t           opt_log_level      = LOG_INFO;
int16_t           opt_log_verbosity  = LOG_MED;

# if defined(ENABLE_USE_MME)
int16_t           gtpu_log_level     = LOG_DEBUG;
int16_t           gtpu_log_verbosity = LOG_MED;
int16_t           udp_log_level      = LOG_DEBUG;
int16_t           udp_log_verbosity  = LOG_MED;
#endif
#if defined (ENABLE_SECURITY)
int16_t           osa_log_level      = LOG_INFO;
int16_t           osa_log_verbosity  = LOG_MED;
#endif


#ifdef ETHERNET
char rrh_eNB_ip[20] = "127.0.0.1";
int rrh_eNB_port = 50000;
char *rrh_UE_ip = "127.0.0.1";
int rrh_UE_port = 51000;
#endif

char uecap_xer[1024],uecap_xer_in=0;
extern void *UE_thread(void *arg);
extern void init_UE_threads(void);

unsigned int build_rflocal(int txi, int txq, int rxi, int rxq)
{
  return (txi + (txq<<6) + (rxi<<12) + (rxq<<18));
}
unsigned int build_rfdc(int dcoff_i_rxfe, int dcoff_q_rxfe)
{
  return (dcoff_i_rxfe + (dcoff_q_rxfe<<8));
}

#ifdef LOWLATENCY
int sched_setattr(pid_t pid, const struct sched_attr *attr, unsigned int flags)
{

  return syscall(__NR_sched_setattr, pid, attr, flags);
}


int sched_getattr(pid_t pid,struct sched_attr *attr,unsigned int size, unsigned int flags)
{

  return syscall(__NR_sched_getattr, pid, attr, size, flags);
}
#endif
#if !defined(ENABLE_ITTI)
void signal_handler(int sig)
{
  void *array[10];
  size_t size;

  if (sig==SIGSEGV) {
    // get void*'s for all entries on the stack
    size = backtrace(array, 10);

    // print out all the frames to stderr
    fprintf(stderr, "Error: signal %d:\n", sig);
    backtrace_symbols_fd(array, size, 2);
    exit(-1);
  } else {
    printf("trying to exit gracefully...\n");
    oai_exit = 1;
  }
}
#endif

void exit_fun(const char* s)
{
  if (s != NULL) {
    printf("%s %s() Exiting OAI softmodem: %s\n",__FILE__, __FUNCTION__, s);
  }

  oai_exit = 1;

#if defined(ENABLE_ITTI)
  sleep(1); //allow lte-softmodem threads to exit first
  itti_terminate_tasks (TASK_UNKNOWN);
#endif

  //rt_sleep_ns(FRAME_PERIOD);

  //exit (-1);
}

static int latency_target_fd = -1;
static int32_t latency_target_value = 0;
/* Latency trick - taken from cyclictest.c
 * if the file /dev/cpu_dma_latency exists,
 * open it and write a zero into it. This will tell
 * the power management system not to transition to
 * a high cstate (in fact, the system acts like idle=poll)
 * When the fd to /dev/cpu_dma_latency is closed, the behavior
 * goes back to the system default.
 *
 * Documentation/power/pm_qos_interface.txt
 */
static void set_latency_target(void)
{
  struct stat s;
  int ret;

  if (stat("/dev/cpu_dma_latency", &s) == 0) {
    latency_target_fd = open("/dev/cpu_dma_latency", O_RDWR);

    if (latency_target_fd == -1)
      return;

    ret = write(latency_target_fd, &latency_target_value, 4);

    if (ret == 0) {
      printf("# error setting cpu_dma_latency to %d!: %s\n", latency_target_value, strerror(errno));
      close(latency_target_fd);
      return;
    }

    printf("# /dev/cpu_dma_latency set to %dus\n", latency_target_value);
  }
}

#ifdef XFORMS

void reset_stats(FL_OBJECT *button, long arg)
{
  int i,j,k;
  PHY_VARS_eNB *phy_vars_eNB = PHY_vars_eNB_g[0][0];

  for (i=0; i<NUMBER_OF_UE_MAX; i++) {
    for (k=0; k<8; k++) { //harq_processes
      for (j=0; j<phy_vars_eNB->dlsch_eNB[i][0]->Mdlharq; j++) {
        phy_vars_eNB->eNB_UE_stats[i].dlsch_NAK[k][j]=0;
        phy_vars_eNB->eNB_UE_stats[i].dlsch_ACK[k][j]=0;
        phy_vars_eNB->eNB_UE_stats[i].dlsch_trials[k][j]=0;
      }

      phy_vars_eNB->eNB_UE_stats[i].dlsch_l2_errors[k]=0;
      phy_vars_eNB->eNB_UE_stats[i].ulsch_errors[k]=0;
      phy_vars_eNB->eNB_UE_stats[i].ulsch_consecutive_errors=0;

      for (j=0; j<phy_vars_eNB->ulsch_eNB[i]->Mdlharq; j++) {
        phy_vars_eNB->eNB_UE_stats[i].ulsch_decoding_attempts[k][j]=0;
        phy_vars_eNB->eNB_UE_stats[i].ulsch_decoding_attempts_last[k][j]=0;
        phy_vars_eNB->eNB_UE_stats[i].ulsch_round_errors[k][j]=0;
        phy_vars_eNB->eNB_UE_stats[i].ulsch_round_fer[k][j]=0;
      }
    }

    phy_vars_eNB->eNB_UE_stats[i].dlsch_sliding_cnt=0;
    phy_vars_eNB->eNB_UE_stats[i].dlsch_NAK_round0=0;
    phy_vars_eNB->eNB_UE_stats[i].dlsch_mcs_offset=0;
  }
}

static void *scope_thread(void *arg)
{
  char stats_buffer[16384];
# ifdef ENABLE_XFORMS_WRITE_STATS
  FILE *UE_stats, *eNB_stats;
# endif
  int len = 0;
  struct sched_param sched_param;
  int UE_id;

  sched_param.sched_priority = sched_get_priority_min(SCHED_FIFO)+1;
  sched_setscheduler(0, SCHED_FIFO,&sched_param);

  printf("Scope thread has priority %d\n",sched_param.sched_priority);

# ifdef ENABLE_XFORMS_WRITE_STATS

  if (UE_flag==1)
    UE_stats  = fopen("UE_stats.txt", "w");
  else
    eNB_stats = fopen("eNB_stats.txt", "w");

#endif

  while (!oai_exit) {
    if (UE_flag==1) {
      len = dump_ue_stats (PHY_vars_UE_g[0][0], stats_buffer, 0, mode,rx_input_level_dBm);
      fl_set_object_label(form_stats->stats_text, stats_buffer);

      phy_scope_UE(form_ue[0],
                   PHY_vars_UE_g[0][0],
                   0,
                   0,7);

    } else {
#ifdef OPENAIR2
      len = dump_eNB_l2_stats (stats_buffer, 0);
      fl_set_object_label(form_stats_l2->stats_text, stats_buffer);
#endif
      len = dump_eNB_stats (PHY_vars_eNB_g[0][0], stats_buffer, 0);

      if (MAX_NUM_CCs>1)
        len += dump_eNB_stats (PHY_vars_eNB_g[0][1], &stats_buffer[len], 0);

      fl_set_object_label(form_stats->stats_text, stats_buffer);

      for(UE_id=0; UE_id<scope_enb_num_ue; UE_id++) {
        phy_scope_eNB(form_enb[UE_id],
                      PHY_vars_eNB_g[0][0],
                      UE_id);
      }

    }

    //printf("doing forms\n");
    //usleep(100000); // 100 ms
    sleep(1);
  }

  printf("%s",stats_buffer);

# ifdef ENABLE_XFORMS_WRITE_STATS

  if (UE_flag==1) {
    if (UE_stats) {
      rewind (UE_stats);
      fwrite (stats_buffer, 1, len, UE_stats);
      fclose (UE_stats);
    }
  } else {
    if (eNB_stats) {
      rewind (eNB_stats);
      fwrite (stats_buffer, 1, len, eNB_stats);
      fclose (eNB_stats);
    }
  }

# endif

  pthread_exit((void*)arg);
}
#endif

#ifdef EMOS
#define NO_ESTIMATES_DISK 100 //No. of estimates that are aquired before dumped to disk

void* gps_thread (void *arg)
{

  struct gps_data_t gps_data;
  struct gps_data_t *gps_data_ptr = &gps_data;
  struct sched_param sched_param;
  int ret;

  sched_param.sched_priority = sched_get_priority_min(SCHED_FIFO)+1;
  sched_setscheduler(0, SCHED_FIFO,&sched_param);

  printf("GPS thread has priority %d\n",sched_param.sched_priority);

  memset(&dummy_gps_data,0,sizeof(struct gps_fix_t));

#if GPSD_API_MAJOR_VERSION>=5
  ret = gps_open("127.0.0.1","2947",gps_data_ptr);

  if (ret!=0)
#else
  gps_data_ptr = gps_open("127.0.0.1","2947");

  if (gps_data_ptr == NULL)
#endif
  {
    printf("[EMOS] Could not open GPS\n");
    pthread_exit((void*)arg);
  }

#if GPSD_API_MAJOR_VERSION>=4
  else if (gps_stream(gps_data_ptr, WATCH_ENABLE,NULL) != 0)
#else
  else if (gps_query(gps_data_ptr, "w+x") != 0)
#endif
  {
    printf("[EMOS] Error sending command to GPS\n");
    pthread_exit((void*) arg);
  } else
    printf("[EMOS] Opened GPS, gps_data=%p\n", gps_data_ptr);


  while (!oai_exit) {
    printf("[EMOS] polling data from gps\n");
#if GPSD_API_MAJOR_VERSION>=5

    if (gps_waiting(gps_data_ptr,500)) {
      if (gps_read(gps_data_ptr) <= 0) {
#else

    if (gps_waiting(gps_data_ptr)) {
      if (gps_poll(gps_data_ptr) != 0) {
#endif
        printf("[EMOS] problem polling data from gps\n");
      } else {
        memcpy(&dummy_gps_data,&(gps_data_ptr->fix),sizeof(struct gps_fix_t));
        printf("[EMOS] lat %g, lon %g\n",gps_data_ptr->fix.latitude,gps_data_ptr->fix.longitude);
      }
    } //gps_waiting
    else {
      printf("[EMOS] WARNING: No GPS data available, storing dummy packet\n");
    }

    //rt_sleep_ns(1000000000LL);
    sleep(1);
  } //oai_exit

  pthread_exit((void*) arg);

}

void *emos_thread (void *arg)
{
  char c;
  char *fifo2file_buffer, *fifo2file_ptr;

  int fifo, counter=0, bytes;

  FILE  *dumpfile_id;
  char  dumpfile_name[1024];
  time_t starttime_tmp;
  struct tm starttime;

  int channel_buffer_size,ret;

  time_t timer;
  struct tm *now;

  struct sched_param sched_param;

  sched_param.sched_priority = sched_get_priority_max(SCHED_FIFO)-1;
  sched_setscheduler(0, SCHED_FIFO,&sched_param);

  printf("EMOS thread has priority %d\n",sched_param.sched_priority);

  timer = time(NULL);
  now = localtime(&timer);

  if (UE_flag==0)
    channel_buffer_size = sizeof(fifo_dump_emos_eNB);
  else
    channel_buffer_size = sizeof(fifo_dump_emos_UE);

  // allocate memory for NO_FRAMES_DISK channes estimations
  fifo2file_buffer = malloc(NO_ESTIMATES_DISK*channel_buffer_size);
  fifo2file_ptr = fifo2file_buffer;

  if (fifo2file_buffer == NULL) {
    printf("[EMOS] Cound not allocate memory for fifo2file_buffer\n");
    exit(EXIT_FAILURE);
  }

  if ((fifo = open(CHANSOUNDER_FIFO_DEV, O_RDONLY)) < 0) {
    fprintf(stderr, "[EMOS] Error opening the fifo\n");
    exit(EXIT_FAILURE);
  }


  time(&starttime_tmp);
  localtime_r(&starttime_tmp,&starttime);
  snprintf(dumpfile_name,1024,"/tmp/%s_data_%d%02d%02d_%02d%02d%02d.EMOS",
           (UE_flag==0) ? "eNB" : "UE",
           1900+starttime.tm_year, starttime.tm_mon+1, starttime.tm_mday, starttime.tm_hour, starttime.tm_min, starttime.tm_sec);

  dumpfile_id = fopen(dumpfile_name,"w");

  if (dumpfile_id == NULL) {
    fprintf(stderr, "[EMOS] Error opening dumpfile %s\n",dumpfile_name);
    exit(EXIT_FAILURE);
  }


  printf("[EMOS] starting dump, channel_buffer_size=%d, fifo %d\n",channel_buffer_size,fifo);

  while (!oai_exit) {
    /*
    bytes = rtf_read_timed(fifo, fifo2file_ptr, channel_buffer_size,100);
    if (bytes==0)
    continue;
    */
    bytes = rtf_read_all_at_once(fifo, fifo2file_ptr, channel_buffer_size);

    if (bytes<=0) {
      usleep(100);
      continue;
    }

    if (bytes != channel_buffer_size) {
      printf("[EMOS] ERROR! Only got %d bytes instead of %d!\n",bytes,channel_buffer_size);
    }

    /*
    if (UE_flag==0)
    printf("eNB: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx,bytes);
    else
    printf("UE: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx,bytes);
    */

    fifo2file_ptr += channel_buffer_size;
    counter ++;

    if (counter == NO_ESTIMATES_DISK) {
      //reset stuff
      fifo2file_ptr = fifo2file_buffer;
      counter = 0;

      //flush buffer to disk
      if (UE_flag==0)
        printf("[EMOS] eNB: count %d, frame %d, flushing buffer to disk\n",
               counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx);
      else
        printf("[EMOS] UE: count %d, frame %d, flushing buffer to disk\n",
               counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx);


      if (fwrite(fifo2file_buffer, sizeof(char), NO_ESTIMATES_DISK*channel_buffer_size, dumpfile_id) != NO_ESTIMATES_DISK*channel_buffer_size) {
        fprintf(stderr, "[EMOS] Error writing to dumpfile\n");
        exit(EXIT_FAILURE);
      }

      if (fwrite(&dummy_gps_data, sizeof(char), sizeof(struct gps_fix_t), dumpfile_id) != sizeof(struct gps_fix_t)) {
        printf("[EMOS] Error writing to dumpfile, stopping recording\n");
        exit(EXIT_FAILURE);
      }
    }
  }

  free(fifo2file_buffer);
  fclose(dumpfile_id);
  close(fifo);

  pthread_exit((void*) arg);

}
#endif



#if defined(ENABLE_ITTI)
static void wait_system_ready (char *message, volatile int *start_flag)
{
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  {
    static char *indicator[] = {".    ", "..   ", "...  ", ".... ", ".....",
                                " ....", "  ...", "   ..", "    .", "     "
                               };
    int i = 0;

    while ((!oai_exit) && (*start_flag == 0)) {
      LOG_N(EMU, message, indicator[i]);
      i = (i + 1) % (sizeof(indicator) / sizeof(indicator[0]));
      usleep(200000);
    }

    LOG_D(EMU,"\n");
  }
}
#endif

#if defined(ENABLE_ITTI)
void *l2l1_task(void *arg)
{
  MessageDef *message_p = NULL;
  int         result;

  itti_set_task_real_time(TASK_L2L1);
  itti_mark_task_ready(TASK_L2L1);

  if (UE_flag == 0) {
    /* Wait for the initialize message */
    do {
      if (message_p != NULL) {
        result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
        AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
      }

      itti_receive_msg (TASK_L2L1, &message_p);

      switch (ITTI_MSG_ID(message_p)) {
      case INITIALIZE_MESSAGE:
        /* Start eNB thread */
        LOG_D(EMU, "L2L1 TASK received %s\n", ITTI_MSG_NAME(message_p));
        start_eNB = 1;
        break;

      case TERMINATE_MESSAGE:
        printf("received terminate message\n");
        oai_exit=1;
        itti_exit_task ();
        break;

      default:
        LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
        break;
      }
    } while (ITTI_MSG_ID(message_p) != INITIALIZE_MESSAGE);

    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  }

  do {
    // Wait for a message
    itti_receive_msg (TASK_L2L1, &message_p);

    switch (ITTI_MSG_ID(message_p)) {
    case TERMINATE_MESSAGE:
      oai_exit=1;
      itti_exit_task ();
      break;

    case ACTIVATE_MESSAGE:
      start_UE = 1;
      break;

    case DEACTIVATE_MESSAGE:
      start_UE = 0;
      break;

    case MESSAGE_TEST:
      LOG_I(EMU, "Received %s\n", ITTI_MSG_NAME(message_p));
      break;

    default:
      LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
      break;
    }

    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  } while(!oai_exit);

  return NULL;
}
#endif


void do_OFDM_mod_rt(int subframe,PHY_VARS_eNB *phy_vars_eNB)
{

  unsigned int aa,slot_offset, slot_offset_F;
  int dummy_tx_b[7680*4] __attribute__((aligned(16)));
  int i, tx_offset;
  int slot_sizeF = (phy_vars_eNB->lte_frame_parms.ofdm_symbol_size)*
                   ((phy_vars_eNB->lte_frame_parms.Ncp==1) ? 6 : 7);

  slot_offset_F = (subframe<<1)*slot_sizeF;

  slot_offset = subframe*phy_vars_eNB->lte_frame_parms.samples_per_tti;

  if ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_DL)||
      ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_S))) {
    //    LOG_D(HW,"Frame %d: Generating slot %d\n",frame,next_slot);


    for (aa=0; aa<phy_vars_eNB->lte_frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->lte_frame_parms.Ncp == EXTENDED) {
        PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
                     dummy_tx_b,
                     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
                     6,
                     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
                     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
                     phy_vars_eNB->lte_frame_parms.rev,
                     CYCLIC_PREFIX);
        PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
                     dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
                     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
                     6,
                     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
                     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
                     phy_vars_eNB->lte_frame_parms.rev,
                     CYCLIC_PREFIX);
      } else {
        normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
                          dummy_tx_b,
                          7,
                          &(phy_vars_eNB->lte_frame_parms));
        normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
                          dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
                          7,
                          &(phy_vars_eNB->lte_frame_parms));
      }

      for (i=0; i<phy_vars_eNB->lte_frame_parms.samples_per_tti; i++) {
        tx_offset = (int)slot_offset+time_offset[aa]+i;

        if (tx_offset<0)
          tx_offset += LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;

        if (tx_offset>=(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti))
          tx_offset -= LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;

        ((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[0]=
#ifdef EXMIMO
          ((short*)dummy_tx_b)[2*i]<<4;
#else
          ((short*)dummy_tx_b)[2*i]<<5;
#endif
        ((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[1]=
#ifdef EXMIMO
          ((short*)dummy_tx_b)[2*i+1]<<4;
#else
          ((short*)dummy_tx_b)[2*i+1]<<5;
#endif
      }
    }
  }
}


/*!
 * \brief The transmit thread of eNB.
 * \ref NUM_ENB_THREADS threads of this type are active at the same time.
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
static void* eNB_thread_tx( void* param )
{
  static int eNB_thread_tx_status[NUM_ENB_THREADS];

  eNB_proc_t *proc = (eNB_proc_t*)param;

  // set default return value
  eNB_thread_tx_status[proc->subframe] = 0;

#ifdef RTAI
  RT_TASK *task;
  char task_name[8];

  sprintf(task_name,"TXC%dS%d",proc->CC_id,proc->subframe);
  task = rt_task_init_schmod(nam2num(task_name), 0, 0, 0, SCHED_FIFO, 0xF);

  if (task == NULL) {
    LOG_E(PHY,"[SCHED][eNB] Problem starting eNB_proc_TX thread_index %d (%s)!!!!\n",proc->subframe,task_name);
    return 0;
  } else {
    LOG_I(PHY,"[SCHED][eNB] eNB TX thread CC %d SF %d started with id %p\n",
          proc->CC_id,
          proc->subframe,
          task);
  }

#else
#ifdef LOWLATENCY
  struct sched_attr attr;
  unsigned int flags = 0;

  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  // This creates a 1ms reservation every 10ms period
  attr.sched_policy   = SCHED_DEADLINE;
  attr.sched_runtime  = 0.9 *  1000000; // each tx thread requires 1ms to finish its job
  attr.sched_deadline = 1   *  1000000; // each tx thread will finish within 1ms
  attr.sched_period   = 1   * 10000000; // each tx thread has a period of 10ms from the starting point

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB tx thread: sched_setattr failed\n");
    return &eNB_thread_tx_status[proc->subframe];
  }

  LOG_I( HW, "[SCHED] eNB TX deadline thread %d(id %ld) started on CPU %d\n", proc->subframe, gettid(), sched_getcpu() );
#else
  LOG_I( HW, "[SCHED][eNB] TX thread %d started on CPU %d\n", proc->subframe, sched_getcpu() );