lte-softmodem.c

/*******************************************************************************
    OpenAirInterface 
    Copyright(c) 1999 - 2014 Eurecom

    OpenAirInterface is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.


    OpenAirInterface is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with OpenAirInterface.The full GNU General Public License is 
    included in this distribution in the file called "COPYING". If not, 
    see <http://www.gnu.org/licenses/>.

   Contact Information
   OpenAirInterface Admin: openair_admin@eurecom.fr
   OpenAirInterface Tech : openair_tech@eurecom.fr
   OpenAirInterface Dev  : openair4g-devel@eurecom.fr
  
   Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

 *******************************************************************************/

/*! \file lte-softmodem.c
 * \brief main program to control HW and scheduling
 * \author R. Knopp, F. Kaltenberger, Navid Nikaein
 * \date 2012
 * \version 0.1
 * \company Eurecom
 * \email: knopp@eurecom.fr,florian.kaltenberger@eurecom.fr, navid.nikaein@eurecom.fr
 * \note
 * \warning
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sched.h>
#include <signal.h>
#include <execinfo.h>
#include <getopt.h>

#include "rt_wrapper.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all

#ifdef USRP
static int hw_subframe;
#endif

#include "assertions.h"

#ifdef EMOS
#include <gps.h>
#endif

#include "PHY/types.h"

#include "PHY/defs.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#ifndef USRP
#include "openair0_lib.h"
#else
#include "../../ARCH/COMMON/common_lib.h"
#endif

#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#include "PHY/vars.h"
#include "MAC_INTERFACE/vars.h"
//#include "SCHED/defs.h"
#include "SCHED/vars.h"
#include "LAYER2/MAC/vars.h"

#include "../../SIMU/USER/init_lte.h"

#ifdef EMOS
#include "SCHED/phy_procedures_emos.h"
#endif

#ifdef OPENAIR2
#include "otg_tx.h"
#include "LAYER2/MAC/defs.h"
#include "LAYER2/MAC/vars.h"
#include "LAYER2/MAC/proto.h"
#ifndef CELLULAR
#include "RRC/LITE/vars.h"
#endif
#include "PHY_INTERFACE/vars.h"
#endif

#ifdef SMBV
#include "PHY/TOOLS/smbv.h"
unsigned short config_frames[4] = {2,9,11,13};
#endif
#include "UTIL/LOG/log_extern.h"
#include "UTIL/OTG/otg.h"
#include "UTIL/OTG/otg_vars.h"
#include "UTIL/MATH/oml.h"
#include "UTIL/LOG/vcd_signal_dumper.h"
#include "enb_config.h"

#if defined(ENABLE_ITTI)
# include "intertask_interface_init.h"
# include "create_tasks.h"
# if defined(ENABLE_USE_MME)
#   include "s1ap_eNB.h"
# endif
#endif

#ifdef XFORMS
#include "PHY/TOOLS/lte_phy_scope.h"
#include "stats.h"
#endif

#define FRAME_PERIOD    100000000ULL
#define DAQ_PERIOD      66667ULL

#define DEBUG_THREADS 1

struct timing_info_t {
  //unsigned int frame, hw_slot, last_slot, next_slot;
  RTIME time_min, time_max, time_avg, time_last, time_now;
  //unsigned int mbox0, mbox1, mbox2, mbox_target;
  unsigned int n_samples;
} timing_info;

extern int16_t* sync_corr_ue0;
extern int16_t prach_ifft[4][1024*2];

int init_dlsch_threads(void);
void cleanup_dlsch_threads(void);
int32_t init_rx_pdsch_thread(void);
void cleanup_rx_pdsch_thread(void);

openair0_config_t openair0_cfg[MAX_CARDS];

int32_t *rxdata;
int32_t *txdata;
int setup_ue_buffers(PHY_VARS_UE **phy_vars_ue, openair0_config_t *openair0_cfg, openair0_rf_map rf_map[MAX_NUM_CCs]);
int setup_eNB_buffers(PHY_VARS_eNB **phy_vars_eNB, openair0_config_t *openair0_cfg, openair0_rf_map rf_map[MAX_NUM_CCs]);

#ifdef XFORMS
// current status is that every UE has a DL scope for a SINGLE eNB (eNB_id=0)
// at eNB 0, an UL scope for every UE 
FD_lte_phy_scope_ue  *form_ue[NUMBER_OF_UE_MAX];
FD_lte_phy_scope_enb *form_enb[NUMBER_OF_UE_MAX];
FD_stats_form                  *form_stats=NULL;
char title[255];
unsigned char                   scope_enb_num_ue = 1;
#endif //XFORMS

#ifdef RTAI
static SEM                     *mutex;
//static CND *cond;

static int                      main_eNB_thread;
static int                      main_ue_thread;
#ifdef USRP
static SEM *sync_sem; // to sync rx & tx streaming
#endif
//static int sync_thread;
#else
pthread_t                       main_eNB_thread;
pthread_t                       main_ue_thread;
pthread_attr_t                  attr_dlsch_threads;
struct sched_param              sched_param_dlsch;

#ifdef USRP
pthread_cond_t sync_cond;
pthread_mutex_t sync_mutex;
#endif
#endif

pthread_attr_t                  attr_UE_init_synch;
pthread_attr_t                  attr_UE_thread_tx;
pthread_attr_t                  attr_UE_thread_rx;
pthread_attr_t                  attr_eNB_proc_tx[MAX_NUM_CCs][10];
pthread_attr_t                  attr_eNB_proc_rx[MAX_NUM_CCs][10];
struct sched_param              sched_param_UE_init_synch;
struct sched_param              sched_param_UE_thread_tx;
struct sched_param              sched_param_UE_thread_rx;
struct sched_param              sched_param_eNB_proc_tx[MAX_NUM_CCs][10];
struct sched_param              sched_param_eNB_proc_rx[MAX_NUM_CCs][10];

#ifdef XFORMS
static pthread_t                forms_thread; //xforms
#endif
#ifdef EMOS
static pthread_t                thread3; //emos
#endif

openair0_device openair0;
openair0_timestamp timestamp;

/*
  static int instance_cnt=-1; //0 means worker is busy, -1 means its free
  int instance_cnt_ptr_kern,*instance_cnt_ptr_user;
  int pci_interface_ptr_kern;
*/
//extern unsigned int bigphys_top;
//extern unsigned int mem_base;

int                             card = 0;


#if defined(ENABLE_ITTI)
static volatile int             start_eNB = 0;
static volatile int             start_UE = 0;
#endif
volatile int                    oai_exit = 0;

//static int                      time_offset[4] = {-138,-138,-138,-138};
//static int                      time_offset[4] = {-145,-145,-145,-145};
static int                      time_offset[4] = {0,0,0,0};


static char                     UE_flag=0;
static uint8_t                  eNB_id=0,UE_id=0;

uint32_t                        carrier_freq[MAX_NUM_CCs][4] =           {{1907600000,1907600000,1907600000,1907600000}}; /* For UE! */
static uint32_t                 downlink_frequency[MAX_NUM_CCs][4] =     {{1907600000,1907600000,1907600000,1907600000}};
static int32_t                  uplink_frequency_offset[MAX_NUM_CCs][4]= {{-120000000,-120000000,-120000000,-120000000}};
static char                    *conf_config_file_name = NULL;

#if defined(ENABLE_ITTI)
static char                    *itti_dump_file = NULL;
#endif

#ifndef USRP
double tx_gain[MAX_NUM_CCs][4] = {{20,10,0,0}};
double rx_gain[MAX_NUM_CCs][4] = {{10,10,0,0}};
// these are for EXMIMO2 target only
static unsigned int             rxg_max[4] =    {133,133,133,133};
static unsigned int             rxg_med[4] =    {127,127,127,127};
static unsigned int             rxg_byp[4] =    {120,120,120,120};
static rx_gain_t                rx_gain_mode[MAX_NUM_CCs][4] = {{max_gain,max_gain,max_gain,max_gain}};
#else
double tx_gain[MAX_NUM_CCs][4] = {{120,0,0,0}};
double rx_gain[MAX_NUM_CCs][4] = {{50,0,0,0}};
#endif

double sample_rate=30.72e6;
double bw = 14e6;

static int                      tx_max_power[MAX_NUM_CCs] =  {{0}};

#ifdef USRP
char ref[128] = "internal";
char channels[128] = "0";

unsigned int samples_per_frame = 307200;
unsigned int samples_per_packets = 2048; // samples got every recv or send
unsigned int tx_forward_nsamps;

int sf_bounds_5[10] = {8, 15, 23, 30, 38, 45, 53, 60, 68, 75};
int sf_bounds_10[10] = {8, 15, 23, 30, 38, 45, 53, 60, 68, 75};
int sf_bounds_20[10] = {15, 30, 45, 60, 75, 90, 105, 120, 135, 150};
int *sf_bounds;
int max_cnt;
int tx_delay;

#endif
/*
  uint32_t rf_mode_max[4]     = {55759,55759,55759,55759};
  uint32_t rf_mode_med[4]     = {39375,39375,39375,39375};
  uint32_t rf_mode_byp[4]     = {22991,22991,22991,22991};
*/
//static uint32_t                      rf_mode[4] =        {MY_RF_MODE,0,0,0};
//static uint32_t                      rf_local[4] =       {8255000,8255000,8255000,8255000}; // UE zepto
//{8254617, 8254617, 8254617, 8254617}; //eNB khalifa
//{8255067,8254810,8257340,8257340}; // eNB PETRONAS

//static uint32_t                      rf_vcocal[4] =      {910,910,910,910};
//static uint32_t                      rf_vcocal_850[4] =  {2015, 2015, 2015, 2015};
//static uint32_t                      rf_rxdc[4] =        {32896,32896,32896,32896};
//static uint32_t                      rxgain[4] =         {20,20,20,20};
//static uint32_t                      txgain[4] =         {20,20,20,20};

static runmode_t                mode;
static int                      rx_input_level_dBm;
static int                      online_log_messages=0;
#ifdef XFORMS
extern int                      otg_enabled;
static char                     do_forms=0;
#else
int                             otg_enabled;
#endif
//int                             number_of_cards =   1;
#ifndef USRP
static int                      mbox_bounds[20] =   {8,16,24,30,38,46,54,60,68,76,84,90,98,106,114,120,128,136,144, 0}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
//static int                      mbox_bounds[20] =   {6,14,22,28,36,44,52,58,66,74,82,88,96,104,112,118,126,134,142, 148}; ///boundaries of slots in terms ob mbox counter rounded up to even numbers
#endif

static LTE_DL_FRAME_PARMS      *frame_parms[MAX_NUM_CCs];

int multi_thread=1;
uint32_t target_dl_mcs = 28; //maximum allowed mcs
uint32_t target_ul_mcs = 8;


int16_t           glog_level=LOG_DEBUG;
int16_t           glog_verbosity=LOG_MED;
int16_t           hw_log_level=LOG_DEBUG;
int16_t           hw_log_verbosity=LOG_MED;
int16_t           phy_log_level=LOG_DEBUG;
int16_t           phy_log_verbosity=LOG_MED;
int16_t           mac_log_level=LOG_DEBUG;
int16_t           mac_log_verbosity=LOG_MED;
int16_t           rlc_log_level=LOG_DEBUG;
int16_t           rlc_log_verbosity=LOG_MED;
int16_t           pdcp_log_level=LOG_DEBUG;
int16_t           pdcp_log_verbosity=LOG_MED;
int16_t           rrc_log_level=LOG_DEBUG;
int16_t           rrc_log_verbosity=LOG_MED;

unsigned int build_rflocal(int txi, int txq, int rxi, int rxq)
{
  return (txi + (txq<<6) + (rxi<<12) + (rxq<<18));
}
unsigned int build_rfdc(int dcoff_i_rxfe, int dcoff_q_rxfe)
{
  return (dcoff_i_rxfe + (dcoff_q_rxfe<<8));
}

#if !defined(ENABLE_ITTI)
void signal_handler(int sig)
{
  void *array[10];
  size_t size;

  if (sig==SIGSEGV) {
    // get void*'s for all entries on the stack
    size = backtrace(array, 10);
    
    // print out all the frames to stderr
    fprintf(stderr, "Error: signal %d:\n", sig);
    backtrace_symbols_fd(array, size, 2);
    exit(-1);
  }
  else {
    oai_exit = 1;
  }
}
#endif

void exit_fun(const char* s)
{
  if (s != NULL) {
    printf("%s %s() Exiting: %s\n",__FILE__, __FUNCTION__, s);
  }

  oai_exit = 1;

#if defined(ENABLE_ITTI)
  itti_terminate_tasks (TASK_UNKNOWN);
#endif

  //rt_sleep_ns(FRAME_PERIOD);

  //exit (-1);
}

static int latency_target_fd = -1;
static int32_t latency_target_value = 0;
/* Latency trick - taken from cyclictest.c 
* if the file /dev/cpu_dma_latency exists,
* open it and write a zero into it. This will tell
* the power management system not to transition to
* a high cstate (in fact, the system acts like idle=poll)
* When the fd to /dev/cpu_dma_latency is closed, the behavior
* goes back to the system default.
*
* Documentation/power/pm_qos_interface.txt
*/
static void set_latency_target(void)
{
  struct stat s;
  int ret;
  if (stat("/dev/cpu_dma_latency", &s) == 0) {
    latency_target_fd = open("/dev/cpu_dma_latency", O_RDWR);
    if (latency_target_fd == -1)
      return;
    ret = write(latency_target_fd, &latency_target_value, 4);
    if (ret == 0) {
      printf("# error setting cpu_dma_latency to %d!: %s\n", latency_target_value, strerror(errno));
      close(latency_target_fd);
      return;
    }
    printf("# /dev/cpu_dma_latency set to %dus\n", latency_target_value);
  }
}

#ifdef XFORMS
static void *scope_thread(void *arg) {
  char stats_buffer[16384];
# ifdef ENABLE_XFORMS_WRITE_STATS
  FILE *UE_stats, *eNB_stats;
  int len = 0;
# endif
  struct sched_param sched_param;

  sched_param.sched_priority = sched_get_priority_min(SCHED_FIFO)+1; 
  sched_setscheduler(0, SCHED_FIFO,&sched_param);

  printf("Scope thread has priority %d\n",sched_param.sched_priority);
    
  /*
    if (UE_flag==1) 
    UE_stats  = fopen("UE_stats.txt", "w");
    else 
    eNB_stats = fopen("eNB_stats.txt", "w");
  */
    
  while (!oai_exit) {
    if (UE_flag==1) {
# ifdef ENABLE_XFORMS_WRITE_STATS
      len =
# endif
	dump_ue_stats (PHY_vars_UE_g[0][0], stats_buffer, 0, mode,rx_input_level_dBm);
      fl_set_object_label(form_stats->stats_text, stats_buffer);
# ifdef ENABLE_XFORMS_WRITE_STATS
      rewind (UE_stats);
      fwrite (stats_buffer, 1, len, UE_stats);
# endif
      phy_scope_UE(form_ue[UE_id], 
		   PHY_vars_UE_g[UE_id][0],
		   eNB_id,
		   UE_id,7);
            
    } else {
# ifdef ENABLE_XFORMS_WRITE_STATS
      len =
# endif
	dump_eNB_stats (PHY_vars_eNB_g[0][0], stats_buffer, 0);
      fl_set_object_label(form_stats->stats_text, stats_buffer);
# ifdef ENABLE_XFORMS_WRITE_STATS
      rewind (eNB_stats);
      fwrite (stats_buffer, 1, len, eNB_stats);
# endif
      for(UE_id=0;UE_id<scope_enb_num_ue;UE_id++) {
	phy_scope_eNB(form_enb[UE_id], 
		      PHY_vars_eNB_g[eNB_id][0],
		      UE_id);
      }
              
    }
    //printf("doing forms\n");
    //usleep(100000); // 100 ms
    sleep(1);
  }
    
# ifdef ENABLE_XFORMS_WRITE_STATS
  fclose (UE_stats);
  fclose (eNB_stats);
# endif
    
  pthread_exit((void*)arg);
}
#endif

#ifdef EMOS
#define NO_ESTIMATES_DISK 100 //No. of estimates that are aquired before dumped to disk

void *emos_thread (void *arg)
{
  char c;
  char *fifo2file_buffer, *fifo2file_ptr;

  int fifo, counter=0, bytes;

  FILE  *dumpfile_id;
  char  dumpfile_name[1024];
  time_t starttime_tmp;
  struct tm starttime;
  
  int channel_buffer_size;
  
  time_t timer;
  struct tm *now;

  struct gps_data_t *gps_data = NULL;
  struct gps_fix_t dummy_gps_data;

  struct sched_param sched_param;
  
  sched_param.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; 
  sched_setscheduler(0, SCHED_FIFO,&sched_param);
  
  printf("EMOS thread has priority %d\n",sched_param.sched_priority);
 
  timer = time(NULL);
  now = localtime(&timer);

  memset(&dummy_gps_data,1,sizeof(struct gps_fix_t));
  
  gps_data = gps_open("127.0.0.1","2947");
  if (gps_data == NULL) 
    {
      printf("[EMOS] Could not open GPS\n");
      //exit(-1);
    }
#if GPSD_API_MAJOR_VERSION>=4
  else if (gps_stream(gps_data, WATCH_ENABLE,NULL) != 0)
#else
  else if (gps_query(gps_data, "w+x") != 0)
#endif
    {
      //sprintf(tmptxt,"Error sending command to GPS, gps_data = %x", gps_data);
      printf("[EMOS] Error sending command to GPS\n");
      //exit(-1);
    }
  else 
    printf("[EMOS] Opened GPS, gps_data=%p\n");
  
  if (UE_flag==0)
    channel_buffer_size = sizeof(fifo_dump_emos_eNB);
  else
    channel_buffer_size = sizeof(fifo_dump_emos_UE);

  // allocate memory for NO_FRAMES_DISK channes estimations
  fifo2file_buffer = malloc(NO_ESTIMATES_DISK*channel_buffer_size);
  fifo2file_ptr = fifo2file_buffer;

  if (fifo2file_buffer == NULL)
    {
      printf("[EMOS] Cound not allocate memory for fifo2file_buffer\n");
      exit(EXIT_FAILURE);
    }

  if ((fifo = open(CHANSOUNDER_FIFO_DEV, O_RDONLY)) < 0)
    {
      fprintf(stderr, "[EMOS] Error opening the fifo\n");
      exit(EXIT_FAILURE);
    }


  time(&starttime_tmp);
  localtime_r(&starttime_tmp,&starttime);
  snprintf(dumpfile_name,1024,"/tmp/%s_data_%d%02d%02d_%02d%02d%02d.EMOS",
	   (UE_flag==0) ? "eNB" : "UE",
	   1900+starttime.tm_year, starttime.tm_mon+1, starttime.tm_mday, starttime.tm_hour, starttime.tm_min, starttime.tm_sec);

  dumpfile_id = fopen(dumpfile_name,"w");
  if (dumpfile_id == NULL)
    {
      fprintf(stderr, "[EMOS] Error opening dumpfile %s\n",dumpfile_name);
      exit(EXIT_FAILURE);
    }


  printf("[EMOS] starting dump, channel_buffer_size=%d ...\n",channel_buffer_size);
  while (!oai_exit)
    {
      bytes = rtf_read_timed(fifo, fifo2file_ptr, channel_buffer_size,100);
      if (bytes==0)
	continue;

      /*
	if (UE_flag==0)
	printf("eNB: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx,bytes);
	else
	printf("UE: count %d, frame %d, read: %d bytes from the fifo\n",counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx,bytes);
      */

      fifo2file_ptr += channel_buffer_size;
      counter ++;

      if (counter == NO_ESTIMATES_DISK)
        {
          //reset stuff
          fifo2file_ptr = fifo2file_buffer;
          counter = 0;

          //flush buffer to disk
	  if (UE_flag==0)
	    printf("[EMOS] eNB: count %d, frame %d, flushing buffer to disk\n",
		   counter, ((fifo_dump_emos_eNB*)fifo2file_ptr)->frame_tx);
	  else
	    printf("[EMOS] UE: count %d, frame %d, flushing buffer to disk\n",
		   counter, ((fifo_dump_emos_UE*)fifo2file_ptr)->frame_rx);


          if (fwrite(fifo2file_buffer, sizeof(char), NO_ESTIMATES_DISK*channel_buffer_size, dumpfile_id) != NO_ESTIMATES_DISK*channel_buffer_size)
            {
              fprintf(stderr, "[EMOS] Error writing to dumpfile\n");
              exit(EXIT_FAILURE);
            }
	  if (gps_data)
	    {
	      if (gps_poll(gps_data) != 0) {
		printf("[EMOS] problem polling data from gps\n");
	      }
	      else {
		printf("[EMOS] lat %g, lon %g\n",gps_data->fix.latitude,gps_data->fix.longitude);
	      }
	      if (fwrite(&(gps_data->fix), sizeof(char), sizeof(struct gps_fix_t), dumpfile_id) != sizeof(struct gps_fix_t))
		{
		  printf("[EMOS] Error writing to dumpfile, stopping recording\n");
		  exit(EXIT_FAILURE);
		}
	    }
	  else
	    {
	      printf("[EMOS] WARNING: No GPS data available, storing dummy packet\n");
	      if (fwrite(&(dummy_gps_data), sizeof(char), sizeof(struct gps_fix_t), dumpfile_id) != sizeof(struct gps_fix_t))
		{
		  printf("[EMOS] Error writing to dumpfile, stopping recording\n");
		  exit(EXIT_FAILURE);
		}
	    } 
        }
    }
  
  free(fifo2file_buffer);
  fclose(dumpfile_id);
  close(fifo);
  
  pthread_exit((void*) arg);

}
#endif

#if defined(ENABLE_ITTI)
static void wait_system_ready (char *message, volatile int *start_flag)
{
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  {
    static char *indicator[] = {".    ", "..   ", "...  ", ".... ", ".....",
                                " ....", "  ...", "   ..", "    .", "     "};
    int i = 0;

    while ((!oai_exit) && (*start_flag == 0)) {
      LOG_N(EMU, message, indicator[i]);
      i = (i + 1) % (sizeof(indicator) / sizeof(indicator[0]));
      usleep(200000);
    }
    LOG_D(EMU,"\n");
  }
}
#endif

#if defined(ENABLE_ITTI)
void *l2l1_task(void *arg)
{
  MessageDef *message_p = NULL;
  int         result;

  itti_set_task_real_time(TASK_L2L1);
  itti_mark_task_ready(TASK_L2L1);

  if (UE_flag == 0) {
    /* Wait for the initialize message */
    do {
      if (message_p != NULL) {
	result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
	AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
      }
      itti_receive_msg (TASK_L2L1, &message_p);

      switch (ITTI_MSG_ID(message_p)) {
      case INITIALIZE_MESSAGE:
	/* Start eNB thread */
	LOG_D(EMU, "L2L1 TASK received %s\n", ITTI_MSG_NAME(message_p));
	start_eNB = 1;
	break;

      case TERMINATE_MESSAGE:
	oai_exit=1;
	itti_exit_task ();
	break;

      default:
	LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
	break;
      }
    } while (ITTI_MSG_ID(message_p) != INITIALIZE_MESSAGE);
    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  }

  do {
    // Wait for a message
    itti_receive_msg (TASK_L2L1, &message_p);

    switch (ITTI_MSG_ID(message_p)) {
    case TERMINATE_MESSAGE:
      oai_exit=1;
      itti_exit_task ();
      break;

    case ACTIVATE_MESSAGE:
      start_UE = 1;
      break;

    case DEACTIVATE_MESSAGE:
      start_UE = 0;
      break;

    case MESSAGE_TEST:
      LOG_I(EMU, "Received %s\n", ITTI_MSG_NAME(message_p));
      break;

    default:
      LOG_E(EMU, "Received unexpected message %s\n", ITTI_MSG_NAME(message_p));
      break;
    }

    result = itti_free (ITTI_MSG_ORIGIN_ID(message_p), message_p);
    AssertFatal (result == EXIT_SUCCESS, "Failed to free memory (%d)!\n", result);
  } while(1);

  return NULL;
}
#endif


void do_OFDM_mod_rt(int subframe,PHY_VARS_eNB *phy_vars_eNB) {

  unsigned int aa,slot_offset, slot_offset_F;
  int dummy_tx_b[7680*4] __attribute__((aligned(16)));
  int i, tx_offset;
  int slot_sizeF = (phy_vars_eNB->lte_frame_parms.ofdm_symbol_size)*
    ((phy_vars_eNB->lte_frame_parms.Ncp==1) ? 6 : 7);

  slot_offset_F = (subframe<<1)*slot_sizeF;
    
  slot_offset = subframe*phy_vars_eNB->lte_frame_parms.samples_per_tti;

  if ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_DL)||
      ((subframe_select(&phy_vars_eNB->lte_frame_parms,subframe)==SF_S))) {
    //	  LOG_D(HW,"Frame %d: Generating slot %d\n",frame,next_slot);

    
    for (aa=0; aa<phy_vars_eNB->lte_frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->lte_frame_parms.Ncp == EXTENDED){ 
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
		     dummy_tx_b,
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
	PHY_ofdm_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
		     dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
		     phy_vars_eNB->lte_frame_parms.log2_symbol_size,
		     6,
		     phy_vars_eNB->lte_frame_parms.nb_prefix_samples,
		     phy_vars_eNB->lte_frame_parms.twiddle_ifft,
		     phy_vars_eNB->lte_frame_parms.rev,
		     CYCLIC_PREFIX);
      }
      else {
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F],
			  dummy_tx_b,
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
	normal_prefix_mod(&phy_vars_eNB->lte_eNB_common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
			  dummy_tx_b+(phy_vars_eNB->lte_frame_parms.samples_per_tti>>1),
			  7,
			  &(phy_vars_eNB->lte_frame_parms));
      }

      for (i=0; i<phy_vars_eNB->lte_frame_parms.samples_per_tti; i++) {
	tx_offset = (int)slot_offset+time_offset[aa]+i;
	if (tx_offset<0)
	  tx_offset += LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;
	if (tx_offset>=(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti))
	  tx_offset -= LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->lte_frame_parms.samples_per_tti;
	((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[0]=
	  ((short*)dummy_tx_b)[2*i]<<4;
	((short*)&phy_vars_eNB->lte_eNB_common_vars.txdata[0][aa][tx_offset])[1]=
	  ((short*)dummy_tx_b)[2*i+1]<<4;
      }
    }
  }
}


int eNB_thread_tx_status[10];
static void * eNB_thread_tx(void *param) {

  //unsigned long cpuid;
  eNB_proc_t *proc = (eNB_proc_t*)param;
  //  RTIME time_in,time_out;
#ifdef RTAI
  RT_TASK *task;
  char task_name[8];
#endif

 

#if defined(ENABLE_ITTI)
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif

#ifdef RTAI
  sprintf(task_name,"TXC%dS%d",proc->CC_id,proc->subframe);
  task = rt_task_init_schmod(nam2num(task_name), 0, 0, 0, SCHED_FIFO, 0xF);

  if (task==NULL) {
    LOG_E(PHY,"[SCHED][eNB] Problem starting eNB_proc_TX thread_index %d (%s)!!!!\n",proc->subframe,task_name);
    return 0;
  }
  else {
    LOG_I(PHY,"[SCHED][eNB] eNB TX thread CC %d SF %d started with id %p\n",
	  proc->CC_id,
	  proc->subframe,
	  task);
  }
#else
  //  LOG_I(PHY,
	printf("[SCHED][eNB] eNB TX thread %d started on CPU %d\n",
	       proc->subframe,sched_getcpu());
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

  //rt_set_runnable_on_cpuid(task,1);
  //cpuid = rtai_cpuid();

#ifdef HARD_RT
  rt_make_hard_real_time();
#endif

  while (!oai_exit){
    
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),0);
    
    
    //    LOG_I(PHY,"Locking mutex for eNB proc %d (IC %d,mutex %p)\n",proc->subframe,proc->instance_cnt,&proc->mutex);
    //printf("Locking mutex for eNB proc %d (subframe_tx %d))\n",proc->subframe,subframe_tx);

    if (pthread_mutex_lock(&proc->mutex_tx) != 0) {
      LOG_E(PHY,"[SCHED][eNB] error locking mutex for eNB TX proc %d\n",proc->subframe);
      oai_exit=1;
    }
    else {
      
      while (proc->instance_cnt_tx < 0) {
	//	LOG_I(PHY,"Waiting and unlocking mutex for eNB proc %d (IC %d,lock %d)\n",proc->subframe,proc->instance_cnt,pthread_mutex_trylock(&proc->mutex));
	//printf("Waiting and unlocking mutex for eNB proc %d (subframe_tx %d)\n",proc->subframe,subframe_tx);
	
	pthread_cond_wait(&proc->cond_tx,&proc->mutex_tx);
      }
      //      LOG_I(PHY,"Waking up and unlocking mutex for eNB proc %d\n",proc->subframe);
      if (pthread_mutex_unlock(&proc->mutex_tx) != 0) {	
	LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB TX proc %d\n",proc->subframe);
	oai_exit=1;
      }
    }
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),1);    
    vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_ENB, proc->frame_tx);
    vcd_signal_dumper_dump_variable_by_name(VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_ENB, proc->subframe*2);
    
    if (oai_exit) break;

    
    if ((((PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms.frame_type == TDD)&&
	  (subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms,proc->subframe_tx)==SF_DL))||
	 (PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms.frame_type == FDD))) {
      
      phy_procedures_eNB_TX(proc->subframe,PHY_vars_eNB_g[0][proc->CC_id],0,no_relay,NULL);
      
    }
    if ((subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms,proc->subframe_tx)==SF_S)) {
      phy_procedures_eNB_TX(proc->subframe,PHY_vars_eNB_g[0][proc->CC_id],0,no_relay,NULL);
    }
    
    
    do_OFDM_mod_rt(proc->subframe_tx,PHY_vars_eNB_g[0][proc->CC_id]);  
    
    if (pthread_mutex_lock(&proc->mutex_tx) != 0) {
      printf("[openair][SCHED][eNB] error locking mutex for eNB TX proc %d\n",proc->subframe);
    }
    else {
      proc->instance_cnt_tx--;
      
      if (pthread_mutex_unlock(&proc->mutex_tx) != 0) {	
	printf("[openair][SCHED][eNB] error unlocking mutex for eNB TX proc %d\n",proc->subframe);
      }
    }
    
    proc->frame_tx++;
    if (proc->frame_tx==1024)
      proc->frame_tx=0;
  }    
  vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_TX0+(2*proc->subframe),0);        
#ifdef HARD_RT
  rt_make_soft_real_time();
#endif
  
#ifdef DEBUG_THREADS
  printf("Exiting eNB thread TX %d\n",proc->subframe);
#endif
  // clean task
#ifdef RTAI
  rt_task_delete(task);
#else
  eNB_thread_tx_status[proc->subframe]=0;
  pthread_exit(&eNB_thread_tx_status[proc->subframe]);
#endif
  
#ifdef DEBUG_THREADS
  printf("Exiting eNB TX thread %d\n",proc->subframe);
#endif
}

int eNB_thread_rx_status[10];
static void * eNB_thread_rx(void *param) {

  //unsigned long cpuid;
  eNB_proc_t *proc = (eNB_proc_t*)param;
  //  RTIME time_in,time_out;
#ifdef RTAI
  RT_TASK *task;
  char task_name[8];
#endif

#if defined(ENABLE_ITTI)
  /* Wait for eNB application initialization to be complete (eNB registration to MME) */
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif

#ifdef RTAI
  sprintf(task_name,"RXC%1dS%1d",proc->CC_id,proc->subframe);
  task = rt_task_init_schmod(nam2num(task_name), 0, 0, 0, SCHED_FIFO, 0xF);

  if (task==NULL) {
    LOG_E(PHY,"[SCHED][eNB] Problem starting eNB_proc_RX thread_index %d (%s)!!!!\n",proc->subframe,task_name);
    return 0;
  }
  else {
    LOG_I(PHY,"[SCHED][eNB] eNB RX thread CC_id %d SF %d started with id %p\n", /*  on CPU %d*/
	  proc->CC_id,
	  proc->subframe,
	  task); /*,rtai_cpuid()*/
  }
#else
  LOG_I(PHY,"[SCHED][eNB] eNB RX thread %d started on CPU %d\n",
	proc->subframe,sched_getcpu());
#endif

  mlockall(MCL_CURRENT | MCL_FUTURE);

  //rt_set_runnable_on_cpuid(task,1);
  //cpuid = rtai_cpuid();

#ifdef HARD_RT
  rt_make_hard_real_time();
#endif

  while (!oai_exit){

    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),0);


    //    LOG_I(PHY,"Locking mutex for eNB proc %d (IC %d,mutex %p)\n",proc->subframe,proc->instance_cnt,&proc->mutex);
    if (pthread_mutex_lock(&proc->mutex_rx) != 0) {
      LOG_E(PHY,"[SCHED][eNB] error locking mutex for eNB RX proc %d\n",proc->subframe);
    }
    else {
        
      while (proc->instance_cnt_rx < 0) {
	//	LOG_I(PHY,"Waiting and unlocking mutex for eNB proc %d (IC %d,lock %d)\n",proc->subframe,proc->instance_cnt,pthread_mutex_trylock(&proc->mutex));

	pthread_cond_wait(&proc->cond_rx,&proc->mutex_rx);
      }
      //      LOG_I(PHY,"Waking up and unlocking mutex for eNB proc %d\n",proc->subframe);
      if (pthread_mutex_unlock(&proc->mutex_rx) != 0) {	
	LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB RX proc %d\n",proc->subframe);
      }
    }
    vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),1);    

    if (oai_exit) break;
    
    if ((((PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms.frame_type == TDD )&&(subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms,proc->subframe_rx)==SF_UL)) ||
	 (PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms.frame_type == FDD))){
      phy_procedures_eNB_RX(proc->subframe,PHY_vars_eNB_g[0][proc->CC_id],0,no_relay);
    }
    if ((subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->lte_frame_parms,proc->subframe_rx)==SF_S)){
      phy_procedures_eNB_S_RX(proc->subframe,PHY_vars_eNB_g[0][proc->CC_id],0,no_relay);
    }
      
    if (pthread_mutex_lock(&proc->mutex_rx) != 0) {
      printf("[openair][SCHED][eNB] error locking mutex for eNB RX proc %d\n",proc->subframe);
    }
    else {
      proc->instance_cnt_rx--;
      
      if (pthread_mutex_unlock(&proc->mutex_rx) != 0) {	
	printf("[openair][SCHED][eNB] error unlocking mutex for eNB RX proc %d\n",proc->subframe);
      }
    }

    proc->frame_rx++;
    if (proc->frame_rx==1024)
      proc->frame_rx=0;
    
  }
  vcd_signal_dumper_dump_function_by_name(VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX0+(2*proc->subframe),0);        
#ifdef HARD_RT
  rt_make_soft_real_time();
#endif

#ifdef DEBUG_THREADS
  printf("Exiting eNB thread RX %d\n",proc->subframe);
#endif
  // clean task
#ifdef RTAI
  rt_task_delete(task);
#else
  eNB_thread_rx_status[proc->subframe]=0;
  pthread_exit(&eNB_thread_rx_status[proc->subframe]);
#endif

#ifdef DEBUG_THREADS
  printf("Exiting eNB RX thread %d\n",proc->subframe);
#endif
}




void init_eNB_proc(void) {

  int i;
  int CC_id;

  for (CC_id=0;CC_id<MAX_NUM_CCs;CC_id++) {
    for (i=0;i<10;i++) {
      pthread_attr_init (&attr_eNB_proc_tx[CC_id][i]);
      pthread_attr_setstacksize(&attr_eNB_proc_tx[CC_id][i],OPENAIR_THREAD_STACK_SIZE);
      //attr_dlsch_threads.priority = 1;
      sched_param_eNB_proc_tx[CC_id][i].sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
      pthread_attr_setschedparam  (&attr_eNB_proc_tx[CC_id][i], &sched_param_eNB_proc_tx[CC_id][i]);
      pthread_attr_setschedpolicy (&attr_eNB_proc_tx[CC_id][i], SCHED_FIFO);
      
      pthread_attr_init (&attr_eNB_proc_rx[CC_id][i]);
      pthread_attr_setstacksize(&attr_eNB_proc_rx[CC_id][i],OPENAIR_THREAD_STACK_SIZE);
      //attr_dlsch_threads.priority = 1;
      sched_param_eNB_proc_rx[CC_id][i].sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
      pthread_attr_setschedparam  (&attr_eNB_proc_rx[CC_id][i], &sched_param_eNB_proc_rx[CC_id][i]);
      pthread_attr_setschedpolicy (&attr_eNB_proc_rx[CC_id][i], SCHED_FIFO);
      
	PHY_vars_eNB_g[0][CC_id]->proc[i].instance_cnt_tx=-1;
	PHY_vars_eNB_g[0][CC_id]->proc[i].instance_cnt_rx=-1;
	PHY_vars_eNB_g[0][CC_id]->proc[i].subframe=i;
	PHY_vars_eNB_g[0][CC_id]->proc[i].CC_id = CC_id;
	pthread_mutex_init(&PHY_vars_eNB_g[0][CC_id]->proc[i].mutex_tx,NULL);
	pthread_mutex_init(&PHY_vars_eNB_g[0][CC_id]->proc[i].mutex_rx,NULL);
	pthread_cond_init(&PHY_vars_eNB_g[0][CC_id]->proc[i].cond_tx,NULL);
	pthread_cond_init(&PHY_vars_eNB_g[0][CC_id]->proc[i].cond_rx,NULL);
	pthread_create(&PHY_vars_eNB_g[0][CC_id]->proc[i].pthread_tx,NULL,eNB_thread_tx,(void*)&PHY_vars_eNB_g[0][CC_id]->proc[i]);
	pthread_create(&PHY_vars_eNB_g[0][CC_id]->proc[i].pthread_rx,NULL,eNB_thread_rx,(void*)&PHY_vars_eNB_g[0][CC_id]->proc[i]);
	PHY_vars_eNB_g[0][CC_id]->proc[i].frame_tx = 0;
	PHY_vars_eNB_g[0][CC_id]->proc[i].frame_rx = 0;
#ifndef USRP
	PHY_vars_eNB_g[0][CC_id]->proc[i].subframe_rx = (i+9)%10;
	PHY_vars_eNB_g[0][CC_id]->proc[i].subframe_tx = (i+1)%10;
#else
	PHY_vars_eNB_g[0][CC_id]->proc[i].subframe_rx = i;
	PHY_vars_eNB_g[0][CC_id]->proc[i].subframe_tx = (i+2)%10;
#endif
    }
  
  
#ifndef USRP
  // TX processes subframe + 1, RX subframe -1
  // Note this inialization is because the first process awoken for frame 0 is number 1 and so processes 9 and 0 have to start with frame 1
    
  //PHY_vars_eNB_g[0][CC_id]->proc[0].frame_rx = 1023;
    PHY_vars_eNB_g[0][CC_id]->proc[9].frame_tx = 1;
    PHY_vars_eNB_g[0][CC_id]->proc[0].frame_tx = 1;
#else
    // TX processes subframe +2, RX subframe
    // Note this inialization is because the first process awoken for frame 0 is number 1 and so processes 8,9 and 0 have to start with frame 1
    PHY_vars_eNB_g[0][CC_id]->proc[8].frame_tx = 1;
    PHY_vars_eNB_g[0][CC_id]->proc[9].frame_tx = 1;
    PHY_vars_eNB_g[0][CC_id]->proc[0].frame_tx = 1;
#endif
  }
}

void kill_eNB_proc(void) {

  int i;
  int *status_tx,*status_rx;
  int CC_id;

  for (CC_id=0;CC_id<MAX_NUM_CCs;CC_id++) 
    for (i=0;i<10;i++) {
      
#ifdef DEBUG_THREADS
      printf("Killing TX CC_id %d thread %d\n",CC_id,i);
#endif
      PHY_vars_eNB_g[0][CC_id]->proc[i].instance_cnt_tx=0; 
      pthread_cond_signal(&PHY_vars_eNB_g[0][CC_id]->proc[i].cond_tx);
#ifdef DEBUG_THREADS
      printf("Joining eNB TX CC_id %d thread %d...\n",CC_id,i);
#endif
      pthread_join(PHY_vars_eNB_g[0][CC_id]->proc[i].pthread_tx,(void**)status_tx);
#ifdef DEBUG_THREADS
      if (status_tx) printf("status %d...\n",*status_tx);
#endif
#ifdef DEBUG_THREADS
      printf("Killing RX CC_id %d thread %d\n",CC_id,i);