lte-enb.c 45.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
/*******************************************************************************
    OpenAirInterface
    Copyright(c) 1999 - 2014 Eurecom

    OpenAirInterface is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.


    OpenAirInterface is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with OpenAirInterface.The full GNU General Public License is
    included in this distribution in the file called "COPYING". If not,
    see <http://www.gnu.org/licenses/>.

   Contact Information
   OpenAirInterface Admin: openair_admin@eurecom.fr
   OpenAirInterface Tech : openair_tech@eurecom.fr
   OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr

   Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

*******************************************************************************/

/*! \file lte-enb.c
 * \brief Top-level threads for eNodeB
 * \author R. Knopp, F. Kaltenberger, Navid Nikaein
 * \date 2012
 * \version 0.1
 * \company Eurecom
 * \email: knopp@eurecom.fr,florian.kaltenberger@eurecom.fr, navid.nikaein@eurecom.fr
 * \note
 * \warning
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sched.h>
#include <linux/sched.h>
#include <signal.h>
#include <execinfo.h>
#include <getopt.h>
#include <sys/sysinfo.h>
#include "rt_wrapper.h"

#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all

#include "assertions.h"
#include "msc.h"

#include "PHY/types.h"

#include "PHY/defs.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#include "../../ARCH/COMMON/common_lib.h"

//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

71
72
#include "PHY/LTE_TRANSPORT/if4_tools.h"

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#include "PHY/extern.h"
#include "SCHED/extern.h"
#include "LAYER2/MAC/extern.h"

#include "../../SIMU/USER/init_lte.h"

#include "LAYER2/MAC/defs.h"
#include "LAYER2/MAC/extern.h"
#include "LAYER2/MAC/proto.h"
#include "RRC/LITE/extern.h"
#include "PHY_INTERFACE/extern.h"

#ifdef SMBV
#include "PHY/TOOLS/smbv.h"
unsigned short config_frames[4] = {2,9,11,13};
#endif
#include "UTIL/LOG/log_extern.h"
#include "UTIL/OTG/otg_tx.h"
#include "UTIL/OTG/otg_externs.h"
#include "UTIL/MATH/oml.h"
#include "UTIL/LOG/vcd_signal_dumper.h"
#include "UTIL/OPT/opt.h"
#include "enb_config.h"
//#include "PHY/TOOLS/time_meas.h"

#ifndef OPENAIR2
#include "UTIL/OTG/otg_extern.h"
#endif

#if defined(ENABLE_ITTI)
# if defined(ENABLE_USE_MME)
#   include "s1ap_eNB.h"
#ifdef PDCP_USE_NETLINK
#   include "SIMULATION/ETH_TRANSPORT/proto.h"
#endif
# endif
#endif

//#define DEBUG_THREADS 1

//#define USRP_DEBUG 1
struct timing_info_t {
  //unsigned int frame, hw_slot, last_slot, next_slot;
  RTIME time_min, time_max, time_avg, time_last, time_now;
  //unsigned int mbox0, mbox1, mbox2, mbox_target;
  unsigned int n_samples;
} timing_info;

121
122
// Fix per CC openair rf/if device update
// extern openair0_device openair0;
123
124
125
126
127
128
129

#if defined(ENABLE_ITTI)
extern volatile int             start_eNB;
extern volatile int             start_UE;
#endif
extern volatile int                    oai_exit;

Raymond Knopp's avatar
Raymond Knopp committed
130
extern openair0_config_t openair0_cfg[MAX_CARDS];
131
132
133
134
135

extern pthread_cond_t sync_cond;
extern pthread_mutex_t sync_mutex;
extern int sync_var;

136
//pthread_t                       main_eNB_thread;
137
138
139

time_stats_t softmodem_stats_mt; // main thread
time_stats_t softmodem_stats_hw; //  hw acquisition
knopp's avatar
knopp committed
140
time_stats_t softmodem_stats_rxtx_sf; // total tx time
141
time_stats_t softmodem_stats_rx_sf; // total rx time
142
143
144
145
146
147
148
149
150
151
152
153
154
int32_t **rxdata;
int32_t **txdata;

static int                      time_offset[4] = {0,0,0,0};

/* mutex, cond and variable to serialize phy proc TX calls
 * (this mechanism may be relaxed in the future for better
 * performances)
 */
static struct {
  pthread_mutex_t  mutex_phy_proc_tx;
  pthread_cond_t   cond_phy_proc_tx;
  volatile uint8_t phy_proc_CC_id;
155
} sync_phy_proc;
156
157
158

void exit_fun(const char* s);

159
void init_eNB(eNB_func_t node_function);
160
161
162
void stop_eNB(void);


Sandeep Kumar's avatar
Sandeep Kumar committed
163
164
void do_OFDM_mod_rt(int subframe,PHY_VARS_eNB *phy_vars_eNB) {
     
165
166
  unsigned int aa,slot_offset, slot_offset_F;
  int dummy_tx_b[7680*4] __attribute__((aligned(32)));
knopp's avatar
knopp committed
167
  int i,j, tx_offset;
168
169
  int slot_sizeF = (phy_vars_eNB->frame_parms.ofdm_symbol_size)*
                   ((phy_vars_eNB->frame_parms.Ncp==1) ? 6 : 7);
knopp's avatar
knopp committed
170
171
  int len,len2;
  int16_t *txdata;
172
173
174

  slot_offset_F = (subframe<<1)*slot_sizeF;

175
  slot_offset = subframe*phy_vars_eNB->frame_parms.samples_per_tti;
176

177
178
  if ((subframe_select(&phy_vars_eNB->frame_parms,subframe)==SF_DL)||
      ((subframe_select(&phy_vars_eNB->frame_parms,subframe)==SF_S))) {
179
180
    //    LOG_D(HW,"Frame %d: Generating slot %d\n",frame,next_slot);

181
182
183
    for (aa=0; aa<phy_vars_eNB->frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->frame_parms.Ncp == EXTENDED) {
        PHY_ofdm_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F],
184
                     dummy_tx_b,
185
                     phy_vars_eNB->frame_parms.ofdm_symbol_size,
186
                     6,
187
                     phy_vars_eNB->frame_parms.nb_prefix_samples,
188
                     CYCLIC_PREFIX);
189
190
191
        PHY_ofdm_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
                     dummy_tx_b+(phy_vars_eNB->frame_parms.samples_per_tti>>1),
                     phy_vars_eNB->frame_parms.ofdm_symbol_size,
192
                     6,
193
                     phy_vars_eNB->frame_parms.nb_prefix_samples,
194
195
                     CYCLIC_PREFIX);
      } else {
196
        normal_prefix_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F],
197
198
                          dummy_tx_b,
                          7,
199
                          &(phy_vars_eNB->frame_parms));
200
	// if S-subframe generate first slot only
Sandeep Kumar's avatar
Sandeep Kumar committed
201
	if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_DL) 
202
203
	  normal_prefix_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
			    dummy_tx_b+(phy_vars_eNB->frame_parms.samples_per_tti>>1),
204
			    7,
205
			    &(phy_vars_eNB->frame_parms));
206
207
208
      }

      // if S-subframe generate first slot only
209
210
      if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_S)
	len = phy_vars_eNB->frame_parms.samples_per_tti>>1;
211
      else
212
	len = phy_vars_eNB->frame_parms.samples_per_tti;
213
214
215
216
217
218
219
      /*
      for (i=0;i<len;i+=4) {
	dummy_tx_b[i] = 0x100;
	dummy_tx_b[i+1] = 0x01000000;
	dummy_tx_b[i+2] = 0xff00;
	dummy_tx_b[i+3] = 0xff000000;
	}*/
knopp's avatar
knopp committed
220
221
222
223
224
225
226
227
228
229
230
231
232
233
      
      if (slot_offset+time_offset[aa]<0) {
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti)+tx_offset];
        len2 = -(slot_offset+time_offset[aa]);
	len2 = (len2>len) ? len : len2;
	for (i=0; i<(len2<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
	if (len2<len) {
	  txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][0];
	  for (j=0; i<(len<<1); i++,j++) {
	    txdata[j++] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	  }
	}
Sandeep Kumar's avatar
Sandeep Kumar committed
234
      }  
knopp's avatar
knopp committed
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
      else if ((slot_offset+time_offset[aa]+len)>(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti)) {
	tx_offset = (int)slot_offset+time_offset[aa];
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][tx_offset];
	len2 = -tx_offset+LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
	for (i=0; i<(len2<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][0];
	for (j=0; i<(len<<1); i++,j++) {
	  txdata[j++] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
      }
      else {
	tx_offset = (int)slot_offset+time_offset[aa];
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][tx_offset];
250

knopp's avatar
knopp committed
251
252
253
254
255
	for (i=0; i<(len<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
      }
      
256
     // if S-subframe switch to RX in second subframe
knopp's avatar
knopp committed
257
      /*
258
     if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_S) {
259
       for (i=0; i<len; i++) {
260
	 phy_vars_eNB->common_vars.txdata[0][aa][tx_offset++] = 0x00010001;
261
262
       }
     }
knopp's avatar
knopp committed
263
      */
264
     if ((((phy_vars_eNB->frame_parms.tdd_config==0) ||
Sandeep Kumar's avatar
Sandeep Kumar committed
265
266
267
268
	   (phy_vars_eNB->frame_parms.tdd_config==1) ||
	   (phy_vars_eNB->frame_parms.tdd_config==2) ||
	   (phy_vars_eNB->frame_parms.tdd_config==6)) && 
	   (subframe==0)) || (subframe==5)) {
269
270
271
       // turn on tx switch N_TA_offset before
       //LOG_D(HW,"subframe %d, time to switch to tx (N_TA_offset %d, slot_offset %d) \n",subframe,phy_vars_eNB->N_TA_offset,slot_offset);
       for (i=0; i<phy_vars_eNB->N_TA_offset; i++) {
Sandeep Kumar's avatar
Sandeep Kumar committed
272
273
274
         tx_offset = (int)slot_offset+time_offset[aa]+i-phy_vars_eNB->N_TA_offset/2;
         if (tx_offset<0)
           tx_offset += LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
275
	 
Sandeep Kumar's avatar
Sandeep Kumar committed
276
277
         if (tx_offset>=(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti))
           tx_offset -= LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
278
	 
Sandeep Kumar's avatar
Sandeep Kumar committed
279
         phy_vars_eNB->common_vars.txdata[0][aa][tx_offset] = 0x00000000;
280
281
282
283
284
285
       }
     }
    }
  }
}

Sandeep Kumar's avatar
Sandeep Kumar committed
286

287
/*!
knopp's avatar
knopp committed
288
 * \brief The RX UE-specific and TX thread of eNB.
289
290
291
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
292
293
static void* eNB_thread_rxtx( void* param ) {

knopp's avatar
knopp committed
294
  static int eNB_thread_rxtx_status;
295

knopp's avatar
knopp committed
296
  eNB_rxtx_proc_t *proc = (eNB_rxtx_proc_t*)param;
297
298
  FILE  *tx_time_file = NULL;
  char tx_time_name[101];
knopp's avatar
knopp committed
299
  void *txp[PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx]; 
Sandeep Kumar's avatar
Sandeep Kumar committed
300
301
302
303
  
  uint16_t packet_type;
  uint32_t symbol_number=0;
  
304
  if (opp_enabled == 1) {
305
    snprintf(tx_time_name, 100,"/tmp/%s_tx_time_thread_sf", "eNB");
306
307
308
    tx_time_file = fopen(tx_time_name,"w");
  }
  // set default return value
knopp's avatar
knopp committed
309
  eNB_thread_rxtx_status = 0;
310
311
312

  MSC_START_USE();

313
#ifdef DEADLINE_SCHEDULER
314
  struct sched_attr attr;
315

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
  unsigned int flags = 0;
  uint64_t runtime  = 850000 ;  
  uint64_t deadline = 1   *  1000000 ; // each tx thread will finish within 1ms
  uint64_t period   = 1   * 10000000; // each tx thread has a period of 10ms from the starting point

  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy   = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB tx thread: sched_setattr failed\n");
knopp's avatar
knopp committed
333
    return &eNB_thread_rxtx_status;
334
335
  }

knopp's avatar
knopp committed
336
  LOG_I( HW, "[SCHED] eNB RXn-TXnp4 deadline thread (TID %ld) started on CPU %d\n", gettid(), sched_getcpu() );
337
338
339
340
341
342

#else //LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;
Raymond Knopp's avatar
Raymond Knopp committed
343
344
345
346
  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD threads */
  /* CPU 1 is reserved for all TX threads */
  /* Enable CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);

#ifdef CPU_AFFINITY
  if (get_nprocs() > 2)
  {
    for (j = 1; j < get_nprocs(); j++)
        CPU_SET(j, &cpuset);
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
    if (s != 0)
    {
      perror( "pthread_setaffinity_np");
      exit_fun("Error setting processor affinity");
    }
  }
#endif //CPU_AFFINITY

  /* Check the actual affinity mask assigned to the thread */
  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
370
  if (s != 0) {
371
372
373
374
375
    perror( "pthread_getaffinity_np");
    exit_fun("Error getting processor affinity ");
  }
  memset(cpu_affinity,0,sizeof(cpu_affinity));
  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
376
377
378
379
380
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
381

Sandeep Kumar's avatar
Sandeep Kumar committed
382
  memset(&sparam, 0, sizeof(sparam));
383
384
385
386
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-1;
  policy = SCHED_FIFO ; 
  
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
387
388
389
390
391
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");
  }
  
392
  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
393
394
395
396
  if (s != 0) {
    perror("pthread_getschedparam : ");
    exit_fun("Error getting thread priority");
  }
397

Sandeep Kumar's avatar
Sandeep Kumar committed
398
  LOG_I(HW, "[SCHED][eNB] TX thread started on CPU %d TID %ld, sched_policy = %s , priority = %d, CPU Affinity=%s \n",sched_getcpu(),gettid(),
399
400
401
402
403
404
405
406
407
408
409
                   (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
                   (policy == SCHED_RR)    ? "SCHED_RR" :
                   (policy == SCHED_OTHER) ? "SCHED_OTHER" :
                   "???",
                   sparam.sched_priority, cpu_affinity );

#endif //LOW_LATENCY

  mlockall(MCL_CURRENT | MCL_FUTURE);

  while (!oai_exit) {
knopp's avatar
knopp committed
410
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
411

Raymond Knopp's avatar
Raymond Knopp committed
412
    if (pthread_mutex_timedlock(&proc->mutex_rxtx,&wait) != 0) {
knopp's avatar
knopp committed
413
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB RXn-TXnp4\n");
414
415
416
417
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
418
    while (proc->instance_cnt_rxtx < 0) {
419
      // most of the time the thread is waiting here
knopp's avatar
knopp committed
420
421
      // proc->instance_cnt_rxtx is -1
      pthread_cond_wait( &proc->cond_rxtx, &proc->mutex_rxtx ); // this unlocks mutex_rxtx while waiting and then locks it again
422
423
    }

knopp's avatar
knopp committed
424
    if (pthread_mutex_unlock(&proc->mutex_rxtx) != 0) {
425
      LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB TX\n");
426
427
428
429
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
430
431
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 1 );
    start_meas( &softmodem_stats_rxtx_sf );
432
  
433
434
    if (oai_exit) break;

knopp's avatar
knopp committed
435
436
437
438
439
440
441
442
443
    // UE-specific RX processing for subframe n
    if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RRU_IF4) {
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_UESPEC, 1 );
      // this is the ue-specific processing for the subframe and can be multi-threaded later
      phy_procedures_eNB_uespec_RX(PHY_vars_eNB_g[0][proc->CC_id], proc, 0, no_relay );
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_UESPEC, 0 );
    }

    // TX processing for subframe n+4
444
445
446
447
    if (((PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.frame_type == TDD) &&
         ((subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->frame_parms,proc->subframe_tx) == SF_DL) ||
          (subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->frame_parms,proc->subframe_tx) == SF_S))) ||
        (PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.frame_type == FDD)) {
448
449
450
      /* run PHY TX procedures the one after the other for all CCs to avoid race conditions
       * (may be relaxed in the future for performance reasons)
       */
knopp's avatar
knopp committed
451
      
Raymond Knopp's avatar
Raymond Knopp committed
452
      if (pthread_mutex_timedlock(&sync_phy_proc.mutex_phy_proc_tx,&wait) != 0) {
453
        LOG_E(PHY, "[SCHED][eNB] error locking PHY proc mutex for eNB TX\n");
454
455
456
        exit_fun("nothing to add");
        break;
      }
Sandeep Kumar's avatar
Sandeep Kumar committed
457
      
knopp's avatar
knopp committed
458
      // wait for our turn or oai_exit
459
460
461
      while (sync_phy_proc.phy_proc_CC_id != proc->CC_id && !oai_exit) {
        pthread_cond_wait(&sync_phy_proc.cond_phy_proc_tx,
                          &sync_phy_proc.mutex_phy_proc_tx);
462
463
      }

464
465
      if (pthread_mutex_unlock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
        LOG_E(PHY, "[SCHED][eNB] error unlocking PHY proc mutex for eNB TX\n");
466
467
468
        exit_fun("nothing to add");
      }

knopp's avatar
knopp committed
469
470
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_TX_ENB, proc->frame_tx );
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_TX_ENB, proc->subframe_tx );
Sandeep Kumar's avatar
Sandeep Kumar committed
471
472
473
      
      if (oai_exit) break;
      
474
      if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RRU_IF4) { 
Sandeep Kumar's avatar
Sandeep Kumar committed
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
        phy_procedures_eNB_TX(PHY_vars_eNB_g[0][proc->CC_id], proc, 0, no_relay, NULL );

        /* we're done, let the next one proceed */
        if (pthread_mutex_lock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
          LOG_E(PHY, "[SCHED][eNB] error locking PHY proc mutex for eNB TX proc\n");
          exit_fun("nothing to add");
          break;
        }	
        sync_phy_proc.phy_proc_CC_id++;
        sync_phy_proc.phy_proc_CC_id %= MAX_NUM_CCs;
        pthread_cond_broadcast(&sync_phy_proc.cond_phy_proc_tx);
        if (pthread_mutex_unlock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
          LOG_E(PHY, "[SCHED][eNB] error unlocking PHY proc mutex for eNB TX proc\n");
          exit_fun("nothing to add");
          break;
490
	}
491
492
      } else {

493
        /// **** recv_IF4 of txdataF from RCC **** ///             
Sandeep Kumar's avatar
Sandeep Kumar committed
494
495
496
497
        VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF4, 1 );  
        while (symbol_number < PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.symbols_per_tti-1) { 
          recv_IF4(PHY_vars_eNB_g[0][proc->CC_id], proc->frame_tx, proc->subframe_tx, &packet_type, &symbol_number);
        }
498
        VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF4, 0 );   
499
        
500
501
502
      }
    }

Sandeep Kumar's avatar
Sandeep Kumar committed
503
    // eNodeB_3GPP and RRU create txdata and write to RF device
504
    if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RCC_IF4) {
knopp's avatar
knopp committed
505
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_SFGEN , 1 );
506
      do_OFDM_mod_rt( proc->subframe_tx, PHY_vars_eNB_g[0][proc->CC_id] );
knopp's avatar
knopp committed
507
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_SFGEN , 0 );
508
  
Sandeep Kumar's avatar
Sandeep Kumar committed
509
510
511
512
513
514
515
516
517
518
519
520
521
      /*
        short *txdata = (short*)&PHY_vars_eNB_g[0][proc->CC_id]->common_vars.txdata[0][0][proc->subframe_tx*PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.samples_per_tti];
        int i;
        for (i=0;i<PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.samples_per_tti*2;i+=8) {
        txdata[i] = 2047;
        txdata[i+1] = 0;
        txdata[i+2] = 0;
        txdata[i+3] = 2047;
        txdata[i+4] = -2047;
        txdata[i+5] = 0;
        txdata[i+6] = 0;
        txdata[i+7] = -2047;      }
      */
knopp's avatar
knopp committed
522

Sandeep Kumar's avatar
Sandeep Kumar committed
523
      // Transmit TX buffer based on timestamp from RX  
knopp's avatar
knopp committed
524
525
526
527
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 1 );
      // prepare tx buffer pointers
      int i;
      for (i=0; i<PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx; i++)
Sandeep Kumar's avatar
Sandeep Kumar committed
528
529
        txp[i] = (void*)&PHY_vars_eNB_g[0][0]->common_vars.txdata[0][i][proc->subframe_tx*PHY_vars_eNB_g[0][0]->frame_parms.samples_per_tti];
    
knopp's avatar
knopp committed
530
      // if symb_written < spp ==> error 
Sandeep Kumar's avatar
Sandeep Kumar committed
531
532
533
534
535
536
537
      PHY_vars_eNB_g[0][proc->CC_id]->rfdevice.trx_write_func(&PHY_vars_eNB_g[0][proc->CC_id]->rfdevice,
            (proc->timestamp_tx-openair0_cfg[0].tx_sample_advance),
            txp,
            PHY_vars_eNB_g[0][0]->frame_parms.samples_per_tti,
            PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx,
            1);
	      
knopp's avatar
knopp committed
538
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 0 );
Sandeep Kumar's avatar
Sandeep Kumar committed
539
 
knopp's avatar
knopp committed
540
541
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TST, (proc->timestamp_tx-openair0_cfg[0].tx_sample_advance)&0xffffffff );

Sandeep Kumar's avatar
Sandeep Kumar committed
542
543
    } else { 
	 
544
      /// **** send_IF4 of txdataF to RRU **** ///       
545
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_SEND_IF4, 1 );   
Sandeep Kumar's avatar
Sandeep Kumar committed
546
      send_IF4(PHY_vars_eNB_g[0][proc->CC_id], proc->frame_tx, proc->subframe_tx, IF4_PDLFFT, 0);
547
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_SEND_IF4, 0 );
548

549
550
    }

knopp's avatar
knopp committed
551
    if (pthread_mutex_lock(&proc->mutex_rxtx) != 0) {
552
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB TX proc\n");
553
554
555
556
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
557
    proc->instance_cnt_rxtx--;
558

knopp's avatar
knopp committed
559
    if (pthread_mutex_unlock(&proc->mutex_rxtx) != 0) {
560
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB TX proc\n");
561
562
563
564
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
565
    stop_meas( &softmodem_stats_rxtx_sf );
Sandeep Kumar's avatar
Sandeep Kumar committed
566

567
#ifdef DEADLINE_SCHEDULER
568
    if (opp_enabled){
knopp's avatar
knopp committed
569
570
      if(softmodem_stats_rxtx_sf.diff_now/(cpuf) > attr.sched_runtime){
	VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_RUNTIME_TX_ENB, (softmodem_stats_rxtx_sf.diff_now/cpuf - attr.sched_runtime)/1000000.0);
571
572
573
    }
#endif 

Sandeep Kumar's avatar
Sandeep Kumar committed
574
    print_meas_now(&softmodem_stats_rxtx_sf,"eNB_TX_SF",tx_time_file);
575
576
  }

knopp's avatar
knopp committed
577
  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
578

Raymond Knopp's avatar
Raymond Knopp committed
579
580
581

  printf( "Exiting eNB thread RXn_TXnp4\n");

582

knopp's avatar
knopp committed
583
584
  eNB_thread_rxtx_status = 0;
  return &eNB_thread_rxtx_status;
585
586
}

587
#if defined(ENABLE_ITTI)
Sandeep Kumar's avatar
Sandeep Kumar committed
588
589
590
/* Wait for eNB application initialization to be complete (eNB registration to MME) */
static void wait_system_ready (char *message, volatile int *start_flag) {
  
591
  static char *indicator[] = {".    ", "..   ", "...  ", ".... ", ".....",
Sandeep Kumar's avatar
Sandeep Kumar committed
592
			      " ....", "  ...", "   ..", "    .", "     "};
593
594
595
596
597
598
599
  int i = 0;
  
  while ((!oai_exit) && (*start_flag == 0)) {
    LOG_N(EMU, message, indicator[i]);
    fflush(stdout);
    i = (i + 1) % (sizeof(indicator) / sizeof(indicator[0]));
    usleep(200000);
600
  }
601
602
  
  LOG_D(EMU,"\n");
603
604
}
#endif
605

Sandeep Kumar's avatar
Sandeep Kumar committed
606

607
/*!
knopp's avatar
knopp committed
608
 * \brief The RX common thread of eNB.
609
610
611
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
612
613
static void* eNB_thread_rx_common( void* param ) {
  
614
  static int eNB_thread_rx_status;
615
616

  eNB_proc_t *proc = (eNB_proc_t*)param;
617
618
  PHY_VARS_eNB *eNB = PHY_vars_eNB_g[0][proc->CC_id];
  LTE_DL_FRAME_PARMS *fp = &eNB->frame_parms;
619
620
621
622

  FILE  *rx_time_file = NULL;
  char rx_time_name[101];
  //int i;
Raymond Knopp's avatar
Raymond Knopp committed
623
624
625
626
  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;
627

Sandeep Kumar's avatar
Sandeep Kumar committed
628
  if (opp_enabled == 1) {
629
    snprintf(rx_time_name, 100,"/tmp/%s_rx_time_thread_sf", "eNB");
630
631
632
    rx_time_file = fopen(rx_time_name,"w");
  }
  // set default return value
633
  eNB_thread_rx_status = 0;
634
635
636

  MSC_START_USE();

637
#ifdef DEADLINE_SCHEDULER
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
  struct sched_attr attr;
  unsigned int flags = 0;
  uint64_t runtime  = 870000 ;
  uint64_t deadline = 1   *  1000000;
  uint64_t period   = 1   * 10000000; // each rx thread has a period of 10ms from the starting point
 
  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB RX sched_setattr failed\n");
656
    return &eNB_thread_rx_status;
657
658
  }

659
  LOG_I( HW, "[SCHED] eNB RX deadline thread (TID %ld) started on CPU %d\n", gettid(), sched_getcpu() );
660
661
662
663
664
665
666
667
668
669
670
671
672
#else // LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD */
  /* CPU 1 is reserved for all TX threads */
  /* CPU 2..MAX_CPUS is reserved for all RX threads */
  /* Set CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);
#ifdef CPU_AFFINITY
Sandeep Kumar's avatar
Sandeep Kumar committed
673
  if (get_nprocs() >2) {
674
    for (j = 1; j < get_nprocs(); j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
675
      CPU_SET(j, &cpuset);
676
677
  
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
678
    if (s != 0) {
679
680
681
682
683
684
685
686
      perror( "pthread_setaffinity_np");  
      exit_fun (" Error setting processor affinity :");
    }
  }
#endif //CPU_AFFINITY
  /* Check the actual affinity mask assigned to the thread */

  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
687
688
689
  if (s != 0) {
    perror ("pthread_getaffinity_np");
    exit_fun (" Error getting processor affinity :");
690
691
692
693
  }
  memset(cpu_affinity,0, sizeof(cpu_affinity));

  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
694
695
696
697
698
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
699
700

  memset(&sparam, 0 , sizeof (sparam)); 
knopp's avatar
knopp committed
701
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO);
702
703
704

  policy = SCHED_FIFO ; 
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
705
706
707
708
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");     
  }
709
710
711
712

  memset(&sparam, 0 , sizeof (sparam));

  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
713
714
715
716
  if (s != 0) {
    perror("pthread_getschedparam");
    exit_fun("Error getting thread priority");
  }
717

Sandeep Kumar's avatar
Sandeep Kumar committed
718
  LOG_I(HW, "[SCHED][eNB] RX thread started on CPU %d TID %ld, sched_policy = %s, priority = %d, CPU Affinity = %s\n", sched_getcpu(),gettid(),
719
720
721
722
723
724
725
	 (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
	 (policy == SCHED_RR)    ? "SCHED_RR" :
	 (policy == SCHED_OTHER) ? "SCHED_OTHER" :
	 "???",
	 sparam.sched_priority, cpu_affinity);
  
  
726
#endif // DEADLINE_SCHEDULER
727
728


Sandeep Kumar's avatar
Sandeep Kumar committed
729
  mlockall(MCL_CURRENT | MCL_FUTURE);
730

Sandeep Kumar's avatar
Sandeep Kumar committed
731
732
733
  // wait for top-level synchronization and do one acquisition to get timestamp for setting frame/subframe of TX and RX threads
  printf( "waiting for sync (eNB_thread_rx_common)\n");
  pthread_mutex_lock( &sync_mutex );
734

Sandeep Kumar's avatar
Sandeep Kumar committed
735
736
  while (sync_var<0)
    pthread_cond_wait( &sync_cond, &sync_mutex );
737
 
Sandeep Kumar's avatar
Sandeep Kumar committed
738
  pthread_mutex_unlock(&sync_mutex);
739
 
Sandeep Kumar's avatar
Sandeep Kumar committed
740
  printf( "got sync (eNB_thread)\n" );
741
742
743
744
 
#if defined(ENABLE_ITTI)
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif 
745
746
  
  // Start RF device for this CC
747
  if (eNB->node_function != NGFI_RCC_IF4) {
748
749
750
751
752
    if (eNB->rfdevice.trx_start_func(&eNB->rfdevice) != 0 ) 
      LOG_E(HW,"Could not start the RF device\n");
  }
    
  // Start IF device for this CC
753
  if (eNB->node_function != eNodeB_3GPP) {
754
755
756
757
    if (eNB->ifdevice.trx_start_func(&eNB->ifdevice) != 0 ) 
      LOG_E(HW,"Could not start the IF device\n");
  }

Sandeep Kumar's avatar
Sandeep Kumar committed
758
759
760
761
762
  // This is a forever while loop, it loops over subframes which are scheduled by incoming samples from HW devices
  while (!oai_exit) {
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX, 0 );
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 0 );
    start_meas( &softmodem_stats_rx_sf );
763
   
Sandeep Kumar's avatar
Sandeep Kumar committed
764
    if (oai_exit) break;
765
   
Sandeep Kumar's avatar
Sandeep Kumar committed
766
767
768
769
770
771
772
773
    if ((((fp->frame_type == TDD )&&(subframe_select(fp,proc->subframe_rx)==SF_UL)) ||
	 (fp->frame_type == FDD))) {
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 1 );
      // this spawns the prach inside and updates the frame and subframe counters
      phy_procedures_eNB_common_RX(eNB, 0);
      
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 0 );
    }
knopp's avatar
knopp committed
774

Sandeep Kumar's avatar
Sandeep Kumar committed
775
776
777
778
779
    // choose even or odd thread for RXn-TXnp4 processing 
    eNB_rxtx_proc_t *proc_rxtx = &proc->proc_rxtx[proc->subframe_rx&1];

    // wake up TX for subframe n+4
    // lock the TX mutex and make sure the thread is ready
Raymond Knopp's avatar
Raymond Knopp committed
780
    if (pthread_mutex_timedlock(&proc_rxtx->mutex_rxtx,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
      LOG_E( PHY, "[eNB] ERROR pthread_mutex_lock for eNB TX thread %d (IC %d)\n", proc_rxtx->instance_cnt_rxtx );
      exit_fun( "error locking mutex_rxtx" );
      break;
    }
    int cnt_rxtx = ++proc_rxtx->instance_cnt_rxtx;
    // We have just received and processed the common part of a subframe, say n. 
    // TS_rx is the last received timestamp (start of 1st slot), TS_tx is the desired 
    // transmitted timestamp of the next TX slot (first).
    // The last (TS_rx mod samples_pexr_frame) was n*samples_per_tti, 
    // we want to generate subframe (n+3), so TS_tx = TX_rx+3*samples_per_tti,
    // and proc->subframe_tx = proc->subframe_rx+3
    proc_rxtx->timestamp_tx = proc->timestamp_rx + (4*fp->samples_per_tti);
    proc_rxtx->frame_rx     = proc->frame_rx;
    proc_rxtx->subframe_rx  = proc->subframe_rx;
    proc_rxtx->frame_tx     = (proc_rxtx->subframe_rx > 5) ? (proc_rxtx->frame_rx+1)&1023 : proc_rxtx->frame_rx;
    proc_rxtx->subframe_tx  = (proc_rxtx->subframe_rx + 4)%10;
knopp's avatar
knopp committed
797
   
Sandeep Kumar's avatar
Sandeep Kumar committed
798
    pthread_mutex_unlock( &proc_rxtx->mutex_rxtx );
knopp's avatar
knopp committed
799
   
Sandeep Kumar's avatar
Sandeep Kumar committed
800
801
802
803
804
805
806
807
808
809
810
811
    if (cnt_rxtx == 0) {
      // the thread was presumably waiting where it should and can now be woken up
      if (pthread_cond_signal(&proc_rxtx->cond_rxtx) != 0) {
        LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB RXn-TXnp4 thread\n");
        exit_fun( "ERROR pthread_cond_signal" );
        break;
      }
    } else {
      LOG_W( PHY,"[eNB] Frame %d, eNB RXn-TXnp4 thread busy!! (cnt_rxtx %i)\n", proc_rxtx->frame_tx, cnt_rxtx );
      exit_fun( "TX thread busy" );
      break;
    }       
812
   
Sandeep Kumar's avatar
Sandeep Kumar committed
813

814
    stop_meas( &softmodem_stats_rxtx_sf );
815
#ifdef DEADLINE_SCHEDULER
Sandeep Kumar's avatar
Sandeep Kumar committed
816
817
    if (opp_enabled){
      if(softmodem_stats_rxtx_sf.diff_now/(cpuf) > attr.sched_runtime){
818
	VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_RUNTIME_RXTX_ENB, (softmodem_stats_rxtx_sf.diff_now/cpuf - attr.sched_runtime)/1000000.0);
Sandeep Kumar's avatar
Sandeep Kumar committed
819
820
      }
    }
821
#endif // DEADLINE_SCHEDULER  
822
823
824
    print_meas_now(&softmodem_stats_rx_sf,"eNB_RX_SF", rx_time_file);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
    
Sandeep Kumar's avatar
Sandeep Kumar committed
825
826
827
    print_meas_now(&softmodem_stats_rx_sf,"eNB_RX_SF", rx_time_file);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
  }
knopp's avatar
knopp committed
828
 
Raymond Knopp's avatar
Raymond Knopp committed
829
830
831

  printf( "Exiting eNB thread rx_common\n");

knopp's avatar
knopp committed
832
 
Sandeep Kumar's avatar
Sandeep Kumar committed
833
834
  eNB_thread_rx_status = 0;
  return &eNB_thread_rx_status;
835
836
837
838
839
840
841
842
}


/*!
 * \brief The prach receive thread of eNB.
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
843
static void* eNB_thread_prach( void* param ) {
844
845
  static int eNB_thread_prach_status;

Raymond Knopp's avatar
Raymond Knopp committed
846
847
 

848
849
  eNB_proc_t *proc = (eNB_proc_t*)param;
  PHY_VARS_eNB *eNB= PHY_vars_eNB_g[0][proc->CC_id];
Raymond Knopp's avatar
Raymond Knopp committed
850
851
852
853
854
855

  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;

856
857
858
859
860
  // set default return value
  eNB_thread_prach_status = 0;

  MSC_START_USE();
    
861
#ifdef DEADLINE_SCHEDULER
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
  struct sched_attr attr;
  unsigned int flags = 0;
  uint64_t runtime  = 870000 ;
  uint64_t deadline = 1   *  1000000;
  uint64_t period   = 1   * 10000000; // each prach thread has a period of 10ms from the starting point
 
  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB PRACH sched_setattr failed\n");
    return &eNB_thread_prach_status;
  }

  LOG_I( HW, "[SCHED] eNB PRACH deadline thread (TID %ld) started on CPU %d\n", 0, gettid(), sched_getcpu() );
#else // LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD */
  /* CPU 1 is reserved for all TX threads */
  /* CPU 2..MAX_CPUS is reserved for all RX threads */
  /* Set CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);
#ifdef CPU_AFFINITY
Sandeep Kumar's avatar
Sandeep Kumar committed
897
  if (get_nprocs() >2) {
898
    for (j = 1; j < get_nprocs(); j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
899
      CPU_SET(j, &cpuset);
900
901
  
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
902
    if (s != 0) {
903
904
905
906
907
908
      perror( "pthread_setaffinity_np");  
      exit_fun (" Error setting processor affinity :");
    }
  }
#endif //CPU_AFFINITY

Sandeep Kumar's avatar
Sandeep Kumar committed
909
  /* Check the actual affinity mask assigned to the thread */
910
  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
911
912
913
  if (s != 0) {
    perror ("pthread_getaffinity_np");
    exit_fun (" Error getting processor affinity :");
914
915
916
917
  }
  memset(cpu_affinity,0, sizeof(cpu_affinity));

  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
918
919
920
921
922
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
923
924

  memset(&sparam, 0 , sizeof (sparam)); 
knopp's avatar
knopp committed
925
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-2;
926
927
928

  policy = SCHED_FIFO ; 
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
929
930
931
932
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");
  }
933
934
935
936

  memset(&sparam, 0 , sizeof (sparam));

  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
937
938
939
940
  if (s != 0) {
    perror("pthread_getschedparam");
    exit_fun("Error getting thread priority");
  }
941

Raymond Knopp's avatar
Raymond Knopp committed
942
  LOG_I(HW, "[SCHED][eNB] PRACH thread started on CPU %d TID %ld, IC %d, sched_policy = %s, priority = %d, CPU Affinity = %s\n", sched_getcpu(),gettid(),proc->instance_cnt_prach,
943
944
945
946
947
948
	 (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
	 (policy == SCHED_RR)    ? "SCHED_RR" :
	 (policy == SCHED_OTHER) ? "SCHED_OTHER" :
	 "???",
	 sparam.sched_priority, cpu_affinity);
  
949
#endif // DEADLINE_SCHEDULER
950

Sandeep Kumar's avatar
Sandeep Kumar committed
951
  mlockall(MCL_CURRENT | MCL_FUTURE);
952

Sandeep Kumar's avatar
Sandeep Kumar committed
953
954
955
  while (!oai_exit) {
    
    if (oai_exit) break;
956
        
Raymond Knopp's avatar
Raymond Knopp committed
957
    if (pthread_mutex_timedlock(&proc->mutex_prach,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB PRACH\n");
      exit_fun( "error locking mutex" );
      break;
    }

    while (proc->instance_cnt_prach < 0) {
      // most of the time the thread is waiting here
      // proc->instance_cnt_prach is -1
      pthread_cond_wait( &proc->cond_prach, &proc->mutex_prach ); // this unlocks mutex_rxtx while waiting and then locks it again
    }

    if (pthread_mutex_unlock(&proc->mutex_prach) != 0) {
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB PRACH\n");
      exit_fun( "error unlocking mutex" );
      break;
    }
974
   
Sandeep Kumar's avatar
Sandeep Kumar committed
975
976
977
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_PRACH_RX,1);
    prach_procedures(eNB,0);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_PRACH_RX,0);
978
    
Raymond Knopp's avatar
Raymond Knopp committed
979
    if (pthread_mutex_timedlock(&proc->mutex_prach,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
980
981
982
983
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB PRACH proc %d\n");
      exit_fun( "error locking mutex" );
      break;
    }
984
   
Sandeep Kumar's avatar
Sandeep Kumar committed
985
986
987
988
989
990
991
992
    proc->instance_cnt_prach--;
    
    if (pthread_mutex_unlock(&proc->mutex_prach) != 0) {
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB RX proc %d\n");
      exit_fun( "error unlocking mutex" );
      break;
    } 
  }
993

Raymond Knopp's avatar
Raymond Knopp committed
994

995
  printf( "Exiting eNB thread PRACH\n");
Raymond Knopp's avatar
Raymond Knopp committed
996

997

998
999
  eNB_thread_prach_status = 0;
  return &eNB_thread_prach_status;
1000
1001
1002
}


Sandeep Kumar's avatar
Sandeep Kumar committed
1003
1004
void init_eNB_proc(void) {
  
1005
1006
  int i;
  int CC_id;
1007
1008
  PHY_VARS_eNB *eNB;
  eNB_proc_t *proc;
knopp's avatar
knopp committed
1009
  eNB_rxtx_proc_t *proc_rxtx;
1010
1011

  for (CC_id=0; CC_id<MAX_NUM_CCs; CC_id++) {
1012
1013
    eNB = PHY_vars_eNB_g[0][CC_id];

1014
    proc = &eNB->proc;
knopp's avatar
knopp committed
1015
    proc_rxtx = proc->proc_rxtx;
1016
#ifndef DEADLINE_SCHEDULER 
1017
1018
1019
    /*  
	pthread_attr_init( &attr_eNB_proc_tx[CC_id][i] );
	if (pthread_attr_setstacksize( &attr_eNB_proc_tx[CC_id][i], 64 *PTHREAD_STACK_MIN ) != 0)
1020
        perror("[ENB_PROC_TX] setting thread stack size failed\n");
1021
1022
1023
	
	pthread_attr_init( &attr_eNB_proc_rx[CC_id][i] );
	if (pthread_attr_setstacksize( &attr_eNB_proc_rx[CC_id][i], 64 * PTHREAD_STACK_MIN ) != 0)
1024
        perror("[ENB_PROC_RX] setting thread stack size failed\n");
1025
1026
    */
    // set the kernel scheduling policy and priority
knopp's avatar
knopp committed
1027
1028
1029
1030
1031
1032
    proc_rxtx[0].sched_param_rxtx.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc_rxtx[0].attr_rxtx, &proc_rxtx[0].sched_param_rxtx);
    pthread_attr_setschedpolicy (&proc_rxtx[0].attr_rxtx, SCHED_FIFO);
    proc_rxtx[1].sched_param_rxtx.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc_rxtx[1].attr_rxtx, &proc_rxtx[1].sched_param_rxtx);
    pthread_attr_setschedpolicy (&proc_rxtx[1].attr_rxtx, SCHED_FIFO);
1033
    
knopp's avatar
knopp committed
1034
    proc->sched_param_rx.sched_priority = sched_get_priority_max(SCHED_FIFO); //OPENAIR_THREAD_PRIORITY;
1035
1036
1037
1038
1039
1040
1041
1042
    pthread_attr_setschedparam  (&proc->attr_rx, &proc->sched_param_rx);
    pthread_attr_setschedpolicy (&proc->attr_rx, SCHED_FIFO);
    
    proc->sched_param_prach.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc->attr_prach, &proc->sched_param_prach);
    pthread_attr_setschedpolicy (&proc->attr_prach, SCHED_FIFO);
    
    printf("Setting OS scheduler to SCHED_FIFO for eNB [cc%d][thread%d] \n",CC_id, i);
1043
#endif
knopp's avatar
knopp committed
1044
1045
    proc_rxtx[0].instance_cnt_rxtx = -1;
    proc_rxtx[1].instance_cnt_rxtx = -1;
1046
1047
    proc->instance_cnt_prach = -1;
    proc->CC_id = CC_id;
1048
1049
1050

    proc->first_rx=1;

knopp's avatar
knopp committed
1051
1052
1053
1054
1055
    pthread_mutex_init( &proc_rxtx[0].mutex_rxtx, NULL);
    pthread_mutex_init( &proc_rxtx[1].mutex_rxtx, NULL);
    pthread_mutex_init( &proc->mutex_prach, NULL);
    pthread_cond_init( &proc_rxtx[0].cond_rxtx, NULL);
    pthread_cond_init( &proc_rxtx[1].cond_rxtx, NULL);
1056
    pthread_cond_init( &proc->cond_prach, NULL);
1057
#ifndef DEADLINE_SCHEDULER
knopp's avatar
knopp committed
1058
1059
1060
    pthread_create( &proc_rxtx[0].pthread_rxtx, &proc_rxtx[0].attr_rxtx, eNB_thread_rxtx, &proc_rxtx[0] );
    pthread_create( &proc_rxtx[1].pthread_rxtx, &proc_rxtx[1].attr_rxtx, eNB_thread_rxtx, &proc_rxtx[1] );
    pthread_create( &proc->pthread_rx, &proc->attr_rx, eNB_thread_rx_common, &eNB->proc );
1061
    pthread_create( &proc->pthread_prach, &proc->attr_prach, eNB_thread_prach, &eNB->proc );
1062
#else 
knopp's avatar
knopp committed
1063
1064
1065
    pthread_create( &proc_rxtx[0].pthread_rxtx, NULL, eNB_thread_rxtx, &eNB->proc_rxtx[0] );
    pthread_create( &proc_rxtx[1].pthread_rxtx, NULL, eNB_thread_rxtx, &eNB->proc_rxtx[1] );
    pthread_create( &proc->pthread_rx, NULL, eNB_thread_rx_common, &eNB->proc );
1066
    pthread_create( &proc->pthread_prach, NULL, eNB_thread_prach, &eNB->proc );
1067
#endif
1068
    char name[16];
knopp's avatar
knopp committed
1069
1070
1071
1072
    snprintf( name, sizeof(name), "RXTX0 %d", i );
    pthread_setname_np( proc_rxtx[0].pthread_rxtx, name );
    snprintf( name, sizeof(name), "RXTX1 %d", i );
    pthread_setname_np( proc_rxtx[1].pthread_rxtx, name );
1073
1074
    snprintf( name, sizeof(name), "RX %d", i );
    pthread_setname_np( proc->pthread_rx, name );
1075
  }
1076
  
1077
  /* setup PHY proc TX sync mechanism */
1078
1079
1080
  pthread_mutex_init(&sync_phy_proc.mutex_phy_proc_tx, NULL);
  pthread_cond_init(&sync_phy_proc.cond_phy_proc_tx, NULL);
  sync_phy_proc.phy_proc_CC_id = 0;
1081
1082
}

1083

1084
1085
1086
/*!
 * \brief Terminate eNB TX and RX threads.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
1087
1088
void kill_eNB_proc(void) {

1089
  int *status;
1090
1091
  PHY_VARS_eNB *eNB;
  eNB_proc_t *proc;
knopp's avatar
knopp committed
1092
  eNB_rxtx_proc_t *proc_rxtx;
1093
1094
  for (int CC_id=0; CC_id<MAX_NUM_CCs; CC_id++) {
    eNB=PHY_vars_eNB_g[0][CC_id];
1095
1096
    
    proc = &eNB->proc;
knopp's avatar
knopp committed
1097
    proc_rxtx = &proc->proc_rxtx[0];
1098
    
1099
#ifdef DEBUG_THREADS
1100
    printf( "Killing TX CC_id %d thread %d\n", CC_id, i );
1101
#endif
1102
    
knopp's avatar
knopp committed
1103
1104
    proc_rxtx[0].instance_cnt_rxtx = 0; // FIXME data race!
    proc_rxtx[1].instance_cnt_rxtx = 0; // FIXME data race!
Raymond Knopp's avatar
Raymond Knopp committed
1105
    proc->instance_cnt_prach = 0;
knopp's avatar
knopp committed
1106
    pthread_cond_signal( &proc_rxtx[0].cond_rxtx );    
Raymond Knopp's avatar
Raymond Knopp committed
1107
1108
    pthread_cond_signal( &proc_rxtx[1].cond_rxtx );
    pthread_cond_signal( &proc->cond_prach );
1109
    pthread_cond_broadcast(&sync_phy_proc.cond_phy_proc_tx);
Raymond Knopp's avatar
Raymond Knopp committed
1110
1111
1112

    pthread_join( proc->pthread_rx, (void**)&status );    

Sandeep Kumar's avatar
Sandeep Kumar committed
1113

knopp's avatar
knopp committed
1114
    int result,i;
Raymond Knopp's avatar
Raymond Knopp committed
1115
    for (i=0;i<2;i++) {
knopp's avatar
knopp committed
1116
      pthread_join( proc_rxtx[i].pthread_rxtx, (void**)&status );
1117
    
1118
#ifdef DEBUG_THREADS
1119
    
knopp's avatar
knopp committed
1120
1121
      if (result != 0) {
	printf( "Error joining thread.\n" );
1122
      } else {
knopp's avatar
knopp committed
1123
1124
1125
1126
1127
	if (status) {
	  printf( "status %d\n", *status );
	} else {
	  printf( "The thread was killed. No status available.\n" );
	}
1128
1129
      }
#else
knopp's avatar