lte-enb.c 45.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
/*******************************************************************************
    OpenAirInterface
    Copyright(c) 1999 - 2014 Eurecom

    OpenAirInterface is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.


    OpenAirInterface is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with OpenAirInterface.The full GNU General Public License is
    included in this distribution in the file called "COPYING". If not,
    see <http://www.gnu.org/licenses/>.

   Contact Information
   OpenAirInterface Admin: openair_admin@eurecom.fr
   OpenAirInterface Tech : openair_tech@eurecom.fr
   OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr

   Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

*******************************************************************************/

/*! \file lte-enb.c
 * \brief Top-level threads for eNodeB
 * \author R. Knopp, F. Kaltenberger, Navid Nikaein
 * \date 2012
 * \version 0.1
 * \company Eurecom
 * \email: knopp@eurecom.fr,florian.kaltenberger@eurecom.fr, navid.nikaein@eurecom.fr
 * \note
 * \warning
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sched.h>
#include <linux/sched.h>
#include <signal.h>
#include <execinfo.h>
#include <getopt.h>
#include <sys/sysinfo.h>
#include "rt_wrapper.h"

#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all

#include "assertions.h"
#include "msc.h"

#include "PHY/types.h"

#include "PHY/defs.h"
#undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

#include "../../ARCH/COMMON/common_lib.h"

//#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all

71
72
#include "PHY/LTE_TRANSPORT/if4_tools.h"

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#include "PHY/extern.h"
#include "SCHED/extern.h"
#include "LAYER2/MAC/extern.h"

#include "../../SIMU/USER/init_lte.h"

#include "LAYER2/MAC/defs.h"
#include "LAYER2/MAC/extern.h"
#include "LAYER2/MAC/proto.h"
#include "RRC/LITE/extern.h"
#include "PHY_INTERFACE/extern.h"

#ifdef SMBV
#include "PHY/TOOLS/smbv.h"
unsigned short config_frames[4] = {2,9,11,13};
#endif
#include "UTIL/LOG/log_extern.h"
#include "UTIL/OTG/otg_tx.h"
#include "UTIL/OTG/otg_externs.h"
#include "UTIL/MATH/oml.h"
#include "UTIL/LOG/vcd_signal_dumper.h"
#include "UTIL/OPT/opt.h"
#include "enb_config.h"
//#include "PHY/TOOLS/time_meas.h"

#ifndef OPENAIR2
#include "UTIL/OTG/otg_extern.h"
#endif

#if defined(ENABLE_ITTI)
# if defined(ENABLE_USE_MME)
#   include "s1ap_eNB.h"
#ifdef PDCP_USE_NETLINK
#   include "SIMULATION/ETH_TRANSPORT/proto.h"
#endif
# endif
#endif

//#define DEBUG_THREADS 1

//#define USRP_DEBUG 1
struct timing_info_t {
  //unsigned int frame, hw_slot, last_slot, next_slot;
  RTIME time_min, time_max, time_avg, time_last, time_now;
  //unsigned int mbox0, mbox1, mbox2, mbox_target;
  unsigned int n_samples;
} timing_info;

121
122
// Fix per CC openair rf/if device update
// extern openair0_device openair0;
123
124
125
126
127
128
129

#if defined(ENABLE_ITTI)
extern volatile int             start_eNB;
extern volatile int             start_UE;
#endif
extern volatile int                    oai_exit;

Raymond Knopp's avatar
Raymond Knopp committed
130
extern openair0_config_t openair0_cfg[MAX_CARDS];
131
132
133
134
135

extern pthread_cond_t sync_cond;
extern pthread_mutex_t sync_mutex;
extern int sync_var;

136
//pthread_t                       main_eNB_thread;
137
138
139

time_stats_t softmodem_stats_mt; // main thread
time_stats_t softmodem_stats_hw; //  hw acquisition
knopp's avatar
knopp committed
140
time_stats_t softmodem_stats_rxtx_sf; // total tx time
141
time_stats_t softmodem_stats_rx_sf; // total rx time
142
143
144
145
146
147
148
149
150
151
152
153
154
int32_t **rxdata;
int32_t **txdata;

static int                      time_offset[4] = {0,0,0,0};

/* mutex, cond and variable to serialize phy proc TX calls
 * (this mechanism may be relaxed in the future for better
 * performances)
 */
static struct {
  pthread_mutex_t  mutex_phy_proc_tx;
  pthread_cond_t   cond_phy_proc_tx;
  volatile uint8_t phy_proc_CC_id;
155
} sync_phy_proc;
156
157
158

void exit_fun(const char* s);

159
void init_eNB(eNB_func_t node_function);
160
161
162
void stop_eNB(void);


Sandeep Kumar's avatar
Sandeep Kumar committed
163
164
void do_OFDM_mod_rt(int subframe,PHY_VARS_eNB *phy_vars_eNB) {
     
165
166
  unsigned int aa,slot_offset, slot_offset_F;
  int dummy_tx_b[7680*4] __attribute__((aligned(32)));
knopp's avatar
knopp committed
167
  int i,j, tx_offset;
168
169
  int slot_sizeF = (phy_vars_eNB->frame_parms.ofdm_symbol_size)*
                   ((phy_vars_eNB->frame_parms.Ncp==1) ? 6 : 7);
knopp's avatar
knopp committed
170
171
  int len,len2;
  int16_t *txdata;
172
173
174

  slot_offset_F = (subframe<<1)*slot_sizeF;

175
  slot_offset = subframe*phy_vars_eNB->frame_parms.samples_per_tti;
176

177
178
  if ((subframe_select(&phy_vars_eNB->frame_parms,subframe)==SF_DL)||
      ((subframe_select(&phy_vars_eNB->frame_parms,subframe)==SF_S))) {
179
180
    //    LOG_D(HW,"Frame %d: Generating slot %d\n",frame,next_slot);

181
182
183
    for (aa=0; aa<phy_vars_eNB->frame_parms.nb_antennas_tx; aa++) {
      if (phy_vars_eNB->frame_parms.Ncp == EXTENDED) {
        PHY_ofdm_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F],
184
                     dummy_tx_b,
185
                     phy_vars_eNB->frame_parms.ofdm_symbol_size,
186
                     6,
187
                     phy_vars_eNB->frame_parms.nb_prefix_samples,
188
                     CYCLIC_PREFIX);
189
190
191
        PHY_ofdm_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
                     dummy_tx_b+(phy_vars_eNB->frame_parms.samples_per_tti>>1),
                     phy_vars_eNB->frame_parms.ofdm_symbol_size,
192
                     6,
193
                     phy_vars_eNB->frame_parms.nb_prefix_samples,
194
195
                     CYCLIC_PREFIX);
      } else {
196
        normal_prefix_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F],
197
198
                          dummy_tx_b,
                          7,
199
                          &(phy_vars_eNB->frame_parms));
200
	// if S-subframe generate first slot only
Sandeep Kumar's avatar
Sandeep Kumar committed
201
	if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_DL) 
202
203
	  normal_prefix_mod(&phy_vars_eNB->common_vars.txdataF[0][aa][slot_offset_F+slot_sizeF],
			    dummy_tx_b+(phy_vars_eNB->frame_parms.samples_per_tti>>1),
204
			    7,
205
			    &(phy_vars_eNB->frame_parms));
206
207
208
      }

      // if S-subframe generate first slot only
209
210
      if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_S)
	len = phy_vars_eNB->frame_parms.samples_per_tti>>1;
211
      else
212
	len = phy_vars_eNB->frame_parms.samples_per_tti;
213
214
215
216
217
218
219
      /*
      for (i=0;i<len;i+=4) {
	dummy_tx_b[i] = 0x100;
	dummy_tx_b[i+1] = 0x01000000;
	dummy_tx_b[i+2] = 0xff00;
	dummy_tx_b[i+3] = 0xff000000;
	}*/
knopp's avatar
knopp committed
220
221
222
223
224
225
226
227
228
229
230
231
232
233
      
      if (slot_offset+time_offset[aa]<0) {
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti)+tx_offset];
        len2 = -(slot_offset+time_offset[aa]);
	len2 = (len2>len) ? len : len2;
	for (i=0; i<(len2<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
	if (len2<len) {
	  txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][0];
	  for (j=0; i<(len<<1); i++,j++) {
	    txdata[j++] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	  }
	}
Sandeep Kumar's avatar
Sandeep Kumar committed
234
      }  
knopp's avatar
knopp committed
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
      else if ((slot_offset+time_offset[aa]+len)>(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti)) {
	tx_offset = (int)slot_offset+time_offset[aa];
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][tx_offset];
	len2 = -tx_offset+LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
	for (i=0; i<(len2<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][0];
	for (j=0; i<(len<<1); i++,j++) {
	  txdata[j++] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
      }
      else {
	tx_offset = (int)slot_offset+time_offset[aa];
	txdata = (int16_t*)&phy_vars_eNB->common_vars.txdata[0][aa][tx_offset];
250

knopp's avatar
knopp committed
251
252
253
254
255
	for (i=0; i<(len<<1); i++) {
	  txdata[i] = ((int16_t*)dummy_tx_b)[i]<<openair0_cfg[0].iq_txshift;
	}
      }
      
256
     // if S-subframe switch to RX in second subframe
knopp's avatar
knopp committed
257
      /*
258
     if (subframe_select(&phy_vars_eNB->frame_parms,subframe) == SF_S) {
259
       for (i=0; i<len; i++) {
260
	 phy_vars_eNB->common_vars.txdata[0][aa][tx_offset++] = 0x00010001;
261
262
       }
     }
knopp's avatar
knopp committed
263
      */
264
     if ((((phy_vars_eNB->frame_parms.tdd_config==0) ||
Sandeep Kumar's avatar
Sandeep Kumar committed
265
266
267
268
	   (phy_vars_eNB->frame_parms.tdd_config==1) ||
	   (phy_vars_eNB->frame_parms.tdd_config==2) ||
	   (phy_vars_eNB->frame_parms.tdd_config==6)) && 
	   (subframe==0)) || (subframe==5)) {
269
270
271
       // turn on tx switch N_TA_offset before
       //LOG_D(HW,"subframe %d, time to switch to tx (N_TA_offset %d, slot_offset %d) \n",subframe,phy_vars_eNB->N_TA_offset,slot_offset);
       for (i=0; i<phy_vars_eNB->N_TA_offset; i++) {
Sandeep Kumar's avatar
Sandeep Kumar committed
272
273
274
         tx_offset = (int)slot_offset+time_offset[aa]+i-phy_vars_eNB->N_TA_offset/2;
         if (tx_offset<0)
           tx_offset += LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
275
	 
Sandeep Kumar's avatar
Sandeep Kumar committed
276
277
         if (tx_offset>=(LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti))
           tx_offset -= LTE_NUMBER_OF_SUBFRAMES_PER_FRAME*phy_vars_eNB->frame_parms.samples_per_tti;
278
	 
Sandeep Kumar's avatar
Sandeep Kumar committed
279
         phy_vars_eNB->common_vars.txdata[0][aa][tx_offset] = 0x00000000;
280
281
282
283
284
285
       }
     }
    }
  }
}

Sandeep Kumar's avatar
Sandeep Kumar committed
286

287
/*!
knopp's avatar
knopp committed
288
 * \brief The RX UE-specific and TX thread of eNB.
289
290
291
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
292
293
static void* eNB_thread_rxtx( void* param ) {

knopp's avatar
knopp committed
294
  static int eNB_thread_rxtx_status;
295

knopp's avatar
knopp committed
296
  eNB_rxtx_proc_t *proc = (eNB_rxtx_proc_t*)param;
297
298
  FILE  *tx_time_file = NULL;
  char tx_time_name[101];
knopp's avatar
knopp committed
299
  void *txp[PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx]; 
300
301

  if (opp_enabled == 1) {
302
    snprintf(tx_time_name, 100,"/tmp/%s_tx_time_thread_sf", "eNB");
303
304
305
    tx_time_file = fopen(tx_time_name,"w");
  }
  // set default return value
knopp's avatar
knopp committed
306
  eNB_thread_rxtx_status = 0;
307
308
309

  MSC_START_USE();

310
#ifdef DEADLINE_SCHEDULER
311
  struct sched_attr attr;
312

313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
  unsigned int flags = 0;
  uint64_t runtime  = 850000 ;  
  uint64_t deadline = 1   *  1000000 ; // each tx thread will finish within 1ms
  uint64_t period   = 1   * 10000000; // each tx thread has a period of 10ms from the starting point

  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy   = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB tx thread: sched_setattr failed\n");
knopp's avatar
knopp committed
330
    return &eNB_thread_rxtx_status;
331
332
  }

knopp's avatar
knopp committed
333
  LOG_I( HW, "[SCHED] eNB RXn-TXnp4 deadline thread (TID %ld) started on CPU %d\n", gettid(), sched_getcpu() );
334
335
336
337
338
339

#else //LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;
Raymond Knopp's avatar
Raymond Knopp committed
340
341
342
343
  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD threads */
  /* CPU 1 is reserved for all TX threads */
  /* Enable CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);

#ifdef CPU_AFFINITY
  if (get_nprocs() > 2)
  {
    for (j = 1; j < get_nprocs(); j++)
        CPU_SET(j, &cpuset);
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
    if (s != 0)
    {
      perror( "pthread_setaffinity_np");
      exit_fun("Error setting processor affinity");
    }
  }
#endif //CPU_AFFINITY

  /* Check the actual affinity mask assigned to the thread */
  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
367
  if (s != 0) {
368
369
370
371
372
    perror( "pthread_getaffinity_np");
    exit_fun("Error getting processor affinity ");
  }
  memset(cpu_affinity,0,sizeof(cpu_affinity));
  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
373
374
375
376
377
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
378

Sandeep Kumar's avatar
Sandeep Kumar committed
379
  memset(&sparam, 0, sizeof(sparam));
380
381
382
383
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-1;
  policy = SCHED_FIFO ; 
  
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
384
385
386
387
388
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");
  }
  
389
  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
390
391
392
393
  if (s != 0) {
    perror("pthread_getschedparam : ");
    exit_fun("Error getting thread priority");
  }
394

Sandeep Kumar's avatar
Sandeep Kumar committed
395
  LOG_I(HW, "[SCHED][eNB] TX thread started on CPU %d TID %ld, sched_policy = %s , priority = %d, CPU Affinity=%s \n",sched_getcpu(),gettid(),
396
397
398
399
400
401
402
403
404
405
406
                   (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
                   (policy == SCHED_RR)    ? "SCHED_RR" :
                   (policy == SCHED_OTHER) ? "SCHED_OTHER" :
                   "???",
                   sparam.sched_priority, cpu_affinity );

#endif //LOW_LATENCY

  mlockall(MCL_CURRENT | MCL_FUTURE);

  while (!oai_exit) {
knopp's avatar
knopp committed
407
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
408

Raymond Knopp's avatar
Raymond Knopp committed
409
    if (pthread_mutex_timedlock(&proc->mutex_rxtx,&wait) != 0) {
knopp's avatar
knopp committed
410
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB RXn-TXnp4\n");
411
412
413
414
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
415
    while (proc->instance_cnt_rxtx < 0) {
416
      // most of the time the thread is waiting here
knopp's avatar
knopp committed
417
418
      // proc->instance_cnt_rxtx is -1
      pthread_cond_wait( &proc->cond_rxtx, &proc->mutex_rxtx ); // this unlocks mutex_rxtx while waiting and then locks it again
419
420
    }

knopp's avatar
knopp committed
421
    if (pthread_mutex_unlock(&proc->mutex_rxtx) != 0) {
422
      LOG_E(PHY,"[SCHED][eNB] error unlocking mutex for eNB TX\n");
423
424
425
426
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
427
428
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 1 );
    start_meas( &softmodem_stats_rxtx_sf );
429
  
430
431
    if (oai_exit) break;

knopp's avatar
knopp committed
432
433
434
435
436
437
438
439
440
    // UE-specific RX processing for subframe n
    if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RRU_IF4) {
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_UESPEC, 1 );
      // this is the ue-specific processing for the subframe and can be multi-threaded later
      phy_procedures_eNB_uespec_RX(PHY_vars_eNB_g[0][proc->CC_id], proc, 0, no_relay );
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_UESPEC, 0 );
    }

    // TX processing for subframe n+4
441
442
443
444
    if (((PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.frame_type == TDD) &&
         ((subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->frame_parms,proc->subframe_tx) == SF_DL) ||
          (subframe_select(&PHY_vars_eNB_g[0][proc->CC_id]->frame_parms,proc->subframe_tx) == SF_S))) ||
        (PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.frame_type == FDD)) {
445
446
447
      /* run PHY TX procedures the one after the other for all CCs to avoid race conditions
       * (may be relaxed in the future for performance reasons)
       */
knopp's avatar
knopp committed
448
      
Raymond Knopp's avatar
Raymond Knopp committed
449
      if (pthread_mutex_timedlock(&sync_phy_proc.mutex_phy_proc_tx,&wait) != 0) {
450
        LOG_E(PHY, "[SCHED][eNB] error locking PHY proc mutex for eNB TX\n");
451
452
453
        exit_fun("nothing to add");
        break;
      }
Sandeep Kumar's avatar
Sandeep Kumar committed
454
      
knopp's avatar
knopp committed
455
      // wait for our turn or oai_exit
456
457
458
      while (sync_phy_proc.phy_proc_CC_id != proc->CC_id && !oai_exit) {
        pthread_cond_wait(&sync_phy_proc.cond_phy_proc_tx,
                          &sync_phy_proc.mutex_phy_proc_tx);
459
460
      }

461
462
      if (pthread_mutex_unlock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
        LOG_E(PHY, "[SCHED][eNB] error unlocking PHY proc mutex for eNB TX\n");
463
464
465
        exit_fun("nothing to add");
      }

knopp's avatar
knopp committed
466
467
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_TX_ENB, proc->frame_tx );
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_TX_ENB, proc->subframe_tx );
Sandeep Kumar's avatar
Sandeep Kumar committed
468
469
470
      
      if (oai_exit) break;
      
471
      if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RRU_IF4) { 
Sandeep Kumar's avatar
Sandeep Kumar committed
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
        phy_procedures_eNB_TX(PHY_vars_eNB_g[0][proc->CC_id], proc, 0, no_relay, NULL );

        /* we're done, let the next one proceed */
        if (pthread_mutex_lock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
          LOG_E(PHY, "[SCHED][eNB] error locking PHY proc mutex for eNB TX proc\n");
          exit_fun("nothing to add");
          break;
        }	
        sync_phy_proc.phy_proc_CC_id++;
        sync_phy_proc.phy_proc_CC_id %= MAX_NUM_CCs;
        pthread_cond_broadcast(&sync_phy_proc.cond_phy_proc_tx);
        if (pthread_mutex_unlock(&sync_phy_proc.mutex_phy_proc_tx) != 0) {
          LOG_E(PHY, "[SCHED][eNB] error unlocking PHY proc mutex for eNB TX proc\n");
          exit_fun("nothing to add");
          break;
487
	}
488
489
      } else {

490
491
492
        /// **** recv_IF4 of txdataF from RCC **** ///     
        
        VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF4, 1 );   
Sandeep Kumar's avatar
Sandeep Kumar committed
493
        //recv_IF4(eNB, frame, subframe, packet_type, symbol_number);        
494
        VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF4, 0 );   
495
        
496
497
498
      }
    }

Sandeep Kumar's avatar
Sandeep Kumar committed
499
    // eNodeB_3GPP and RRU create txdata and write to RF device
500
    if (PHY_vars_eNB_g[0][proc->CC_id]->node_function != NGFI_RCC_IF4) {
knopp's avatar
knopp committed
501
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_SFGEN , 1 );
502
      do_OFDM_mod_rt( proc->subframe_tx, PHY_vars_eNB_g[0][proc->CC_id] );
knopp's avatar
knopp committed
503
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_SFGEN , 0 );
504
  
Sandeep Kumar's avatar
Sandeep Kumar committed
505
506
507
508
509
510
511
512
513
514
515
516
517
      /*
        short *txdata = (short*)&PHY_vars_eNB_g[0][proc->CC_id]->common_vars.txdata[0][0][proc->subframe_tx*PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.samples_per_tti];
        int i;
        for (i=0;i<PHY_vars_eNB_g[0][proc->CC_id]->frame_parms.samples_per_tti*2;i+=8) {
        txdata[i] = 2047;
        txdata[i+1] = 0;
        txdata[i+2] = 0;
        txdata[i+3] = 2047;
        txdata[i+4] = -2047;
        txdata[i+5] = 0;
        txdata[i+6] = 0;
        txdata[i+7] = -2047;      }
      */
knopp's avatar
knopp committed
518

Sandeep Kumar's avatar
Sandeep Kumar committed
519
      // Transmit TX buffer based on timestamp from RX  
knopp's avatar
knopp committed
520
521
522
523
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 1 );
      // prepare tx buffer pointers
      int i;
      for (i=0; i<PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx; i++)
Sandeep Kumar's avatar
Sandeep Kumar committed
524
525
        txp[i] = (void*)&PHY_vars_eNB_g[0][0]->common_vars.txdata[0][i][proc->subframe_tx*PHY_vars_eNB_g[0][0]->frame_parms.samples_per_tti];
    
knopp's avatar
knopp committed
526
      // if symb_written < spp ==> error 
Sandeep Kumar's avatar
Sandeep Kumar committed
527
528
529
530
531
532
533
      PHY_vars_eNB_g[0][proc->CC_id]->rfdevice.trx_write_func(&PHY_vars_eNB_g[0][proc->CC_id]->rfdevice,
            (proc->timestamp_tx-openair0_cfg[0].tx_sample_advance),
            txp,
            PHY_vars_eNB_g[0][0]->frame_parms.samples_per_tti,
            PHY_vars_eNB_g[0][0]->frame_parms.nb_antennas_tx,
            1);
	      
knopp's avatar
knopp committed
534
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 0 );
Sandeep Kumar's avatar
Sandeep Kumar committed
535
 
knopp's avatar
knopp committed
536
537
      VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TST, (proc->timestamp_tx-openair0_cfg[0].tx_sample_advance)&0xffffffff );

Sandeep Kumar's avatar
Sandeep Kumar committed
538
539
    } else { 
	 
540
541
542
      /// **** send_IF4 of txdataF to RRU **** /// 
      
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_SEND_IF4, 1 );   
Sandeep Kumar's avatar
Sandeep Kumar committed
543
      send_IF4(PHY_vars_eNB_g[0][proc->CC_id], proc->frame_tx, proc->subframe_tx, IF4_PDLFFT);
544
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_SEND_IF4, 0 );
545

546
547
    }

knopp's avatar
knopp committed
548
    if (pthread_mutex_lock(&proc->mutex_rxtx) != 0) {
549
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB TX proc\n");
550
551
552
553
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
554
    proc->instance_cnt_rxtx--;
555

knopp's avatar
knopp committed
556
    if (pthread_mutex_unlock(&proc->mutex_rxtx) != 0) {
557
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB TX proc\n");
558
559
560
561
      exit_fun("nothing to add");
      break;
    }

knopp's avatar
knopp committed
562
    stop_meas( &softmodem_stats_rxtx_sf );
Sandeep Kumar's avatar
Sandeep Kumar committed
563

564
#ifdef DEADLINE_SCHEDULER
565
    if (opp_enabled){
knopp's avatar
knopp committed
566
567
      if(softmodem_stats_rxtx_sf.diff_now/(cpuf) > attr.sched_runtime){
	VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_RUNTIME_TX_ENB, (softmodem_stats_rxtx_sf.diff_now/cpuf - attr.sched_runtime)/1000000.0);
568
569
570
    }
#endif 

Sandeep Kumar's avatar
Sandeep Kumar committed
571
    print_meas_now(&softmodem_stats_rxtx_sf,"eNB_TX_SF",tx_time_file);
572
573
  }

knopp's avatar
knopp committed
574
  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
575

Raymond Knopp's avatar
Raymond Knopp committed
576
577
578

  printf( "Exiting eNB thread RXn_TXnp4\n");

579

knopp's avatar
knopp committed
580
581
  eNB_thread_rxtx_status = 0;
  return &eNB_thread_rxtx_status;
582
583
}

584
#if defined(ENABLE_ITTI)
Sandeep Kumar's avatar
Sandeep Kumar committed
585
586
587
/* Wait for eNB application initialization to be complete (eNB registration to MME) */
static void wait_system_ready (char *message, volatile int *start_flag) {
  
588
  static char *indicator[] = {".    ", "..   ", "...  ", ".... ", ".....",
Sandeep Kumar's avatar
Sandeep Kumar committed
589
			      " ....", "  ...", "   ..", "    .", "     "};
590
591
592
593
594
595
596
  int i = 0;
  
  while ((!oai_exit) && (*start_flag == 0)) {
    LOG_N(EMU, message, indicator[i]);
    fflush(stdout);
    i = (i + 1) % (sizeof(indicator) / sizeof(indicator[0]));
    usleep(200000);
597
  }
598
599
  
  LOG_D(EMU,"\n");
600
601
}
#endif
602

Sandeep Kumar's avatar
Sandeep Kumar committed
603

604
/*!
knopp's avatar
knopp committed
605
 * \brief The RX common thread of eNB.
606
607
608
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
609
610
static void* eNB_thread_rx_common( void* param ) {
  
611
  static int eNB_thread_rx_status;
612
613

  eNB_proc_t *proc = (eNB_proc_t*)param;
614
615
  PHY_VARS_eNB *eNB = PHY_vars_eNB_g[0][proc->CC_id];
  LTE_DL_FRAME_PARMS *fp = &eNB->frame_parms;
616
617
618
619

  FILE  *rx_time_file = NULL;
  char rx_time_name[101];
  //int i;
Raymond Knopp's avatar
Raymond Knopp committed
620
621
622
623
  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;
624

Sandeep Kumar's avatar
Sandeep Kumar committed
625
  if (opp_enabled == 1) {
626
    snprintf(rx_time_name, 100,"/tmp/%s_rx_time_thread_sf", "eNB");
627
628
629
    rx_time_file = fopen(rx_time_name,"w");
  }
  // set default return value
630
  eNB_thread_rx_status = 0;
631
632
633

  MSC_START_USE();

634
#ifdef DEADLINE_SCHEDULER
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
  struct sched_attr attr;
  unsigned int flags = 0;
  uint64_t runtime  = 870000 ;
  uint64_t deadline = 1   *  1000000;
  uint64_t period   = 1   * 10000000; // each rx thread has a period of 10ms from the starting point
 
  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB RX sched_setattr failed\n");
653
    return &eNB_thread_rx_status;
654
655
  }

656
  LOG_I( HW, "[SCHED] eNB RX deadline thread (TID %ld) started on CPU %d\n", gettid(), sched_getcpu() );
657
658
659
660
661
662
663
664
665
666
667
668
669
#else // LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD */
  /* CPU 1 is reserved for all TX threads */
  /* CPU 2..MAX_CPUS is reserved for all RX threads */
  /* Set CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);
#ifdef CPU_AFFINITY
Sandeep Kumar's avatar
Sandeep Kumar committed
670
  if (get_nprocs() >2) {
671
    for (j = 1; j < get_nprocs(); j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
672
      CPU_SET(j, &cpuset);
673
674
  
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
675
    if (s != 0) {
676
677
678
679
680
681
682
683
      perror( "pthread_setaffinity_np");  
      exit_fun (" Error setting processor affinity :");
    }
  }
#endif //CPU_AFFINITY
  /* Check the actual affinity mask assigned to the thread */

  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
684
685
686
  if (s != 0) {
    perror ("pthread_getaffinity_np");
    exit_fun (" Error getting processor affinity :");
687
688
689
690
  }
  memset(cpu_affinity,0, sizeof(cpu_affinity));

  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
691
692
693
694
695
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
696
697

  memset(&sparam, 0 , sizeof (sparam)); 
knopp's avatar
knopp committed
698
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO);
699
700
701

  policy = SCHED_FIFO ; 
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
702
703
704
705
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");     
  }
706
707
708
709

  memset(&sparam, 0 , sizeof (sparam));

  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
710
711
712
713
  if (s != 0) {
    perror("pthread_getschedparam");
    exit_fun("Error getting thread priority");
  }
714

Sandeep Kumar's avatar
Sandeep Kumar committed
715
  LOG_I(HW, "[SCHED][eNB] RX thread started on CPU %d TID %ld, sched_policy = %s, priority = %d, CPU Affinity = %s\n", sched_getcpu(),gettid(),
716
717
718
719
720
721
722
	 (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
	 (policy == SCHED_RR)    ? "SCHED_RR" :
	 (policy == SCHED_OTHER) ? "SCHED_OTHER" :
	 "???",
	 sparam.sched_priority, cpu_affinity);
  
  
723
#endif // DEADLINE_SCHEDULER
724
725


Sandeep Kumar's avatar
Sandeep Kumar committed
726
  mlockall(MCL_CURRENT | MCL_FUTURE);
727

Sandeep Kumar's avatar
Sandeep Kumar committed
728
729
730
  // wait for top-level synchronization and do one acquisition to get timestamp for setting frame/subframe of TX and RX threads
  printf( "waiting for sync (eNB_thread_rx_common)\n");
  pthread_mutex_lock( &sync_mutex );
731

Sandeep Kumar's avatar
Sandeep Kumar committed
732
733
  while (sync_var<0)
    pthread_cond_wait( &sync_cond, &sync_mutex );
734
 
Sandeep Kumar's avatar
Sandeep Kumar committed
735
  pthread_mutex_unlock(&sync_mutex);
736
 
Sandeep Kumar's avatar
Sandeep Kumar committed
737
  printf( "got sync (eNB_thread)\n" );
738
739
740
741
 
#if defined(ENABLE_ITTI)
  wait_system_ready ("Waiting for eNB application to be ready %s\r", &start_eNB);
#endif 
742
743
  
  // Start RF device for this CC
744
  if (eNB->node_function != NGFI_RCC_IF4) {
745
746
747
748
749
    if (eNB->rfdevice.trx_start_func(&eNB->rfdevice) != 0 ) 
      LOG_E(HW,"Could not start the RF device\n");
  }
    
  // Start IF device for this CC
750
  if (eNB->node_function != eNodeB_3GPP) {
751
752
753
754
    if (eNB->ifdevice.trx_start_func(&eNB->ifdevice) != 0 ) 
      LOG_E(HW,"Could not start the IF device\n");
  }

Sandeep Kumar's avatar
Sandeep Kumar committed
755
756
757
758
759
  // This is a forever while loop, it loops over subframes which are scheduled by incoming samples from HW devices
  while (!oai_exit) {
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RX, 0 );
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 0 );
    start_meas( &softmodem_stats_rx_sf );
760
   
Sandeep Kumar's avatar
Sandeep Kumar committed
761
    if (oai_exit) break;
762
   
Sandeep Kumar's avatar
Sandeep Kumar committed
763
764
765
766
767
768
769
770
    if ((((fp->frame_type == TDD )&&(subframe_select(fp,proc->subframe_rx)==SF_UL)) ||
	 (fp->frame_type == FDD))) {
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 1 );
      // this spawns the prach inside and updates the frame and subframe counters
      phy_procedures_eNB_common_RX(eNB, 0);
      
      VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_ENB_RX_COMMON, 0 );
    }
knopp's avatar
knopp committed
771

Sandeep Kumar's avatar
Sandeep Kumar committed
772
773
774
775
776
    // choose even or odd thread for RXn-TXnp4 processing 
    eNB_rxtx_proc_t *proc_rxtx = &proc->proc_rxtx[proc->subframe_rx&1];

    // wake up TX for subframe n+4
    // lock the TX mutex and make sure the thread is ready
Raymond Knopp's avatar
Raymond Knopp committed
777
    if (pthread_mutex_timedlock(&proc_rxtx->mutex_rxtx,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
      LOG_E( PHY, "[eNB] ERROR pthread_mutex_lock for eNB TX thread %d (IC %d)\n", proc_rxtx->instance_cnt_rxtx );
      exit_fun( "error locking mutex_rxtx" );
      break;
    }
    int cnt_rxtx = ++proc_rxtx->instance_cnt_rxtx;
    // We have just received and processed the common part of a subframe, say n. 
    // TS_rx is the last received timestamp (start of 1st slot), TS_tx is the desired 
    // transmitted timestamp of the next TX slot (first).
    // The last (TS_rx mod samples_pexr_frame) was n*samples_per_tti, 
    // we want to generate subframe (n+3), so TS_tx = TX_rx+3*samples_per_tti,
    // and proc->subframe_tx = proc->subframe_rx+3
    proc_rxtx->timestamp_tx = proc->timestamp_rx + (4*fp->samples_per_tti);
    proc_rxtx->frame_rx     = proc->frame_rx;
    proc_rxtx->subframe_rx  = proc->subframe_rx;
    proc_rxtx->frame_tx     = (proc_rxtx->subframe_rx > 5) ? (proc_rxtx->frame_rx+1)&1023 : proc_rxtx->frame_rx;
    proc_rxtx->subframe_tx  = (proc_rxtx->subframe_rx + 4)%10;
knopp's avatar
knopp committed
794
   
Sandeep Kumar's avatar
Sandeep Kumar committed
795
    pthread_mutex_unlock( &proc_rxtx->mutex_rxtx );
knopp's avatar
knopp committed
796
   
Sandeep Kumar's avatar
Sandeep Kumar committed
797
798
799
800
801
802
803
804
805
806
807
808
    if (cnt_rxtx == 0) {
      // the thread was presumably waiting where it should and can now be woken up
      if (pthread_cond_signal(&proc_rxtx->cond_rxtx) != 0) {
        LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB RXn-TXnp4 thread\n");
        exit_fun( "ERROR pthread_cond_signal" );
        break;
      }
    } else {
      LOG_W( PHY,"[eNB] Frame %d, eNB RXn-TXnp4 thread busy!! (cnt_rxtx %i)\n", proc_rxtx->frame_tx, cnt_rxtx );
      exit_fun( "TX thread busy" );
      break;
    }       
809
   
Sandeep Kumar's avatar
Sandeep Kumar committed
810

811
    stop_meas( &softmodem_stats_rxtx_sf );
812
#ifdef DEADLINE_SCHEDULER
Sandeep Kumar's avatar
Sandeep Kumar committed
813
814
    if (opp_enabled){
      if(softmodem_stats_rxtx_sf.diff_now/(cpuf) > attr.sched_runtime){
815
	VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_RUNTIME_RXTX_ENB, (softmodem_stats_rxtx_sf.diff_now/cpuf - attr.sched_runtime)/1000000.0);
Sandeep Kumar's avatar
Sandeep Kumar committed
816
817
      }
    }
818
#endif // DEADLINE_SCHEDULER  
819
820
821
    print_meas_now(&softmodem_stats_rx_sf,"eNB_RX_SF", rx_time_file);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
    
Sandeep Kumar's avatar
Sandeep Kumar committed
822
823
824
    print_meas_now(&softmodem_stats_rx_sf,"eNB_RX_SF", rx_time_file);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 );
  }
knopp's avatar
knopp committed
825
 
Raymond Knopp's avatar
Raymond Knopp committed
826
827
828

  printf( "Exiting eNB thread rx_common\n");

knopp's avatar
knopp committed
829
 
Sandeep Kumar's avatar
Sandeep Kumar committed
830
831
  eNB_thread_rx_status = 0;
  return &eNB_thread_rx_status;
832
833
834
835
836
837
838
839
}


/*!
 * \brief The prach receive thread of eNB.
 * \param param is a \ref eNB_proc_t structure which contains the info what to process.
 * \returns a pointer to an int. The storage is not on the heap and must not be freed.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
840
static void* eNB_thread_prach( void* param ) {
841
842
  static int eNB_thread_prach_status;

Raymond Knopp's avatar
Raymond Knopp committed
843
844
 

845
846
  eNB_proc_t *proc = (eNB_proc_t*)param;
  PHY_VARS_eNB *eNB= PHY_vars_eNB_g[0][proc->CC_id];
Raymond Knopp's avatar
Raymond Knopp committed
847
848
849
850
851
852

  struct timespec wait;

  wait.tv_sec=0;
  wait.tv_nsec=5000000L;

853
854
855
856
857
  // set default return value
  eNB_thread_prach_status = 0;

  MSC_START_USE();
    
858
#ifdef DEADLINE_SCHEDULER
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
  struct sched_attr attr;
  unsigned int flags = 0;
  uint64_t runtime  = 870000 ;
  uint64_t deadline = 1   *  1000000;
  uint64_t period   = 1   * 10000000; // each prach thread has a period of 10ms from the starting point
 
  attr.size = sizeof(attr);
  attr.sched_flags = 0;
  attr.sched_nice = 0;
  attr.sched_priority = 0;

  attr.sched_policy = SCHED_DEADLINE;
  attr.sched_runtime  = runtime;
  attr.sched_deadline = deadline;
  attr.sched_period   = period; 

  if (sched_setattr(0, &attr, flags) < 0 ) {
    perror("[SCHED] eNB PRACH sched_setattr failed\n");
    return &eNB_thread_prach_status;
  }

  LOG_I( HW, "[SCHED] eNB PRACH deadline thread (TID %ld) started on CPU %d\n", 0, gettid(), sched_getcpu() );
#else // LOW_LATENCY
  int policy, s, j;
  struct sched_param sparam;
  char cpu_affinity[1024];
  cpu_set_t cpuset;

  /* Set affinity mask to include CPUs 1 to MAX_CPUS */
  /* CPU 0 is reserved for UHD */
  /* CPU 1 is reserved for all TX threads */
  /* CPU 2..MAX_CPUS is reserved for all RX threads */
  /* Set CPU Affinity only if number of CPUs >2 */
  CPU_ZERO(&cpuset);
#ifdef CPU_AFFINITY
Sandeep Kumar's avatar
Sandeep Kumar committed
894
  if (get_nprocs() >2) {
895
    for (j = 1; j < get_nprocs(); j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
896
      CPU_SET(j, &cpuset);
897
898
  
    s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
899
    if (s != 0) {
900
901
902
903
904
905
      perror( "pthread_setaffinity_np");  
      exit_fun (" Error setting processor affinity :");
    }
  }
#endif //CPU_AFFINITY

Sandeep Kumar's avatar
Sandeep Kumar committed
906
  /* Check the actual affinity mask assigned to the thread */
907
  s = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
Sandeep Kumar's avatar
Sandeep Kumar committed
908
909
910
  if (s != 0) {
    perror ("pthread_getaffinity_np");
    exit_fun (" Error getting processor affinity :");
911
912
913
914
  }
  memset(cpu_affinity,0, sizeof(cpu_affinity));

  for (j = 0; j < CPU_SETSIZE; j++)
Sandeep Kumar's avatar
Sandeep Kumar committed
915
916
917
918
919
    if (CPU_ISSET(j, &cpuset)) {  
      char temp[1024];
      sprintf (temp, " CPU_%d", j);
      strcat(cpu_affinity, temp);
    }
920
921

  memset(&sparam, 0 , sizeof (sparam)); 
knopp's avatar
knopp committed
922
  sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-2;
923
924
925

  policy = SCHED_FIFO ; 
  s = pthread_setschedparam(pthread_self(), policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
926
927
928
929
  if (s != 0) {
    perror("pthread_setschedparam : ");
    exit_fun("Error setting thread priority");
  }
930
931
932
933

  memset(&sparam, 0 , sizeof (sparam));

  s = pthread_getschedparam(pthread_self(), &policy, &sparam);
Sandeep Kumar's avatar
Sandeep Kumar committed
934
935
936
937
  if (s != 0) {
    perror("pthread_getschedparam");
    exit_fun("Error getting thread priority");
  }
938

Raymond Knopp's avatar
Raymond Knopp committed
939
  LOG_I(HW, "[SCHED][eNB] PRACH thread started on CPU %d TID %ld, IC %d, sched_policy = %s, priority = %d, CPU Affinity = %s\n", sched_getcpu(),gettid(),proc->instance_cnt_prach,
940
941
942
943
944
945
	 (policy == SCHED_FIFO)  ? "SCHED_FIFO" :
	 (policy == SCHED_RR)    ? "SCHED_RR" :
	 (policy == SCHED_OTHER) ? "SCHED_OTHER" :
	 "???",
	 sparam.sched_priority, cpu_affinity);
  
946
#endif // DEADLINE_SCHEDULER
947

Sandeep Kumar's avatar
Sandeep Kumar committed
948
  mlockall(MCL_CURRENT | MCL_FUTURE);
949

Sandeep Kumar's avatar
Sandeep Kumar committed
950
951
952
  while (!oai_exit) {
    
    if (oai_exit) break;
953
        
Raymond Knopp's avatar
Raymond Knopp committed
954
    if (pthread_mutex_timedlock(&proc->mutex_prach,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB PRACH\n");
      exit_fun( "error locking mutex" );
      break;
    }

    while (proc->instance_cnt_prach < 0) {
      // most of the time the thread is waiting here
      // proc->instance_cnt_prach is -1
      pthread_cond_wait( &proc->cond_prach, &proc->mutex_prach ); // this unlocks mutex_rxtx while waiting and then locks it again
    }

    if (pthread_mutex_unlock(&proc->mutex_prach) != 0) {
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB PRACH\n");
      exit_fun( "error unlocking mutex" );
      break;
    }
971
   
Sandeep Kumar's avatar
Sandeep Kumar committed
972
973
974
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_PRACH_RX,1);
    prach_procedures(eNB,0);
    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_ENB_PRACH_RX,0);
975
    
Raymond Knopp's avatar
Raymond Knopp committed
976
    if (pthread_mutex_timedlock(&proc->mutex_prach,&wait) != 0) {
Sandeep Kumar's avatar
Sandeep Kumar committed
977
978
979
980
      LOG_E( PHY, "[SCHED][eNB] error locking mutex for eNB PRACH proc %d\n");
      exit_fun( "error locking mutex" );
      break;
    }
981
   
Sandeep Kumar's avatar
Sandeep Kumar committed
982
983
984
985
986
987
988
989
    proc->instance_cnt_prach--;
    
    if (pthread_mutex_unlock(&proc->mutex_prach) != 0) {
      LOG_E( PHY, "[SCHED][eNB] error unlocking mutex for eNB RX proc %d\n");
      exit_fun( "error unlocking mutex" );
      break;
    } 
  }
990

Raymond Knopp's avatar
Raymond Knopp committed
991

992
  printf( "Exiting eNB thread PRACH\n");
Raymond Knopp's avatar
Raymond Knopp committed
993

994

995
996
  eNB_thread_prach_status = 0;
  return &eNB_thread_prach_status;
997
998
999
}


Sandeep Kumar's avatar
Sandeep Kumar committed
1000
1001
void init_eNB_proc(void) {
  
1002
1003
  int i;
  int CC_id;
1004
1005
  PHY_VARS_eNB *eNB;
  eNB_proc_t *proc;
knopp's avatar
knopp committed
1006
  eNB_rxtx_proc_t *proc_rxtx;
1007
1008

  for (CC_id=0; CC_id<MAX_NUM_CCs; CC_id++) {
1009
1010
    eNB = PHY_vars_eNB_g[0][CC_id];

1011
    proc = &eNB->proc;
knopp's avatar
knopp committed
1012
    proc_rxtx = proc->proc_rxtx;
1013
#ifndef DEADLINE_SCHEDULER 
1014
1015
1016
    /*  
	pthread_attr_init( &attr_eNB_proc_tx[CC_id][i] );
	if (pthread_attr_setstacksize( &attr_eNB_proc_tx[CC_id][i], 64 *PTHREAD_STACK_MIN ) != 0)
1017
        perror("[ENB_PROC_TX] setting thread stack size failed\n");
1018
1019
1020
	
	pthread_attr_init( &attr_eNB_proc_rx[CC_id][i] );
	if (pthread_attr_setstacksize( &attr_eNB_proc_rx[CC_id][i], 64 * PTHREAD_STACK_MIN ) != 0)
1021
        perror("[ENB_PROC_RX] setting thread stack size failed\n");
1022
1023
    */
    // set the kernel scheduling policy and priority
knopp's avatar
knopp committed
1024
1025
1026
1027
1028
1029
    proc_rxtx[0].sched_param_rxtx.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc_rxtx[0].attr_rxtx, &proc_rxtx[0].sched_param_rxtx);
    pthread_attr_setschedpolicy (&proc_rxtx[0].attr_rxtx, SCHED_FIFO);
    proc_rxtx[1].sched_param_rxtx.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc_rxtx[1].attr_rxtx, &proc_rxtx[1].sched_param_rxtx);
    pthread_attr_setschedpolicy (&proc_rxtx[1].attr_rxtx, SCHED_FIFO);
1030
    
knopp's avatar
knopp committed
1031
    proc->sched_param_rx.sched_priority = sched_get_priority_max(SCHED_FIFO); //OPENAIR_THREAD_PRIORITY;
1032
1033
1034
1035
1036
1037
1038
1039
    pthread_attr_setschedparam  (&proc->attr_rx, &proc->sched_param_rx);
    pthread_attr_setschedpolicy (&proc->attr_rx, SCHED_FIFO);
    
    proc->sched_param_prach.sched_priority = sched_get_priority_max(SCHED_FIFO)-1; //OPENAIR_THREAD_PRIORITY;
    pthread_attr_setschedparam  (&proc->attr_prach, &proc->sched_param_prach);
    pthread_attr_setschedpolicy (&proc->attr_prach, SCHED_FIFO);
    
    printf("Setting OS scheduler to SCHED_FIFO for eNB [cc%d][thread%d] \n",CC_id, i);
1040
#endif
knopp's avatar
knopp committed
1041
1042
    proc_rxtx[0].instance_cnt_rxtx = -1;
    proc_rxtx[1].instance_cnt_rxtx = -1;
1043
1044
    proc->instance_cnt_prach = -1;
    proc->CC_id = CC_id;
1045
1046
1047

    proc->first_rx=1;

knopp's avatar
knopp committed
1048
1049
1050
1051
1052
    pthread_mutex_init( &proc_rxtx[0].mutex_rxtx, NULL);
    pthread_mutex_init( &proc_rxtx[1].mutex_rxtx, NULL);
    pthread_mutex_init( &proc->mutex_prach, NULL);
    pthread_cond_init( &proc_rxtx[0].cond_rxtx, NULL);
    pthread_cond_init( &proc_rxtx[1].cond_rxtx, NULL);
1053
    pthread_cond_init( &proc->cond_prach, NULL);
1054
#ifndef DEADLINE_SCHEDULER
knopp's avatar
knopp committed
1055
1056
1057
    pthread_create( &proc_rxtx[0].pthread_rxtx, &proc_rxtx[0].attr_rxtx, eNB_thread_rxtx, &proc_rxtx[0] );
    pthread_create( &proc_rxtx[1].pthread_rxtx, &proc_rxtx[1].attr_rxtx, eNB_thread_rxtx, &proc_rxtx[1] );
    pthread_create( &proc->pthread_rx, &proc->attr_rx, eNB_thread_rx_common, &eNB->proc );
1058
    pthread_create( &proc->pthread_prach, &proc->attr_prach, eNB_thread_prach, &eNB->proc );
1059
#else 
knopp's avatar
knopp committed
1060
1061
1062
    pthread_create( &proc_rxtx[0].pthread_rxtx, NULL, eNB_thread_rxtx, &eNB->proc_rxtx[0] );
    pthread_create( &proc_rxtx[1].pthread_rxtx, NULL, eNB_thread_rxtx, &eNB->proc_rxtx[1] );
    pthread_create( &proc->pthread_rx, NULL, eNB_thread_rx_common, &eNB->proc );
1063
    pthread_create( &proc->pthread_prach, NULL, eNB_thread_prach, &eNB->proc );
1064
#endif
1065
    char name[16];
knopp's avatar
knopp committed
1066
1067
1068
1069
    snprintf( name, sizeof(name), "RXTX0 %d", i );
    pthread_setname_np( proc_rxtx[0].pthread_rxtx, name );
    snprintf( name, sizeof(name), "RXTX1 %d", i );
    pthread_setname_np( proc_rxtx[1].pthread_rxtx, name );
1070
1071
    snprintf( name, sizeof(name), "RX %d", i );
    pthread_setname_np( proc->pthread_rx, name );
1072
  }
1073
  
1074
  /* setup PHY proc TX sync mechanism */
1075
1076
1077
  pthread_mutex_init(&sync_phy_proc.mutex_phy_proc_tx, NULL);
  pthread_cond_init(&sync_phy_proc.cond_phy_proc_tx, NULL);
  sync_phy_proc.phy_proc_CC_id = 0;
1078
1079
}

1080

1081
1082
1083
/*!
 * \brief Terminate eNB TX and RX threads.
 */
Sandeep Kumar's avatar
Sandeep Kumar committed
1084
1085
void kill_eNB_proc(void) {

1086
  int *status;
1087
1088
  PHY_VARS_eNB *eNB;
  eNB_proc_t *proc;
knopp's avatar
knopp committed
1089
  eNB_rxtx_proc_t *proc_rxtx;
1090
1091
  for (int CC_id=0; CC_id<MAX_NUM_CCs; CC_id++) {
    eNB=PHY_vars_eNB_g[0][CC_id];
1092
1093
    
    proc = &eNB->proc;
knopp's avatar
knopp committed
1094
    proc_rxtx = &proc->proc_rxtx[0];
1095
    
1096
#ifdef DEBUG_THREADS
1097
    printf( "Killing TX CC_id %d thread %d\n", CC_id, i );
1098
#endif
1099
    
knopp's avatar
knopp committed
1100
1101
    proc_rxtx[0].instance_cnt_rxtx = 0; // FIXME data race!
    proc_rxtx[1].instance_cnt_rxtx = 0; // FIXME data race!
Raymond Knopp's avatar
Raymond Knopp committed
1102
    proc->instance_cnt_prach = 0;
knopp's avatar
knopp committed
1103
    pthread_cond_signal( &proc_rxtx[0].cond_rxtx );    
Raymond Knopp's avatar
Raymond Knopp committed
1104
1105
    pthread_cond_signal( &proc_rxtx[1].cond_rxtx );
    pthread_cond_signal( &proc->cond_prach );
1106
    pthread_cond_broadcast(&sync_phy_proc.cond_phy_proc_tx);
Raymond Knopp's avatar
Raymond Knopp committed
1107
1108
1109

    pthread_join( proc->pthread_rx, (void**)&status );    

Sandeep Kumar's avatar
Sandeep Kumar committed
1110

knopp's avatar
knopp committed
1111
    int result,i;
Raymond Knopp's avatar
Raymond Knopp committed
1112
    for (i=0;i<2;i++) {
knopp's avatar
knopp committed
1113
      pthread_join( proc_rxtx[i].pthread_rxtx, (void**)&status );
1114
    
1115
#ifdef DEBUG_THREADS
1116
    
knopp's avatar
knopp committed
1117
1118
      if (result != 0) {
	printf( "Error joining thread.\n" );
1119
      } else {
knopp's avatar