memory_pools.c 19.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
/*******************************************************************************

 Eurecom OpenAirInterface
 Copyright(c) 1999 - 2012 Eurecom

 This program is free software; you can redistribute it and/or modify it
 under the terms and conditions of the GNU General Public License,
 version 2, as published by the Free Software Foundation.

 This program is distributed in the hope it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 more details.

 You should have received a copy of the GNU General Public License along with
 this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

 The full GNU General Public License is included in this distribution in
 the file called "COPYING".

 Contact Information
 Openair Admin: openair_admin@eurecom.fr
 Openair Tech : openair_tech@eurecom.fr
 Forums       : http://forums.eurecom.fr/openairinterface
 Address      : EURECOM, Campus SophiaTech, 450 Route des Chappes
 06410 Biot FRANCE

 *******************************************************************************/

winckel's avatar
winckel committed
31
32
33
34
#ifdef RTAI
# include <rtai_shm.h>
#endif

35
36
37
#include "assertions.h"
#include "memory_pools.h"

38
39
40
41
42
43
/*------------------------------------------------------------------------------*/
const static int mp_debug = 0;

#ifdef RTAI
# define MP_DEBUG(x, args...) do { if (mp_debug) rt_printk("[MP][D]"x, ##args); } \
    while(0)
44
45
# define MP_ERROR(x, args...) do { rt_printk("[MP][E]"x, ##args); } \
    while(0)
46
47
48
#else
# define MP_DEBUG(x, args...) do { if (mp_debug) fprintf(stdout, "[MP][D]"x, ##args); fflush (stdout); } \
    while(0)
49
50
# define MP_ERROR(x, args...) do { fprintf(stdout, "[MP][E]"x, ##args); } \
    while(0)
51
52
#endif

53
54
55
56
/*------------------------------------------------------------------------------*/
#define CHARS_TO_UINT32(c1, c2, c3, c4) (((c1) << 24) | ((c2) << 16) | ((c3) << 8) | (c4))

/*------------------------------------------------------------------------------*/
57
58
typedef int32_t     items_group_position_t;
typedef int32_t     items_group_index_t;
59

60
61
62
63
64
65
typedef struct items_group_s {
    items_group_position_t          number;
    volatile items_group_position_t current;
    volatile items_group_position_t minimum;
    volatile items_group_index_t   *indexes;
} items_group_t;
66

67
68
69
/*------------------------------------------------------------------------------*/
static const items_group_position_t ITEMS_GROUP_POSITION_INVALID    = -1;
static const items_group_index_t    ITEMS_GROUP_INDEX_INVALID       = -1;
70

71
72
73
74
75
76
77
78
79
80
81
/*------------------------------------------------------------------------------*/
typedef uint32_t    pool_item_start_mark_t;
typedef uint32_t    pool_item_end_mark_t;

typedef uint32_t    memory_pool_data_t;

typedef uint32_t    pool_start_mark_t;

typedef uint32_t    pools_start_mark_t;

typedef uint8_t     pool_id_t;
82
typedef uint8_t     item_status_t;
83
84
85
86
87

typedef struct memory_pool_item_start_s {
    pool_item_start_mark_t      start_mark;

    pool_id_t                   pool_id;
88
89
    item_status_t               item_status;
    uint16_t                    info[2];
90
91
92
93
94
95
96
97
} memory_pool_item_start_t;

typedef struct memory_pool_item_end_s {
    pool_item_end_mark_t        end_mark;
} memory_pool_item_end_t;

typedef struct memory_pool_item_s {
    memory_pool_item_start_t    start;
98
    memory_pool_data_t          data[0];
99
100
101
102
103
104
105
    memory_pool_item_end_t      end;
} memory_pool_item_t;

typedef struct memory_pool_s {
    pool_start_mark_t           start_mark;

    pool_id_t                   pool_id;
106
107
    uint32_t                    item_data_number;
    uint32_t                    pool_item_size;
winckel's avatar
winckel committed
108
    items_group_t               items_group_free;
109
110
111
    memory_pool_item_t         *items;
} memory_pool_t;

112

113
114
115
116
117
118
119
120
121
122
typedef struct memory_pools_s {
    pools_start_mark_t          start_mark;

    uint32_t                    pools_number;
    uint32_t                    pools_defined;
    memory_pool_t              *pools;
} memory_pools_t;

/*------------------------------------------------------------------------------*/
static const uint32_t               MAX_POOLS_NUMBER =      20;
123
static const uint32_t               MAX_POOL_ITEMS_NUMBER = 200 * 1000;
124
125
126
127
128
static const uint32_t               MAX_POOL_ITEM_SIZE =    100 * 1000;

static const pool_item_start_mark_t POOL_ITEM_START_MARK =  CHARS_TO_UINT32 ('P', 'I', 's', 't');
static const pool_item_end_mark_t   POOL_ITEM_END_MARK =    CHARS_TO_UINT32 ('p', 'i', 'E', 'N');

129
130
131
static const item_status_t          ITEM_STATUS_FREE =      'F';
static const item_status_t          ITEM_STATUS_ALLOCATED = 'a';

132
133
134
135
136
static const pool_start_mark_t      POOL_START_MARK =       CHARS_TO_UINT32 ('P', '_', 's', 't');

static const pools_start_mark_t     POOLS_START_MARK =      CHARS_TO_UINT32 ('P', 'S', 's', 't');

/*------------------------------------------------------------------------------*/
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
static inline items_group_index_t items_group_get_free_item (items_group_t *items_group)
{
    items_group_position_t  current;
    items_group_index_t     index = -1;

    /* Get current position and decrease it */
    current = __sync_fetch_and_add (&items_group->current, -1);
    if (current <= ITEMS_GROUP_POSITION_INVALID)
    {
        /* Current index is not valid, restore previous value */
        __sync_fetch_and_add (&items_group->current, 1);
    }
    else
    {
        /* Updates minimum position if needed */
        while (items_group->minimum > current)
        {
            items_group->minimum = current;
        }

        /* Get index at current position */
        index = items_group->indexes[current];
        DevCheck (index > ITEMS_GROUP_INDEX_INVALID, current, index, 0);

        /* Clear index at current position */
        items_group->indexes[current] = ITEMS_GROUP_INDEX_INVALID;
    }

    return (index);
}

static inline void items_group_put_free_item (items_group_t *items_group, items_group_index_t index)
{
    items_group_position_t  next;
    items_group_position_t  current = ITEMS_GROUP_POSITION_INVALID;
    items_group_index_t     index_to_add;
    items_group_index_t     index_previous;

    index_to_add = index - ITEMS_GROUP_INDEX_INVALID;

    do
    {
        /* Calculate next position */
        next = items_group->current + 1;
        /* Checks if next position is free */
182
183
184
185
186
        if (items_group->indexes[next] != ITEMS_GROUP_INDEX_INVALID)
        {
            MP_DEBUG(" items_group_put_free_item (items_group->indexes[next] != ITEMS_GROUP_INDEX_INVALID) %d, %d\n", next, index);
        }
        else
187
188
189
190
191
192
193
194
195
        {
            /* Try to write index in next position */
            index_previous = __sync_fetch_and_add (&items_group->indexes[next], index_to_add);
            /* Checks if next position was still free */
            if (index_previous != ITEMS_GROUP_INDEX_INVALID)
            {
                /* Next position was not free anymore, restore its value */
                __sync_fetch_and_add (&items_group->indexes[next], -index_to_add);
                current = ITEMS_GROUP_POSITION_INVALID;
196
197

                MP_DEBUG(" items_group_put_free_item (index_previous != ITEMS_GROUP_INDEX_INVALID) %d\n", index);
198
199
200
201
202
203
204
205
206
            }
            else
            {
                /* Checks if next position content is correctly set */
                if (items_group->indexes[next] != index)
                {
                    /* Next position content has been changed, restore its value */
                    __sync_fetch_and_add (&items_group->indexes[next], -index_to_add);
                    current = ITEMS_GROUP_POSITION_INVALID;
207
208

                    MP_DEBUG(" items_group_put_free_item (items_group->indexes[next] != index) %d\n", index);
209
210
211
212
213
214
215
216
217
218
219
                }
                else
                {
                    /* Increase current position and get it */
                    current = __sync_add_and_fetch (&items_group->current, 1);

                    if (next != current)
                    {
                        /* Current position does not match calculated next position, restore previous values */
                        __sync_fetch_and_add (&items_group->current, -1);
                        __sync_fetch_and_add (&items_group->indexes[next], -index_to_add);
220
221

                        MP_DEBUG(" items_group_put_free_item (next != current) %d\n", index);
222
223
224
225
226
227
228
229
                    }
                }
            }
        }
    } while (next != current);

    DevCheck (current < items_group->number, current, items_group->number, 0);
}
230
231

/*------------------------------------------------------------------------------*/
232
static inline memory_pools_t *memory_pools_from_handler (memory_pools_handle_t memory_pools_handle)
winckel's avatar
winckel committed
233
234
235
236
237
238
239
240
241
242
243
{
    memory_pools_t *memory_pools;

    /* Recover memory_pools */
    memory_pools = (memory_pools_t *) memory_pools_handle;
    /* Sanity check on passed handle */
    DevAssert (memory_pools->start_mark == POOLS_START_MARK);

    return (memory_pools);
}

244
static inline memory_pool_item_t *memory_pool_item_from_handler (memory_pool_item_handle_t memory_pool_item_handle)
winckel's avatar
winckel committed
245
246
247
248
249
{
    void               *address;
    memory_pool_item_t *memory_pool_item;

    /* Recover memory_pools */
250
    address = memory_pool_item_handle - sizeof(memory_pool_item_start_t);
winckel's avatar
winckel committed
251
    memory_pool_item = (memory_pool_item_t *) address;
252

winckel's avatar
winckel committed
253
254
255
256
257
258
    /* Sanity check on passed handle */
    DevAssert (memory_pool_item->start.start_mark == POOL_ITEM_START_MARK);

    return (memory_pool_item);
}

259
static inline memory_pool_item_t *memory_pool_item_from_index (memory_pool_t *memory_pool, items_group_index_t index)
winckel's avatar
winckel committed
260
{
261
    void               *address;
winckel's avatar
winckel committed
262

263
264
    address = (void *) memory_pool->items;
    address += index * memory_pool->pool_item_size;
winckel's avatar
winckel committed
265

266
    return (address);
winckel's avatar
winckel committed
267
}
268
269

/*------------------------------------------------------------------------------*/
270
memory_pools_handle_t memory_pools_create (uint32_t pools_number)
271
272
{
    memory_pools_t *memory_pools;
winckel's avatar
winckel committed
273
    pool_id_t       pool;
274

275
    DevCheck (pools_number <= MAX_POOLS_NUMBER, pools_number, MAX_POOLS_NUMBER, 0); /* Limit to a reasonable number of pools */
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300

    /* Allocate memory_pools */
    memory_pools = malloc (sizeof(memory_pools_t));
    DevAssert (memory_pools != NULL);

    /* Initialize memory_pools */
    {
        memory_pools->start_mark    = POOLS_START_MARK;
        memory_pools->pools_number  = pools_number;
        memory_pools->pools_defined = 0;

        /* Allocate pools */
        memory_pools->pools         = calloc (pools_number, sizeof(memory_pool_t));
        DevAssert (memory_pools->pools != NULL);

        /* Initialize pools */
        for (pool = 0; pool < pools_number; pool++)
        {
            memory_pools->pools[pool].start_mark = POOL_START_MARK;
        }
    }

    return ((memory_pools_handle_t) memory_pools);
}

301
char *memory_pools_statistics(memory_pools_handle_t memory_pools_handle)
302
{
303
304
305
306
307
308
    memory_pools_t     *memory_pools;
    pool_id_t           pool;
    char               *statistics;
    int                 printed_chars;
    uint32_t            allocated_pool_memory;
    uint32_t            allocated_pools_memory = 0;
309

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    /* Recover memory_pools */
    memory_pools = memory_pools_from_handler (memory_pools_handle);

    statistics = malloc(memory_pools->pools_defined * 200);

    printed_chars = sprintf (&statistics[0], "Pool: number,   size, minimum,   free\n");
    for (pool = 0; pool < memory_pools->pools_defined; pool++)
    {
        allocated_pool_memory = memory_pools->pools[pool].items_group_free.number * memory_pools->pools[pool].pool_item_size;
        allocated_pools_memory += allocated_pool_memory;
        printed_chars += sprintf (&statistics[printed_chars], "  %2u: %6u, %6u,  %6u, %6u, %6u Kbytes\n",
                                  pool,
                                  memory_pools->pools[pool].items_group_free.number,
                                  memory_pools->pools[pool].item_data_number * sizeof(memory_pool_data_t),
                                  memory_pools->pools[pool].items_group_free.minimum + 1,
                                  memory_pools->pools[pool].items_group_free.current + 1,
                                  allocated_pool_memory / (1020));
    }
    printed_chars = sprintf (&statistics[printed_chars], "Pools memory %u Kbytes\n", allocated_pools_memory / (1024));

    return (statistics);
}

int memory_pools_add_pool (memory_pools_handle_t memory_pools_handle, uint32_t pool_items_number, uint32_t pool_item_size)
{
    memory_pools_t     *memory_pools;
    memory_pool_t      *memory_pool;
    pool_id_t           pool;
    items_group_index_t item_index;
    memory_pool_item_t *memory_pool_item;

    DevCheck (pool_items_number <= MAX_POOL_ITEMS_NUMBER, pool_items_number, MAX_POOL_ITEMS_NUMBER, 0);  /* Limit to a reasonable number of items */
    DevCheck (pool_item_size <= MAX_POOL_ITEM_SIZE, pool_item_size, MAX_POOL_ITEM_SIZE, 0);              /* Limit to a reasonable item size */
343
344

    /* Recover memory_pools */
winckel's avatar
winckel committed
345
346
347
348
    memory_pools = memory_pools_from_handler (memory_pools_handle);

    /* Check number of already created pools */
    DevAssert (memory_pools->pools_defined < memory_pools->pools_number);
349
350
351
352
353
354
355

    /* Select pool */
    pool = memory_pools->pools_defined;
    memory_pool = &memory_pools->pools[pool];

    /* Initialize pool */
    {
winckel's avatar
winckel committed
356
        memory_pool->pool_id                  = pool;
357
358
359
360
        /* Item size in memory_pool_data_t items by excess */
        memory_pool->item_data_number         = (pool_item_size + sizeof(memory_pool_data_t) - 1) / sizeof(memory_pool_data_t);
        memory_pool->pool_item_size           = (memory_pool->item_data_number * sizeof(memory_pool_data_t)) + sizeof(memory_pool_item_t);
        memory_pool->items_group_free.number  = pool_items_number;
winckel's avatar
winckel committed
361
362
363
364
        memory_pool->items_group_free.current = pool_items_number - 1;
        memory_pool->items_group_free.minimum = pool_items_number - 1;

        /* Allocate free indexes */
365
        memory_pool->items_group_free.indexes = malloc(pool_items_number * sizeof(items_group_index_t));
winckel's avatar
winckel committed
366
367
368
        DevAssert (memory_pool->items_group_free.indexes != NULL);

        /* Initialize free indexes */
369
        for (item_index = 0; item_index < pool_items_number; item_index++)
winckel's avatar
winckel committed
370
        {
371
            memory_pool->items_group_free.indexes[item_index] = item_index;
winckel's avatar
winckel committed
372
        }
373
374

        /* Allocate items */
375
        memory_pool->items = calloc (pool_items_number, memory_pool->pool_item_size);
376
377
378
        DevAssert (memory_pool->items != NULL);

        /* Initialize items */
379
        for (item_index = 0; item_index < pool_items_number; item_index++)
380
        {
381
382
383
384
385
            memory_pool_item                                      = memory_pool_item_from_index (memory_pool, item_index);
            memory_pool_item->start.start_mark                    = POOL_ITEM_START_MARK;
            memory_pool_item->start.pool_id                       = pool;
            memory_pool_item->start.item_status                   = ITEM_STATUS_FREE;
            memory_pool_item->data[memory_pool->item_data_number] = POOL_ITEM_END_MARK;
386
387
388
389
390
391
392
393
        }
    }

    memory_pools->pools_defined ++;

    return (0);
}

394
memory_pool_item_handle_t memory_pools_allocate (memory_pools_handle_t memory_pools_handle, uint32_t item_size, uint16_t info_0, uint16_t info_1)
winckel's avatar
winckel committed
395
{
396
397
398
399
400
    memory_pools_t             *memory_pools;
    memory_pool_item_t         *memory_pool_item;
    memory_pool_item_handle_t   memory_pool_item_handle = NULL;
    pool_id_t                   pool;
    items_group_index_t         item_index = ITEMS_GROUP_INDEX_INVALID;
winckel's avatar
winckel committed
401
402
403
404

    /* Recover memory_pools */
    memory_pools = memory_pools_from_handler (memory_pools_handle);

405
    for (pool = 0; pool < memory_pools->pools_defined; pool++)
winckel's avatar
winckel committed
406
    {
407
        if ((memory_pools->pools[pool].item_data_number * sizeof(memory_pool_data_t)) < item_size)
winckel's avatar
winckel committed
408
409
410
411
412
        {
            /* This memory pool has too small items, skip it */
            continue;
        }

413
414
        item_index = items_group_get_free_item(&memory_pools->pools[pool].items_group_free);
        if (item_index <= ITEMS_GROUP_INDEX_INVALID)
winckel's avatar
winckel committed
415
416
        {
            /* Allocation failed, skip this pool */
winckel's avatar
winckel committed
417
            continue;
winckel's avatar
winckel committed
418
419
420
421
422
423
424
425
        }
        else
        {
            /* Allocation succeed, exit searching loop */
            break;
        }
    }

426
    if (item_index > ITEMS_GROUP_INDEX_INVALID)
winckel's avatar
winckel committed
427
428
    {
        /* Convert item index into memory_pool_item address */
429
430
431
432
433
434
435
436
437
        memory_pool_item                    = memory_pool_item_from_index (&memory_pools->pools[pool], item_index);
        /* Sanity check on item status, must be free */
        DevCheck (memory_pool_item->start.item_status == ITEM_STATUS_FREE, memory_pool_item->start.item_status, pool, item_index);

        memory_pool_item->start.item_status = ITEM_STATUS_ALLOCATED;
        memory_pool_item->start.info[0]     = info_0;
        memory_pool_item->start.info[1]     = info_1;
        memory_pool_item_handle             = memory_pool_item->data;

winckel's avatar
winckel committed
438
        MP_DEBUG(" Alloc [%2u][%6d]{%6d}, %3u %3u, %6u, %p, %p, %p\n",
439
                 pool, item_index,
winckel's avatar
winckel committed
440
                 memory_pools->pools[pool].items_group_free.current,
441
442
443
444
445
                 info_0, info_1,
                 item_size,
                 memory_pools->pools[pool].items,
                 memory_pool_item,
                 memory_pool_item_handle);
446
447
448
    }
    else
    {
449
        MP_DEBUG(" Alloc [--][------]{------}, %3u %3u, %6u, failed!\n", info_0, info_1, item_size);
winckel's avatar
winckel committed
450
451
    }

452
    return memory_pool_item_handle;
winckel's avatar
winckel committed
453
454
}

455
void memory_pools_free (memory_pools_handle_t memory_pools_handle, memory_pool_item_handle_t memory_pool_item_handle, uint16_t info)
winckel's avatar
winckel committed
456
457
{
    memory_pools_t     *memory_pools;
458
    memory_pool_item_t *memory_pool_item;
winckel's avatar
winckel committed
459
    pool_id_t           pool;
460
    items_group_index_t item_index;
winckel's avatar
winckel committed
461
    uint32_t            item_size;
462
    uint32_t            pool_item_size;
winckel's avatar
winckel committed
463
464
465
466
467
468
469
470
471

    /* Recover memory_pools */
    memory_pools = memory_pools_from_handler (memory_pools_handle);
    /* Recover memory pool item */
    memory_pool_item = memory_pool_item_from_handler (memory_pool_item_handle);
    /* Recover pool index */
    pool = memory_pool_item->start.pool_id;
    DevCheck (pool < memory_pools->pools_defined, pool, memory_pools->pools_defined, 0);

472
473
474
475
    item_size = memory_pools->pools[pool].item_data_number;
    pool_item_size = memory_pools->pools[pool].pool_item_size;
    item_index = (((void *) memory_pool_item) - ((void *) memory_pools->pools[pool].items)) / pool_item_size;

winckel's avatar
winckel committed
476
    MP_DEBUG(" Free  [%2u][%6d]{%6d}, %3u %3u,         %p, %p, %p, %u\n",
477
             pool, item_index, memory_pools->pools[pool].items_group_free.current,
478
479
480
             memory_pool_item->start.info[0], memory_pool_item->start.info[1],
             memory_pool_item_handle, memory_pool_item,
             memory_pools->pools[pool].items, item_size * sizeof(memory_pool_data_t));
481

winckel's avatar
winckel committed
482
    /* Sanity check on calculated item index */
483
484
    DevCheck (memory_pool_item == memory_pool_item_from_index(&memory_pools->pools[pool], item_index), memory_pool_item,
              memory_pool_item_from_index(&memory_pools->pools[pool], item_index), pool);
485
    /* Sanity check on end marker, must still be present (no write overflow) */
486
    DevCheck (memory_pool_item->data[item_size] == POOL_ITEM_END_MARK, pool, 0, 0);
487
488
489
490
    /* Sanity check on item status, must be allocated */
    DevCheck (memory_pool_item->start.item_status == ITEM_STATUS_ALLOCATED, memory_pool_item->start.item_status, pool, item_index);

    memory_pool_item->start.item_status = ITEM_STATUS_FREE;
winckel's avatar
winckel committed
491

492
    items_group_put_free_item(&memory_pools->pools[pool].items_group_free, item_index);
winckel's avatar
winckel committed
493
}