ethernet_lib.c 16.9 KB
Newer Older
knopp's avatar
 
knopp committed
1
/*******************************************************************************
2
    OpenAirInterface
knopp's avatar
 
knopp committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16
    Copyright(c) 1999 - 2014 Eurecom

    OpenAirInterface is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.


    OpenAirInterface is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
17 18
    along with OpenAirInterface.The full GNU General Public License is
    included in this distribution in the file called "COPYING". If not,
knopp's avatar
 
knopp committed
19 20 21 22 23
    see <http://www.gnu.org/licenses/>.

   Contact Information
   OpenAirInterface Admin: openair_admin@eurecom.fr
   OpenAirInterface Tech : openair_tech@eurecom.fr
24
   OpenAirInterface Dev  : openair4g-devel@lists.eurecom.fr
25

knopp's avatar
 
knopp committed
26 27 28
   Address      : Eurecom, Campus SophiaTech, 450 Route des Chappes, CS 50193 - 06904 Biot Sophia Antipolis cedex, FRANCE

 *******************************************************************************/
navid's avatar
navid committed
29
/*! \file ethernet_lib.c 
30
 * \brief API to stream I/Q samples over standard ethernet
31
 * \author  add alcatel Katerina Trilyraki, Navid Nikaein, Pedro Dinis, Lucio Ferreira, Raymond Knopp
32 33 34 35 36 37 38
 * \date 2015
 * \version 0.2
 * \company Eurecom
 * \maintainer:  navid.nikaein@eurecom.fr
 * \note
 * \warning 
 */
knopp's avatar
 
knopp committed
39 40 41 42 43 44 45 46 47 48

#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <net/if.h>
#include <netinet/ether.h>
knopp's avatar
 
knopp committed
49 50
#include <unistd.h>
#include <errno.h>
knopp's avatar
 
knopp committed
51 52

#include "common_lib.h"
53
#include "ethernet_lib.h"
knopp's avatar
 
knopp committed
54

55
int num_devices_eth = 0;
56
struct sockaddr_in dest_addr[MAX_INST];
knopp's avatar
 
knopp committed
57
int dest_addr_len[MAX_INST];
navid's avatar
navid committed
58 59 60 61 62 63


int trx_eth_start(openair0_device *device) {

  eth_state_t *eth = (eth_state_t*)device->priv;
  
64
  /* initialize socket */
65
  if (eth->flags == ETH_RAW_MODE) {     
66 67 68 69 70 71
    if (eth_socket_init_raw(device)!=0)   return -1;
    /* RRH gets openair0 device configuration - BBU sets openair0 device configuration*/
    if (device->host_type == BBU_HOST) {
      if(eth_set_dev_conf_raw(device)!=0)  return -1;
    } else {
      if(eth_get_dev_conf_raw(device)!=0)  return -1;
navid's avatar
navid committed
72
    }
73
    /* adjust MTU wrt number of samples per packet */
74
    if(ethernet_tune (device,MTU_SIZE,RAW_PACKET_SIZE_BYTES(device->openair0_cfg->samples_per_packet))!=0)  return -1;
75 76 77 78 79 80 81 82 83 84

  } else if (eth->flags == ETH_RAW_IF4_MODE) {
    if (eth_socket_init_raw(device)!=0)   return -1;
    /* RRH gets openair0 device configuration - BBU sets openair0 device configuration*/
    if (device->host_type == BBU_HOST) {
      if(eth_set_dev_conf_raw_IF4(device)!=0)  return -1;
    } else {
      if(eth_get_dev_conf_raw_IF4(device)!=0)  return -1;
    }
    /* adjust MTU wrt number of samples per packet */
85
    if(ethernet_tune (device,MTU_SIZE,RAW_IF4_PRACH_SIZE_BYTES)!=0)  return -1;
86 87 88 89
    
  } else if (eth->flags == ETH_UDP_IF4_MODE) {
    
  
90 91 92 93 94 95 96 97 98 99 100
  } else if (eth->flags == ETH_RAW_IF5_MOBIPASS) {
    if (eth_socket_init_raw(device)!=0)   return -1;
    /* RRH gets openair0 device configuration - BBU sets openair0 device configuration*/
    //if (device->host_type == BBU_HOST) {
      //if(eth_set_dev_conf_raw_IF4(device)!=0)  return -1;
    //} else {
      //if(eth_get_dev_conf_raw_IF4(device)!=0)  return -1;
//
    /* adjust MTU wrt number of samples per packet */
   // if(ethernet_tune (device,MTU_SIZE,RAW_PACKET_SIZE_BYTES(device->openair0_cfg->samples_per_packet))!=0)  return -1;

101 102 103 104 105 106 107
  } else {
    if (eth_socket_init_udp(device)!=0)   return -1; 
    /* RRH gets openair0 device configuration - BBU sets openair0 device configuration*/
    if (device->host_type == BBU_HOST) {
      if(eth_set_dev_conf_udp(device)!=0)  return -1;
    } else {
      if(eth_get_dev_conf_udp(device)!=0)  return -1;
108 109
    }
  }
110 111 112 113
  /* apply additional configuration */
  if(ethernet_tune (device, SND_BUF_SIZE,2000000000)!=0)  return -1;
  if(ethernet_tune (device, RCV_BUF_SIZE,2000000000)!=0)  return -1;
  
114
  return 0;
115
}
knopp's avatar
 
knopp committed
116

117

navid's avatar
navid committed
118
void trx_eth_end(openair0_device *device) {
119 120

  eth_state_t *eth = (eth_state_t*)device->priv;
navid's avatar
navid committed
121
  int Mod_id = device->Mod_id;
122
  /* destroys socket only for the processes that call the eth_end fuction-- shutdown() for beaking the pipe */
navid's avatar
navid committed
123 124 125 126
  if ( close(eth->sockfd[Mod_id]) <0 ) {
    perror("ETHERNET: Failed to close socket");
    exit(0);
   } else {
127
    printf("[%s] socket for mod_id %d has been successfully closed.\n",(device->host_type == BBU_HOST)? "BBU":"RRH",Mod_id);
navid's avatar
navid committed
128 129
   }
 
130
}
knopp's avatar
 
knopp committed
131 132


navid's avatar
navid committed
133
int trx_eth_request(openair0_device *device, void *msg, ssize_t msg_len) {
134 135 136

  int 	       Mod_id = device->Mod_id;
  eth_state_t *eth = (eth_state_t*)device->priv;
navid's avatar
navid committed
137 138
 
  /* BBU sends a message to RRH */
139
 if (sendto(eth->sockfd[Mod_id],msg,msg_len,0,(struct sockaddr *)&dest_addr[Mod_id],dest_addr_len[Mod_id])==-1) {
140 141 142 143 144
    perror("ETHERNET: ");
    exit(0);
  }
     
  return 0;
knopp's avatar
 
knopp committed
145 146 147
}


navid's avatar
navid committed
148
int trx_eth_reply(openair0_device *device, void *msg, ssize_t msg_len) {
knopp's avatar
 
knopp committed
149

150 151
  eth_state_t   *eth = (eth_state_t*)device->priv;
  int 		Mod_id = device->Mod_id;
knopp's avatar
 
knopp committed
152

navid's avatar
navid committed
153
  /* RRH receives from BBU a message */
154 155 156 157
  if (recvfrom(eth->sockfd[Mod_id],
	       msg,
	       msg_len,
	       0,
158
	       (struct sockaddr *)&dest_addr[Mod_id],
navid's avatar
navid committed
159
	       (socklen_t *)&dest_addr_len[Mod_id])==-1) {
160 161
    perror("ETHERNET: ");
    exit(0);
navid's avatar
navid committed
162 163
  }	
 
164 165 166
   return 0;
}

navid's avatar
navid committed
167

knopp's avatar
 
knopp committed
168

169 170 171
int trx_eth_stop(int card) {
  return(0);
}
172

173
int trx_eth_set_freq(openair0_device* device, openair0_config_t *openair0_cfg,int exmimo_dump_config) {
174
  return(0);
175
}
knopp's avatar
 
knopp committed
176

177 178
int trx_eth_set_gains(openair0_device* device, openair0_config_t *openair0_cfg) {
  return(0);
knopp's avatar
 
knopp committed
179 180
}

181
int trx_eth_get_stats(openair0_device* device) {
182 183
  return(0);
}
184 185

int trx_eth_reset_stats(openair0_device* device) {
186
  return(0);
187
}
188

189

190
int ethernet_tune(openair0_device *device, unsigned int option, int value) {
navid's avatar
navid committed
191
  
192
  eth_state_t *eth = (eth_state_t*)device->priv;
navid's avatar
navid committed
193
  int Mod_id=device->Mod_id;
194
  struct timeval timeout;
navid's avatar
navid committed
195 196 197
  struct ifreq ifr;   
  char system_cmd[256]; 
  char* if_name=DEFAULT_IF;
198 199 200 201 202 203 204 205
  struct in_addr ia;
  struct if_nameindex *ids;
  int ret=0;
  int i=0;
  
  /****************** socket level options ************************/  
  switch(option) {
  case SND_BUF_SIZE:  /* transmit socket buffer size */   
navid's avatar
navid committed
206 207 208
    if (setsockopt(eth->sockfd[Mod_id],  
		   SOL_SOCKET,  
		   SO_SNDBUF,  
209
		   &value,sizeof(value))) {
navid's avatar
navid committed
210 211
      perror("[ETHERNET] setsockopt()");
    } else {
212 213 214 215 216
      printf("send buffer size= %d bytes\n",value); 
    }   
    break;
    
  case RCV_BUF_SIZE:   /* receive socket buffer size */   
navid's avatar
navid committed
217 218 219
    if (setsockopt(eth->sockfd[Mod_id],  
		   SOL_SOCKET,  
		   SO_RCVBUF,  
220
		   &value,sizeof(value))) {
navid's avatar
navid committed
221 222
      perror("[ETHERNET] setsockopt()");
    } else {     
223
      printf("receive bufffer size= %d bytes\n",value);    
navid's avatar
navid committed
224
    }
225 226 227 228 229
    break;
    
  case RCV_TIMEOUT:
    timeout.tv_sec = value/1000000000;
    timeout.tv_usec = value%1000000000;//less than rt_period?
navid's avatar
navid committed
230 231 232
    if (setsockopt(eth->sockfd[Mod_id],  
		   SOL_SOCKET,  
		   SO_RCVTIMEO,  
233
		   (char *)&timeout,sizeof(timeout))) {
navid's avatar
navid committed
234 235
      perror("[ETHERNET] setsockopt()");  
    } else {   
236
      printf( "receive timeout= %d,%d sec\n",timeout.tv_sec,timeout.tv_usec);  
navid's avatar
navid committed
237
    }  
238 239 240 241 242
    break;
    
  case SND_TIMEOUT:
    timeout.tv_sec = value/1000000000;
    timeout.tv_usec = value%1000000000;//less than rt_period?
navid's avatar
navid committed
243 244 245
    if (setsockopt(eth->sockfd[Mod_id],  
		   SOL_SOCKET,  
		   SO_SNDTIMEO,  
246
		   (char *)&timeout,sizeof(timeout))) {
navid's avatar
navid committed
247 248
      perror("[ETHERNET] setsockopt()");     
    } else {
249
      printf( "send timeout= %d,%d sec\n",timeout.tv_sec,timeout.tv_usec);    
navid's avatar
navid committed
250
    }
251 252 253 254 255
    break;
    
    
    /******************* interface level options  *************************/
  case MTU_SIZE: /* change  MTU of the eth interface */ 
navid's avatar
navid committed
256
    ifr.ifr_addr.sa_family = AF_INET;
257 258
    strncpy(ifr.ifr_name,eth->if_name[Mod_id], sizeof(ifr.ifr_name));
    ifr.ifr_mtu =value;
navid's avatar
navid committed
259 260 261
    if (ioctl(eth->sockfd[Mod_id],SIOCSIFMTU,(caddr_t)&ifr) < 0 )
      perror ("[ETHERNET] Can't set the MTU");
    else 
262 263 264 265
      printf("[ETHERNET] %s MTU size has changed to %d\n",eth->if_name[Mod_id],ifr.ifr_mtu);
    break;
    
  case TX_Q_LEN:  /* change TX queue length of eth interface */ 
navid's avatar
navid committed
266
    ifr.ifr_addr.sa_family = AF_INET;
267 268
    strncpy(ifr.ifr_name,eth->if_name[Mod_id], sizeof(ifr.ifr_name));
    ifr.ifr_qlen =value;
navid's avatar
navid committed
269 270 271
    if (ioctl(eth->sockfd[Mod_id],SIOCSIFTXQLEN,(caddr_t)&ifr) < 0 )
      perror ("[ETHERNET] Can't set the txqueuelen");
    else 
272 273 274
      printf("[ETHERNET] %s txqueuelen size has changed to %d\n",eth->if_name[Mod_id],ifr.ifr_qlen);
    break;
    
navid's avatar
navid committed
275
    /******************* device level options  *************************/
276 277 278 279 280 281 282 283 284
  case COALESCE_PAR:
    ret=snprintf(system_cmd,sizeof(system_cmd),"ethtool -C %s rx-usecs %d",eth->if_name[Mod_id],value);
    if (ret > 0) {
      ret=system(system_cmd);
      if (ret == -1) {
	fprintf (stderr,"[ETHERNET] Can't start shell to execute %s %s",system_cmd, strerror(errno));
      } else {
	printf ("[ETHERNET] status of %s is %i\n",WEXITSTATUS(ret));
      }
navid's avatar
navid committed
285 286 287 288
      printf("[ETHERNET] Coalesce parameters %s\n",system_cmd);
    } else {
      perror("[ETHERNET] Can't set coalesce parameters\n");
    }
289
    break;
navid's avatar
navid committed
290
    
291 292 293 294 295 296 297 298 299 300 301
  case PAUSE_PAR:
    if (value==1) ret=snprintf(system_cmd,sizeof(system_cmd),"ethtool -A %s autoneg off rx off tx off",eth->if_name[Mod_id]);
    else if (value==0) ret=snprintf(system_cmd,sizeof(system_cmd),"ethtool -A %s autoneg on rx on tx on",eth->if_name[Mod_id]);
    else break;
    if (ret > 0) {
      ret=system(system_cmd);
      if (ret == -1) {
	fprintf (stderr,"[ETHERNET] Can't start shell to execute %s %s",system_cmd, strerror(errno));
      } else {
	printf ("[ETHERNET] status of %s is %i\n",WEXITSTATUS(ret));
      }
navid's avatar
navid committed
302 303 304 305
      printf("[ETHERNET] Pause parameters %s\n",system_cmd);
    } else {
      perror("[ETHERNET] Can't set pause parameters\n");
    }
306 307 308 309 310 311 312 313 314 315 316
    break;
    
  case RING_PAR:
    ret=snprintf(system_cmd,sizeof(system_cmd),"ethtool -G %s rx %d tx %d",eth->if_name[Mod_id],value);
    if (ret > 0) {
      ret=system(system_cmd);
      if (ret == -1) {
	fprintf (stderr,"[ETHERNET] Can't start shell to execute %s %s",system_cmd, strerror(errno));
      } else {
	printf ("[ETHERNET] status of %s is %i\n",WEXITSTATUS(ret));
      }            
navid's avatar
navid committed
317 318 319 320
      printf("[ETHERNET] Ring parameters %s\n",system_cmd);
    } else {
      perror("[ETHERNET] Can't set ring parameters\n");
    }
321
    break;
navid's avatar
navid committed
322
    
323 324
  default:
    break;
navid's avatar
navid committed
325
  }
326
  
navid's avatar
navid committed
327
  return 0;
knopp's avatar
 
knopp committed
328 329 330
}


navid's avatar
navid committed
331

332
int transport_init(openair0_device *device, openair0_config_t *openair0_cfg, eth_params_t * eth_params ) {
333 334 335

  eth_state_t *eth = (eth_state_t*)malloc(sizeof(eth_state_t));
  memset(eth, 0, sizeof(eth_state_t));
knopp's avatar
 
knopp committed
336

337 338
  if (eth_params->transp_preference == 1) {
    eth->flags = ETH_RAW_MODE;
339
  } else if (eth_params->transp_preference == 0) {
340
    eth->flags = ETH_UDP_MODE;
341 342 343 344
  } else if (eth_params->transp_preference == 3) {
    eth->flags = ETH_RAW_IF4_MODE;
  } else if (eth_params->transp_preference == 2) {
    eth->flags = ETH_UDP_IF4_MODE;
345 346
  } else if (eth_params->transp_preference == 4) {
    eth->flags = ETH_RAW_IF5_MOBIPASS;
347
  } else {
348 349
    printf("transport_init: Unknown transport preference %d - default to RAW", eth_params->transp_preference);
    eth->flags = ETH_RAW_MODE;
350 351
  }
  
352
  printf("[ETHERNET]: Initializing openair0_device for %s ...\n", ((device->host_type == BBU_HOST) ? "BBU": "RRH"));
353
  device->Mod_id           = num_devices_eth++;
354
  device->transp_type      = ETHERNET_TP;
navid's avatar
navid committed
355
  device->trx_start_func   = trx_eth_start;
356 357
  device->trx_request_func = trx_eth_request;
  device->trx_reply_func   = trx_eth_reply;
358 359
  device->trx_get_stats_func   = trx_eth_get_stats;
  device->trx_reset_stats_func = trx_eth_reset_stats;
360 361
  device->trx_end_func         = trx_eth_end;
  device->trx_stop_func        = trx_eth_stop;
362 363
  device->trx_set_freq_func = trx_eth_set_freq;
  device->trx_set_gains_func = trx_eth_set_gains;
364

365
  if (eth->flags == ETH_RAW_MODE) {
366 367
    device->trx_write_func   = trx_eth_write_raw;
    device->trx_read_func    = trx_eth_read_raw;     
368
  } else if (eth->flags == ETH_UDP_MODE) {
369 370
    device->trx_write_func   = trx_eth_write_udp;
    device->trx_read_func    = trx_eth_read_udp;     
371 372
  } else if (eth->flags == ETH_RAW_IF4_MODE) {
    device->trx_write_func   = trx_eth_write_raw_IF4;
373 374 375
    device->trx_read_func    = trx_eth_read_raw_IF4;     
  } else if (eth->flags == ETH_RAW_IF5_MOBIPASS) {
    device->trx_write_func   = trx_eth_write_raw_IF4;
376 377
    device->trx_read_func    = trx_eth_read_raw_IF4;     
  } else {
378 379
    //device->trx_write_func   = trx_eth_write_udp_IF4;
    //device->trx_read_func    = trx_eth_read_udp_IF4;     
380
  }
381
    
382 383 384 385
  eth->if_name[device->Mod_id] = eth_params->local_if_name;
  device->priv = eth;
 	
  /* device specific */
386
  openair0_cfg[0].iq_rxrescale = 15;//rescale iqs
387 388
  openair0_cfg[0].iq_txshift = eth_params->iq_txshift;// shift
  openair0_cfg[0].tx_sample_advance = eth_params->tx_sample_advance;
389 390 391 392 393 394

  /* RRH does not have any information to make this configuration atm */
  if (device->host_type == BBU_HOST) {
    /*Note scheduling advance values valid only for case 7680000 */    
    switch ((int)openair0_cfg[0].sample_rate) {
    case 30720000:
395
      openair0_cfg[0].samples_per_packet    = 3840;     
396 397
      break;
    case 23040000:     
398
      openair0_cfg[0].samples_per_packet    = 2880;
399 400
      break;
    case 15360000:
401
      openair0_cfg[0].samples_per_packet    = 1920;      
402 403
      break;
    case 7680000:
404
      openair0_cfg[0].samples_per_packet    = 960;     
405
      break;
406
    case 1920000:
407
      openair0_cfg[0].samples_per_packet    = 240;     
408 409 410 411 412 413 414
      break;
    default:
      printf("Error: unknown sampling rate %f\n",openair0_cfg[0].sample_rate);
      exit(-1);
      break;
    }
  }
415
 
416
  device->openair0_cfg=&openair0_cfg[0];
417
  return 0;
knopp's avatar
 
knopp committed
418
}
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451


/**************************************************************************************************************************
 *                                         DEBUGING-RELATED FUNCTIONS                                                     *
 **************************************************************************************************************************/
void dump_packet(char *title, unsigned char* pkt, int bytes, unsigned int tx_rx_flag) {
   
  static int numSend = 1;
  static int numRecv = 1;
  int num, k;
  char tmp[48];
  unsigned short int cksum;
  
  num = (tx_rx_flag)? numSend++:numRecv++;
  for (k = 0; k < 24; k++) sprintf(tmp+k, "%02X", pkt[k]);
  cksum = calc_csum((unsigned short *)pkt, bytes>>2);
  printf("%s-%s (%06d): %s 0x%04X\n", title,(tx_rx_flag)? "TX":"RX", num, tmp, cksum);
}

unsigned short calc_csum (unsigned short *buf, int nwords) {
 
 unsigned long sum;
  for (sum = 0; nwords > 0; nwords--)
    sum += *buf++;
  sum = (sum >> 16) + (sum & 0xffff);
  sum += (sum >> 16);
  return ~sum;
}

void dump_dev(openair0_device *device) {

  eth_state_t *eth = (eth_state_t*)device->priv;
  
452 453
  printf("Ethernet device interface %i configuration:\n" ,device->openair0_cfg->Mod_id);
  printf("       Log level is %i :\n" ,device->openair0_cfg->log_level);	
454
  printf("       RB number: %i, sample rate: %lf \n" ,
455
        device->openair0_cfg->num_rb_dl, device->openair0_cfg->sample_rate);
456
  printf("       BBU configured for %i tx/%i rx channels)\n",
457
	device->openair0_cfg->tx_num_channels,device->openair0_cfg->rx_num_channels);
458 459
   printf("       Running flags: %s %s %s\n",      
	((eth->flags & ETH_RAW_MODE)  ? "RAW socket mode - ":""),
460
	((eth->flags & ETH_UDP_MODE)  ? "UDP socket mode - ":""));	  	
461 462 463 464 465 466
  printf("       Number of iqs dumped when displaying packets: %i\n\n",eth->iqdumpcnt);   
  
}

void inline dump_txcounters(openair0_device *device) {
  eth_state_t *eth = (eth_state_t*)device->priv;  
467
  printf("   Ethernet device interface %i, tx counters:\n" ,device->openair0_cfg->Mod_id);
468 469 470 471 472 473
  printf("   Sent packets: %llu send errors: %i\n",   eth->tx_count, eth->num_tx_errors);	 
}

void inline dump_rxcounters(openair0_device *device) {

  eth_state_t *eth = (eth_state_t*)device->priv;
474
  printf("   Ethernet device interface %i rx counters:\n" ,device->openair0_cfg->Mod_id);
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
  printf("   Received packets: %llu missed packets errors: %i\n", eth->rx_count, eth->num_underflows);	 
}  

void inline dump_buff(openair0_device *device, char *buff,unsigned int tx_rx_flag, int nsamps) {
  
  char *strptr;
  eth_state_t *eth = (eth_state_t*)device->priv;  
  /*need to add ts number of iqs in printf need to fix dump iqs call */
  strptr = (( tx_rx_flag == TX_FLAG) ? "TX" : "RX");
  printf("\n %s, nsamps=%i \n" ,strptr,nsamps);     
  
  if (tx_rx_flag == 1) {
    dump_txcounters(device);
    printf("  First %i iqs of TX buffer\n",eth->iqdumpcnt);
    dump_iqs(buff,eth->iqdumpcnt);
  } else {
    dump_rxcounters(device);
    printf("  First %i iqs of RX buffer\n",eth->iqdumpcnt);
    dump_iqs(buff,eth->iqdumpcnt);      
  }
  
}

void dump_iqs(char * buff, int iq_cnt) {
  int i;
  for (i=0;i<iq_cnt;i++) {
    printf("s%02i: Q=%+ij I=%+i%s",i,
	   ((iqoai_t *)(buff))[i].q,
	   ((iqoai_t *)(buff))[i].i,
	   ((i+1)%3 == 0) ? "\n" : "  ");
  }   
}