codes_mapping.c 20.8 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5 6 7 8 9 10 11
 */

/* SUMMARY:
 * CODES custom mapping file for ROSS
 */
#include "codes/codes_mapping.h"

12 13
#define CODES_MAPPING_DEBUG 0

14 15 16 17 18 19
/* number of LPs assigned to the current PE (abstraction of MPI rank).
 * for lp counts which are not divisible by the number of ranks, keep 
 * modulus around */
static int lps_per_pe_floor = 0;
static int lps_leftover = 0;

20
static int mem_factor = 256;
21

22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
static int mini(int a, int b){ return a < b ? a : b; }

// compare passed in annotation strings (NULL or nonempty) against annotation
// strings in the config (empty or nonempty)
static int cmp_anno(const char * anno_user, const char * anno_config){
    return anno_user == NULL ? anno_config[0]=='\0'
                             : !strcmp(anno_user, anno_config);
}


#if 0
// TODO: this code seems useful, but I'm not sure where to put it for the time
// being. It should be useful when we are directly reading from the
// configuration file, which has the unprocessed annotated strings

// return code matches that of strcmp and friends, checks
// lp_type_name@annotation against full name
// NOTE: no argument should be NULL or invalid strings
static int strcmp_anno(
        const char * full_name,
        const char * prefix,
        const char * annotation){
    int i;
    // first phase - check full_name against prefix
    for (i = 0; i < MAX_NAME_LENGTH; i++){
        char cf = full_name[i], cl = prefix[i];
        if (cl == '\0')
            break; // match successful
        else if (cf != cl) // captures case where cf is null char or @
            return (int)(cf-cl); // lp name on full doesn't match
    }
    if (i==MAX_NAME_LENGTH) return 1;
    // second phase - ensure the next character after the match is an @
    if (full_name[i] != '@') return -1;
    // third phase, compare the remaining full_name against annotation
    return strcmp(full_name+i+1, annotation);
}
#endif
60

61
/* char arrays for holding lp type name and group name*/
62 63 64
// (unused var atm) static char local_group_name[MAX_NAME_LENGTH];
static char local_lp_name[MAX_NAME_LENGTH],
            local_annotation[MAX_NAME_LENGTH];
65

66 67
int codes_mapping_get_lps_for_pe()
{
68 69 70 71 72 73
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#if CODES_MAPPING_DEBUG
    printf("%d lps for rank %d\n", lps_per_pe_floor+(g_tw_mynode < lps_leftover), rank);
#endif
  return lps_per_pe_floor + (g_tw_mynode < lps_leftover);
74 75 76 77 78
}

/* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped */
tw_peid codes_mapping( tw_lpid gid)
{
79 80 81 82 83 84 85 86
    int lps_on_pes_with_leftover = lps_leftover * (lps_per_pe_floor+1);
    if (gid < lps_on_pes_with_leftover){
        return gid / (lps_per_pe_floor+1);
    }
    else{
        return (gid-lps_on_pes_with_leftover)/lps_per_pe_floor + lps_leftover;
    }
  /*return gid / lps_per_pe_floor;*/
87 88
}

89
int codes_mapping_get_group_reps(const char* group_name)
90
{
91 92 93
  int grp;
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
  {
94
     if(strcmp(lpconf.lpgroups[grp].name, group_name) == 0)
95 96 97 98
	     return lpconf.lpgroups[grp].repetitions;
  }
  return -1;
}
99

100 101 102 103 104 105 106 107 108 109
int codes_mapping_get_lp_count(
        const char * group_name,
        int          ignore_repetitions,
        const char * lp_type_name,
        const char * annotation,
        int          ignore_annos){
    int lp_type_ct_total = 0;

    // check - if group name is null, then disable ignore_repetitions (the
    // former takes precedence)
110
    if (group_name == NULL)
111 112
        ignore_repetitions = 0;
    for (int g = 0; g < lpconf.lpgroups_count; g++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
113
        const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
114 115 116 117
        // iterate over the lps if the group is null (count across all groups)
        // or if the group names match
        if (group_name == NULL || strcmp(lpg->name, group_name) == 0){
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
118
                const config_lptype_t *lpt = &lpg->lptypes[l];
119 120 121 122 123 124 125 126 127
                if (strcmp(lp_type_name, lpt->name) == 0){
                    // increment the count if we are ignoring annotations,
                    // query and entry are both unannotated, or if the
                    // annotations match
                    if (ignore_annos || cmp_anno(annotation, lpt->anno)){
                        if (ignore_repetitions)
                            lp_type_ct_total += lpt->count;
                        else
                            lp_type_ct_total += lpt->count * lpg->repetitions;
128 129
                        if (ignore_annos == 1)
                            break;
130 131
                    }
                }
132
            }
133
            if (group_name != NULL) break; // we've processed the exact group
134 135
        }
    }
136 137
    // no error needed here - 0 is a valid value
    return lp_type_ct_total;
138 139
}

140 141 142 143 144 145 146 147 148 149 150 151 152 153
void codes_mapping_get_lp_id(
        const char * group_name,
        const char * lp_type_name,
        const char * annotation,
        int          ignore_anno,
        int          rep_id,
        int          offset,
        tw_lpid    * gid){
    // sanity checks
    if (rep_id < 0 || offset < 0 || group_name == NULL || lp_type_name == NULL)
        goto ERROR;
    *gid = 0;
    // for each group
    for (int g = 0; g < lpconf.lpgroups_count; g++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
154
        const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
155 156 157 158 159 160 161
        // in any case, count up the lps (need for repetition handling)
        tw_lpid rep_count = 0;
        for (int l = 0; l < lpg->lptypes_count; l++){
            rep_count += lpg->lptypes[l].count;
        }
        // does group name match?
        if (strcmp(lpg->name, group_name) == 0){
162
            tw_lpid local_lp_count = 0;
163 164
            // for each lp type
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
165
                const config_lptype_t *lpt = &lpg->lptypes[l];
166 167 168 169 170 171 172 173 174 175
                // does lp name match?
                if (strcmp(lpt->name, lp_type_name) == 0){
                    // does annotation match (or are we ignoring annotations?)
                    if (ignore_anno || cmp_anno(annotation, lpt->anno)){
                        // return if sane offset 
                        if (offset >= (int) lpt->count){
                            goto ERROR;
                        }
                        else{
                            *gid += (rep_count * (tw_lpid)rep_id) + 
176
                                local_lp_count + (tw_lpid)offset;
177 178 179 180
                            return;
                        }
                    }
                }
181
                local_lp_count += lpt->count;
182 183 184 185 186 187 188 189
            }
            // after a group match, short circuit to end if lp not found
            goto ERROR;
        }
        // group name doesn't match, add count to running total and move on
        else{
            *gid += lpg->repetitions * rep_count;
        }
190
    }
191 192 193 194 195
ERROR:
    // LP not found
    tw_error(TW_LOC, "Unable to find LP id given "
                     "group \"%s\", "
                     "typename \"%s\", "
196
                     "annotation \"%s\", "
197
                     "repetition %d, and offset %d",
198 199 200
                     group_name==NULL   ? "<NULL>" : group_name,
                     lp_type_name==NULL ? "<NULL>" : lp_type_name,
                     annotation==NULL   ? "<NULL>" : annotation,
201
                     rep_id, offset);
202 203
}

204 205 206 207 208 209 210
int codes_mapping_get_lp_relative_id(
        tw_lpid gid,
        int     group_wise,
        int     annotation_wise){
    int group_index, lp_type_index, rep_id, offset;
    codes_mapping_get_lp_info(gid, NULL, &group_index,
            local_lp_name, &lp_type_index, local_annotation, &rep_id, &offset);
Jonathan Jenkins's avatar
Jonathan Jenkins committed
211
    const char * anno = (local_annotation[0]=='\0') ? NULL : local_annotation;
212 213 214 215 216 217

    uint64_t group_lp_count = 0;
    // if not group_specific, then count up LPs of all preceding groups
    if (!group_wise){
        for (int g = 0; g < group_index; g++){
            uint64_t lp_count = 0;
Jonathan Jenkins's avatar
Jonathan Jenkins committed
218
            const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
219
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
220
                const config_lptype_t *lpt = &lpg->lptypes[l];
221 222 223 224 225 226 227
                if (strcmp(local_lp_name, lpt->name) == 0){
                    // increment the count if we are ignoring annotations,
                    // both LPs are unannotated, or if the annotations match
                    if (!annotation_wise || cmp_anno(anno, lpt->anno)){
                        lp_count += lpt->count;
                    }
                }
228
            }
229
            group_lp_count += lp_count * lpg->repetitions;
230 231
        }
    }
232 233 234 235 236 237
    // count up LPs within my group occuring before me 
    // (the loop is necessary because different annotated LPs may exist in the
    // same group)
    uint64_t lp_count = 0;
    uint64_t lp_pre_count = 0;
    for (int l = 0; l < lpconf.lpgroups[group_index].lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
238
        const config_lptype_t *lpt = &lpconf.lpgroups[group_index].lptypes[l];
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
        if (strcmp(local_lp_name, lpt->name) == 0){
            if (!annotation_wise || cmp_anno(anno, lpt->anno)){
                lp_count += lpt->count;
                // special case: if we find an LP entry that matches, but is not
                // the same entry where the input gid comes from, then increment
                // a separate "pre" counter
                if (l < lp_type_index){
                    lp_pre_count += lpt->count;
                }
            }
        }
    }
    // now we have the necessary counts
    // return lps in groups that came before + 
    // lps within the group that came before the target LP
    return (int) (group_lp_count + (lp_count * rep_id) + lp_pre_count + offset);
255 256
}

257 258 259 260 261 262 263 264 265 266 267 268
tw_lpid codes_mapping_get_lpid_from_relative(
        int          relative_id,
        const char * group_name,
        const char * lp_type_name,
        const char * annotation,
        int          annotation_wise){
    // strategy: count up all preceding LPs. When we reach a point where an LP
    // type matches, count up the relative ID. When the accumulated relative ID
    // matches or surpasses the input, then we've found our LP
    int rel_id_count = 0;
    tw_lpid gid_count = 0;
    for (int g = 0; g < lpconf.lpgroups_count; g++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
269
        const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
270 271 272 273 274
        if (group_name == NULL || strcmp(group_name, lpg->name) == 0){
            // consider this group for counting
            tw_lpid local_gid_count = 0;
            int local_rel_id_count = 0;
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
275
                const config_lptype_t *lpt = &lpg->lptypes[l];
276 277 278 279 280 281
                local_gid_count += lpt->count;
                if (strcmp(lp_type_name, lpt->name) == 0 &&
                        (!annotation_wise || cmp_anno(annotation, lpt->anno))){
                    local_rel_id_count += lpt->count;
                }
            }
282 283
            // is our relative id within this group?
            if (relative_id < rel_id_count +
284 285 286 287 288 289 290 291
                    lpg->repetitions * local_rel_id_count){
                tw_lpid gid = gid_count;
                int rem = relative_id - rel_id_count;
                int rep = rem / local_rel_id_count;
                rem -= (rep * local_rel_id_count);
                gid += local_gid_count * rep;
                // count up lps listed prior to this entry
                for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
292
                    const config_lptype_t *lpt = &lpg->lptypes[l];
293
                    if (    strcmp(lp_type_name, lpt->name) == 0 &&
294
                            (!annotation_wise ||
295 296 297 298 299
                            cmp_anno(annotation, lpt->anno))){
                        if (rem < (int) lpt->count){
                            return gid + (tw_lpid) rem;
                        }
                        else{
300
                            rem -= lpt->count;
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
                        }
                    }
                    gid += lpt->count;
                }
                // this shouldn't happen
                goto NOT_FOUND;
            }
            else if (group_name != NULL){
                // immediate error - found the group, but not the id
                goto NOT_FOUND;
            }
            else{
                // increment and move to the next group
                rel_id_count += local_rel_id_count * lpg->repetitions;
                gid_count    += local_gid_count    * lpg->repetitions;
            }
        }
        else{
            // just count up the group LPs
            tw_lpid local_gid_count = 0;
            for (int l = 0; l < lpg->lptypes_count; l++){
                local_gid_count += lpg->lptypes[l].count;
            }
324
            gid_count += local_gid_count * lpg->repetitions;
325 326 327 328 329 330 331 332 333 334 335 336
        }
    }
NOT_FOUND:
    tw_error(TW_LOC, "Unable to find LP-ID for ID %d relative to group %s, "
            "lp name %s, annotation %s",
            relative_id,
            group_name == NULL ? "(all)" : group_name,
            lp_type_name,
            annotation_wise ? annotation : "(all)");
    return 0; // dummy return
}

337 338 339 340 341 342 343 344 345 346 347 348 349
void codes_mapping_get_lp_info(
        tw_lpid gid,
        char  * group_name,
        int   * group_index,
        char  * lp_type_name,
        int   * lp_type_index,
        char  * annotation,
        int   * rep_id,
        int   * offset){
    // running total of lp's we've seen so far
    tw_lpid id_total = 0;
    // for each group
    for (int g = 0; g < lpconf.lpgroups_count; g++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
350
        const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
        tw_lpid num_id_group, num_id_per_rep = 0;
        // count up the number of ids in this group
        for (int l = 0; l < lpg->lptypes_count; l++){
            num_id_per_rep += lpg->lptypes[l].count;
        }
        num_id_group = num_id_per_rep * lpg->repetitions;
        if (num_id_group+id_total > gid){
            // we've found the group
            tw_lpid rem = gid - id_total;
            if (group_name != NULL)
                strncpy(group_name, lpg->name, MAX_NAME_LENGTH);
            *group_index = g;
            // find repetition within group
            *rep_id = (int) (rem / num_id_per_rep);
            rem -=  num_id_per_rep * (tw_lpid)*rep_id;
            num_id_per_rep = 0;
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
368
                const config_lptype_t *lpt = &lpg->lptypes[l];
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
                if (rem < num_id_per_rep + lpt->count){
                    // found the specific LP
                    if (lp_type_name != NULL)
                        strncpy(lp_type_name, lpt->name, MAX_NAME_LENGTH);
                    if (annotation != NULL)
                        strncpy(annotation, lpt->anno, MAX_NAME_LENGTH);
                    *offset = (int) (rem - num_id_per_rep);
                    *lp_type_index = l;
                    return; // done
                }
                else{
                    num_id_per_rep += lpg->lptypes[l].count;
                }
            }
        }
        else{
            id_total += num_id_group;
        }
387
    }
388 389
    // LP not found
    tw_error(TW_LOC, "Unable to find LP info given gid %lu", gid);
390 391 392
}

/* This function assigns local and global LP Ids to LPs */
393
static void codes_mapping_init(void)
394 395 396 397 398
{
     int grp_id, lpt_id, rep_id, offset;
     tw_lpid ross_gid, ross_lid; /* ross global and local IDs */
     tw_pe * pe;
     char lp_type_name[MAX_NAME_LENGTH];
399
     int nkp_per_pe = g_tw_nkp;
400
     tw_lpid         lpid, kpid;
401
     const tw_lptype *lptype;
402 403 404 405 406

     /* have 16 kps per pe, this is the optimized configuration for ROSS custom mapping */
     for(kpid = 0; kpid < nkp_per_pe; kpid++)
	tw_kp_onpe(kpid, g_tw_pe[0]);

407 408 409 410
     int lp_start =
         g_tw_mynode * lps_per_pe_floor + mini(g_tw_mynode,lps_leftover);
     int lp_end =
         (g_tw_mynode+1) * lps_per_pe_floor + mini(g_tw_mynode+1,lps_leftover);
411

412
     for (lpid = lp_start; lpid < lp_end; lpid++)
413 414
      {
	 ross_gid = lpid;
415
	 ross_lid = lpid - lp_start;
416 417
	 kpid = ross_lid % g_tw_nkp;
	 pe = tw_getpe(kpid % g_tw_npe);
418 419
	 codes_mapping_get_lp_info(ross_gid, NULL, &grp_id, lp_type_name,
                 &lpt_id, NULL, &rep_id, &offset);
420 421 422
#if CODES_MAPPING_DEBUG
         printf("lp:%lu --> kp:%lu, pe:%llu\n", ross_gid, kpid, pe->id);
#endif
423
	 tw_lp_onpe(ross_lid, pe, ross_gid);
424
	 tw_lp_onkp(g_tw_lp[ross_lid], g_tw_kp[kpid]);
425 426 427 428 429
         lptype = lp_type_lookup(lp_type_name);
         if (lptype == NULL)
             tw_error(TW_LOC, "could not find LP with type name \"%s\", "
                     "did you forget to register the LP?\n", lp_type_name);
         else
430 431
             /* sorry, const... */
             tw_lp_settype(ross_lid, (tw_lptype*) lptype);
432 433 434 435 436 437 438
     }
     return;
}

/* This function takes the global LP ID, maps it to the local LP ID and returns the LP 
 * lps have global and local LP IDs
 * global LP IDs are unique across all PEs, local LP IDs are unique within a PE */
439
static tw_lp * codes_mapping_to_lp( tw_lpid lpid)
440
{
441 442
   int index = lpid - (g_tw_mynode * lps_per_pe_floor) -
       mini(g_tw_mynode, lps_leftover);
443 444 445 446
//   printf("\n global id %d index %d lps_before %d lps_offset %d local index %d ", lpid, index, lps_before, g_tw_mynode, local_index);
   return g_tw_lp[index];
}

447
/* This function loads the configuration file and sets up the number of LPs on each PE */
448
void codes_mapping_setup_with_seed_offset(int offset)
449 450 451 452
{
  int grp, lpt, message_size;
  int pes = tw_nnodes();

453
  lps_per_pe_floor = 0;
454 455 456
  for (grp = 0; grp < lpconf.lpgroups_count; grp++)
   {
    for (lpt = 0; lpt < lpconf.lpgroups[grp].lptypes_count; lpt++)
457
	lps_per_pe_floor += (lpconf.lpgroups[grp].lptypes[lpt].count * lpconf.lpgroups[grp].repetitions);
458
   }
459
  tw_lpid global_nlps = lps_per_pe_floor;
460 461 462
  lps_leftover = lps_per_pe_floor % pes;
  lps_per_pe_floor /= pes;
 //printf("\n LPs for this PE are %d reps %d ", lps_per_pe_floor,  lpconf.lpgroups[grp].repetitions);
463 464 465
  g_tw_mapping=CUSTOM;
  g_tw_custom_initial_mapping=&codes_mapping_init;
  g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
466 467 468 469 470 471 472 473

  // configure mem-factor
  int mem_factor_conf;
  int rc = configuration_get_value_int(&config, "PARAMS", "pe_mem_factor", NULL,
          &mem_factor_conf);
  if (rc == 0 && mem_factor_conf > 0)
    mem_factor = mem_factor_conf;

474
  g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe();
475
  configuration_get_value_int(&config, "PARAMS", "message_size", NULL, &message_size);
476 477 478 479 480
  if(!message_size)
  {
      message_size = 256;
      printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
  }
481 482 483 484 485

  // we increment the number of RNGs used to let codes_local_latency use the
  // last one
  g_tw_nRNG_per_lp++;

486
  tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0);
487 488 489 490 491 492 493 494 495 496

  // use a similar computation to codes_mapping_init to compute the lpids and
  // offsets to use in tw_rand_initial_seed
  // an "offset" of 0 reverts to default RNG seeding behavior - see
  // ross/rand-clcg4.c for the specific computation
  // an "offset" < 0 is ignored
  if (offset > 0){
      for (tw_lpid l = 0; l < g_tw_nlp; l++){
          for (int i = 0; i < g_tw_nRNG_per_lp; i++){
              tw_rand_initial_seed(&g_tw_lp[l]->rng[i], (g_tw_lp[l]->gid +
497
                          global_nlps * offset) * g_tw_nRNG_per_lp + i);
498 499 500 501 502 503 504
          }
      }
  }
}

void codes_mapping_setup(){
    codes_mapping_setup_with_seed_offset(0);
505 506
}

507
/* given the group and LP type name, return the annotation (or NULL) */
508 509 510 511
const char* codes_mapping_get_annotation_by_name(
        const char * group_name,
        const char * lp_type_name){
    for (int g = 0; g < lpconf.lpgroups_count; g++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
512
        const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
513 514 515
        if (strcmp(lpg->name, group_name) == 0){
            // group found, iterate through types
            for (int l = 0; l < lpg->lptypes_count; l++){
Jonathan Jenkins's avatar
Jonathan Jenkins committed
516
                const config_lptype_t *lpt = &lpg->lptypes[l];
517 518 519 520 521 522 523 524
                if (strcmp(lpt->name, lp_type_name) == 0){
                    // type found, return the annotation
                    if (lpt->anno[0] == '\0')
                        return NULL;
                    else
                        return lpt->anno;
                }
            }
525 526
        }
    }
527 528 529
    tw_error(TW_LOC, "unable to find annotation using "
            "group \"%s\" and lp_type_name \"%s\"", group_name, lp_type_name);
    return NULL;
530 531 532
}

const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){
533 534 535 536 537 538 539 540 541
    int group_index, lp_type_index, dummy;
    codes_mapping_get_lp_info(gid, NULL, &group_index, NULL, &lp_type_index,
            NULL, &dummy, &dummy);
    const char * anno = 
        lpconf.lpgroups[group_index].lptypes[lp_type_index].anno;
    if (anno[0] == '\0')
        return NULL;
    else
        return anno;
542
}
Philip Carns's avatar
Philip Carns committed
543

544 545 546
/*
 * Returns a mapping of LP name to all annotations used with the type
 *
547
 * lp_name - lp name as used in the configuration
548 549 550 551 552 553 554 555 556 557 558
 */
const config_anno_map_t * 
codes_mapping_get_lp_anno_map(const char *lp_name){
    for (uint64_t i = 0; i < lpconf.lpannos_count; i++){
        if (strcmp(lp_name, lpconf.lpannos[i].lp_name) == 0){
            return &lpconf.lpannos[i];
        }
    }
    return NULL;
}

Philip Carns's avatar
Philip Carns committed
559 560 561 562 563 564 565 566
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */