codes-dumpi-trace-nw-wrkld.c 35.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright (C) 2014 University of Chicago
 * See COPYRIGHT notice in top-level directory.
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
#include <ross.h>
#include <assert.h>
#include "dumpi/libundumpi/bindings.h"
#include "dumpi/libundumpi/libundumpi.h"
#include "codes/codes-workload.h"
#include "codes/quickhash.h"
17
#include "codes/codes-jobmap.h"
18
#include "codes/jenkins-hash.h"
19
#include "codes/model-net.h"
20

21 22
#if ENABLE_CORTEX
#include <cortex/cortex.h>
23
#include <cortex/datatype.h>
24 25
#include <cortex/cortex-mpich.h>
#ifdef ENABLE_CORTEX_PYTHON
26
#include <cortex/cortex-python.h>
27
#endif
28
#define PROFILE_TYPE cortex_dumpi_profile*
29
//#define UNDUMPI_OPEN cortex_undumpi_open
30 31 32 33
#define DUMPI_START_STREAM_READ cortex_dumpi_start_stream_read
#define UNDUMPI_CLOSE cortex_undumpi_close
#else
#define PROFILE_TYPE dumpi_profile*
34
//#define UNDUMPI_OPEN undumpi_open
35 36 37 38
#define DUMPI_START_STREAM_READ dumpi_start_stream_read
#define UNDUMPI_CLOSE undumpi_close
#endif

39
#define MAX_LENGTH_FILE 512
40 41
#define MAX_OPERATIONS 32768
#define DUMPI_IGNORE_DELAY 100
42
#define RANK_HASH_TABLE_SIZE 400
43

44 45 46
/* This variable is defined in src/network-workloads/model-net-mpi-replay.c */
extern struct codes_jobmap_ctx *jobmap_ctx; 

47 48 49
static struct qhash_table *rank_tbl = NULL;
static int rank_tbl_pop = 0;

50
static unsigned int max_threshold = INT_MAX;
51 52 53
/* context of the MPI workload */
typedef struct rank_mpi_context
{
54
    PROFILE_TYPE profile;
55
    int my_app_id;
56 57
    // whether we've seen an init op (needed for timing correctness)
    int is_init;
58
    unsigned int num_reqs;
59
    unsigned int num_ops;
60 61
    int64_t my_rank;
    double last_op_time;
62
    double init_time;
63 64
    void* dumpi_mpi_array;	
    struct qhash_head hash_link;
65 66
    
    struct rc_stack * completed_ctx;
67 68
} rank_mpi_context;

69 70 71 72 73 74
typedef struct rank_mpi_compare
{
    int app;
    int rank;
} rank_mpi_compare;

75 76 77 78 79 80 81 82
/* Holds all the data about MPI operations from the log */
typedef struct dumpi_op_data_array
{
	struct codes_workload_op* op_array;
        int64_t op_arr_ndx;
        int64_t op_arr_cnt;
} dumpi_op_data_array;

83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
/* timing utilities */

#ifdef __GNUC__
__attribute__((unused))
#endif
static dumpi_clock timediff(
        dumpi_clock end,
        dumpi_clock start)
{
    dumpi_clock temp;
    if ((end.nsec-start.nsec)<0) {
        temp.sec = end.sec-start.sec-1;
        temp.nsec = 1000000000+end.nsec-start.nsec;
    } else {
        temp.sec = end.sec-start.sec;
        temp.nsec = end.nsec-start.nsec;
    }
    return temp;
}

103
/*static inline double time_to_ms_lf(dumpi_clock t){
104 105 106 107
        return (double) t.sec * 1e3 + (double) t.nsec / 1e6;
}
static inline double time_to_us_lf(dumpi_clock t){
        return (double) t.sec * 1e6 + (double) t.nsec / 1e3;
108
}*/
109 110 111
static inline double time_to_ns_lf(dumpi_clock t){
        return (double) t.sec * 1e9 + (double) t.nsec;
}
112
/*static int32_t get_unique_req_id(int32_t request_id)
113 114 115 116
{
    uint32_t pc = 0, pb = 0;
    bj_hashlittle2(&request_id, sizeof(int32_t), &pc, &pb);
    return pc;
117
}*/
118
/*static inline double time_to_s_lf(dumpi_clock t){
119
        return (double) t.sec + (double) t.nsec / 1e9;
120
}*/
121

122
/* load the trace */
123
static int dumpi_trace_nw_workload_load(const char* params, int app_id, int rank);
124 125

/* dumpi implementation of get next operation in the workload */
126
static void dumpi_trace_nw_workload_get_next(int app_id, int rank, struct codes_workload_op *op);
127 128

/* get number of bytes from the workload data type and count */
129
static uint64_t get_num_bytes(rank_mpi_context* my_ctx, dumpi_datatype dt);
130 131

/* computes the delay between MPI operations */
132
static void update_compute_time(const dumpi_time* time, rank_mpi_context* my_ctx);
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194

/* initializes the data structures */
static void* dumpi_init_op_data();

/* removes next operations from the dynamic array */
static void dumpi_remove_next_op(void *mpi_op_array, struct codes_workload_op *mpi_op,
                                      double last_op_time);

/* resets the counters for the dynamic array once the workload is completely loaded*/
static void dumpi_finalize_mpi_op_data(void *mpi_op_array);

/* insert next operation */
static void dumpi_insert_next_op(void *mpi_op_array, struct codes_workload_op *mpi_op);

/* initialize the array data structure */
static void* dumpi_init_op_data()
{
	dumpi_op_data_array* tmp;
	
	tmp = malloc(sizeof(dumpi_op_data_array));
	assert(tmp);
	tmp->op_array = malloc(MAX_OPERATIONS * sizeof(struct codes_workload_op));
	assert(tmp->op_array);
        tmp->op_arr_ndx = 0;
	tmp->op_arr_cnt = MAX_OPERATIONS;

	return (void *)tmp;	
}

/* inserts next operation in the array */
static void dumpi_insert_next_op(void *mpi_op_array, struct codes_workload_op *mpi_op)
{
	dumpi_op_data_array *array = (dumpi_op_data_array*)mpi_op_array;
	struct codes_workload_op *tmp;

	/*check if array is full.*/
	if (array->op_arr_ndx == array->op_arr_cnt)
	{
		tmp = malloc((array->op_arr_cnt + MAX_OPERATIONS) * sizeof(struct codes_workload_op));
		assert(tmp);
		memcpy(tmp, array->op_array, array->op_arr_cnt * sizeof(struct codes_workload_op));
		free(array->op_array);
	        array->op_array = tmp;
	        array->op_arr_cnt += MAX_OPERATIONS;
	}

	/* add the MPI operation to the op array */
	array->op_array[array->op_arr_ndx] = *mpi_op;
	//printf("\n insert time %f end time %f ", array->op_array[array->op_arr_ndx].start_time, array->op_array[array->op_arr_ndx].end_time);
	array->op_arr_ndx++;
	return;
}

/* resets the counters after file is fully loaded */
static void dumpi_finalize_mpi_op_data(void *mpi_op_array)
{
	struct dumpi_op_data_array* array = (struct dumpi_op_data_array*)mpi_op_array;

	array->op_arr_cnt = array->op_arr_ndx;	
	array->op_arr_ndx = 0;
}

195 196 197 198 199 200 201
/* rolls back to previous index */
static void dumpi_roll_back_prev_op(void * mpi_op_array)
{
    dumpi_op_data_array *array = (dumpi_op_data_array*)mpi_op_array;
    array->op_arr_ndx--;
    assert(array->op_arr_ndx >= 0);
}
202 203 204 205
/* removes the next operation from the array */
static void dumpi_remove_next_op(void *mpi_op_array, struct codes_workload_op *mpi_op,
                                      double last_op_time)
{
206 207
    (void)last_op_time;

208 209 210 211 212 213 214 215 216 217
	dumpi_op_data_array *array = (dumpi_op_data_array*)mpi_op_array;
	//printf("\n op array index %d array count %d ", array->op_arr_ndx, array->op_arr_cnt);
	if (array->op_arr_ndx == array->op_arr_cnt)
	 {
		mpi_op->op_type = CODES_WK_END;
	 }
	else
	{
		struct codes_workload_op *tmp = &(array->op_array[array->op_arr_ndx]);
		*mpi_op = *tmp;
218
        array->op_arr_ndx++;
219
	}
220
	/*if(mpi_op->op_type == CODES_WK_END)
221 222 223
	{
		free(array->op_array);
		free(array);
224
	}*/
225 226
}

227 228 229 230 231 232 233 234 235 236
/* check for initialization and normalize reported time */
static inline void check_set_init_time(const dumpi_time *t, rank_mpi_context * my_ctx)
{
    if (!my_ctx->is_init) {
        my_ctx->is_init = 1;
        my_ctx->init_time = time_to_ns_lf(t->start);
        my_ctx->last_op_time = time_to_ns_lf(t->stop) - my_ctx->init_time;
    }
}

237 238 239
/* introduce delay between operations: delay is the compute time NOT spent in MPI operations*/
void update_compute_time(const dumpi_time* time, rank_mpi_context* my_ctx)
{
240 241
    double start = time_to_ns_lf(time->start) - my_ctx->init_time;
    double stop = time_to_ns_lf(time->stop) - my_ctx->init_time;
242
    if((start - my_ctx->last_op_time) > DUMPI_IGNORE_DELAY)
243
    {
244 245 246 247 248 249 250
        struct codes_workload_op wrkld_per_rank;

        wrkld_per_rank.op_type = CODES_WK_DELAY;
        wrkld_per_rank.start_time = my_ctx->last_op_time;
        wrkld_per_rank.end_time = start;
        wrkld_per_rank.u.delay.seconds = (start - my_ctx->last_op_time) / 1e9;
        dumpi_insert_next_op(my_ctx->dumpi_mpi_array, &wrkld_per_rank); 
251
    }
252
    my_ctx->last_op_time = stop;
253 254
}

255 256 257 258 259 260 261 262
static int handleDUMPIInit(
        const dumpi_init *prm,
        uint16_t thread,
        const dumpi_time *cpu,
        const dumpi_time *wall,
        const dumpi_perfinfo *perf,
        void *uarg)
{
263 264 265 266 267 268
    (void)prm;
    (void)thread;
    (void)cpu;
    (void)wall;
    (void)perf;

269
    rank_mpi_context *myctx = (rank_mpi_context*)uarg;
270
    check_set_init_time(wall, myctx);
271 272 273
    return 0;
}

274 275
int handleDUMPIError(const void* prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *uarg)
{
276 277 278 279 280 281 282
    (void)prm;
    (void)thread;
    (void)cpu;
    (void)wall;
    (void)perf;
    (void)uarg;

283 284 285
    tw_error(TW_LOC, "\n MPI operation not supported by the MPI-Sim Layer ");
}

286
int handleDUMPIIgnore(const void* prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *uarg)
287
{
288 289 290 291 292 293 294
    (void)prm;
    (void)thread;
    (void)cpu;
    (void)wall;
    (void)perf;
	
    rank_mpi_context* myctx = (rank_mpi_context*)uarg;
295

296
    check_set_init_time(wall, myctx);
297
	update_compute_time(wall, myctx);
298 299 300 301

	return 0;
}

302 303 304 305 306
static void update_times_and_insert(
        struct codes_workload_op *op,
        const dumpi_time *t,
        rank_mpi_context *ctx)
{
307
    check_set_init_time(t, ctx);
308 309 310
    op->start_time = time_to_ns_lf(t->start) - ctx->init_time;
    op->end_time = time_to_ns_lf(t->stop) - ctx->init_time;
    update_compute_time(t, ctx);
311
    dumpi_insert_next_op(ctx->dumpi_mpi_array, op);
312 313 314
}


315 316 317 318
int handleDUMPIWait(const dumpi_wait *prm, uint16_t thread,
                    const dumpi_time *cpu, const dumpi_time *wall,
                    const dumpi_perfinfo *perf, void *userarg)
{
319 320 321 322 323
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)perf;
        
324 325 326
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;

327
        wrkld_per_rank.op_type = CODES_WK_WAIT;
328 329
        wrkld_per_rank.u.wait.req_id = prm->request;

330 331
        update_times_and_insert(&wrkld_per_rank, wall, myctx);

332 333 334 335 336 337 338
        return 0;
}

int handleDUMPIWaitsome(const dumpi_waitsome *prm, uint16_t thread,
                    const dumpi_time *cpu, const dumpi_time *wall,
                    const dumpi_perfinfo *perf, void *userarg)
{
339 340 341 342 343
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
        
344 345 346 347
        int i;
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;

348
        wrkld_per_rank.op_type = CODES_WK_WAITSOME;
349
        wrkld_per_rank.u.waits.count = prm->count;
350
        wrkld_per_rank.u.waits.req_ids = (unsigned int*)malloc(prm->count * sizeof(unsigned int));
351 352

        for( i = 0; i < prm->count; i++ )
353
                wrkld_per_rank.u.waits.req_ids[i] = prm->requests[i];
354

355
        update_times_and_insert(&wrkld_per_rank, wall, myctx);
356 357 358 359 360 361 362
        return 0;
}

int handleDUMPIWaitany(const dumpi_waitany *prm, uint16_t thread,
                    const dumpi_time *cpu, const dumpi_time *wall,
                    const dumpi_perfinfo *perf, void *userarg)
{
363 364 365 366 367 368
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
        
369 370 371 372
        int i;
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;

373
        wrkld_per_rank.op_type = CODES_WK_WAITANY;
374
        wrkld_per_rank.u.waits.count = prm->count;
375
        wrkld_per_rank.u.waits.req_ids = (unsigned int*)malloc(prm->count * sizeof(unsigned int));
376 377

        for( i = 0; i < prm->count; i++ )
378
                wrkld_per_rank.u.waits.req_ids[i] = prm->requests[i];
379

380
        update_times_and_insert(&wrkld_per_rank, wall, myctx);
381 382 383 384 385 386 387
        return 0;
}

int handleDUMPIWaitall(const dumpi_waitall *prm, uint16_t thread,
                    const dumpi_time *cpu, const dumpi_time *wall,
                    const dumpi_perfinfo *perf, void *userarg)
{
388 389 390 391 392
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
393
        int i;
394
        
395 396 397
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;

398
        wrkld_per_rank.op_type = CODES_WK_WAITALL;
399 400

        wrkld_per_rank.u.waits.count = prm->count;
401
        wrkld_per_rank.u.waits.req_ids = (unsigned int*)malloc(prm->count * sizeof(unsigned int));
402 403 404
        for( i = 0; i < prm->count; i++ )
                wrkld_per_rank.u.waits.req_ids[i] = prm->requests[i];

405
        update_times_and_insert(&wrkld_per_rank, wall, myctx);
406 407 408
        return 0;
}

409 410
int handleDUMPIISend(const dumpi_isend *prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *userarg)
{
411 412 413 414 415 416 417 418
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;
419

420 421 422 423 424
        wrkld_per_rank.op_type = CODES_WK_ISEND;
        wrkld_per_rank.u.send.tag = prm->tag;
        wrkld_per_rank.u.send.count = prm->count;
        wrkld_per_rank.u.send.data_type = prm->datatype;
        wrkld_per_rank.u.send.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
425
        
426
        assert(wrkld_per_rank.u.send.num_bytes >= 0);
427
    	wrkld_per_rank.u.send.req_id = prm->request;
428 429
        wrkld_per_rank.u.send.dest_rank = prm->dest;
        wrkld_per_rank.u.send.source_rank = myctx->my_rank;
430 431

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
432 433
	
        return 0;
434 435 436 437
}

int handleDUMPIIRecv(const dumpi_irecv *prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *userarg)
{
438 439 440 441 442 443 444 445
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        //printf("\n irecv source %d count %d data type %d", prm->source, prm->count, prm->datatype);
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
446 447 448
        struct codes_workload_op wrkld_per_rank;

        wrkld_per_rank.op_type = CODES_WK_IRECV;
449 450 451
	    wrkld_per_rank.u.recv.data_type = prm->datatype;
	    wrkld_per_rank.u.recv.count = prm->count;
	    wrkld_per_rank.u.recv.tag = prm->tag;
452
        wrkld_per_rank.u.recv.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
453
	    
454
        assert(wrkld_per_rank.u.recv.num_bytes >= 0);
455 456
        wrkld_per_rank.u.recv.source_rank = prm->source;
        wrkld_per_rank.u.recv.dest_rank = -1;
457
	    wrkld_per_rank.u.recv.req_id = prm->request;
458 459

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
460 461 462 463 464 465 466
        return 0;
}

int handleDUMPISend(const dumpi_send *prm, uint16_t thread,
                      const dumpi_time *cpu, const dumpi_time *wall,
                      const dumpi_perfinfo *perf, void *uarg)
{
467 468 469 470 471 472 473
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	    
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
474 475 476
        struct codes_workload_op wrkld_per_rank;

        wrkld_per_rank.op_type = CODES_WK_SEND;
477
	    wrkld_per_rank.u.send.tag = prm->tag;
478 479
        wrkld_per_rank.u.send.count = prm->count;
        wrkld_per_rank.u.send.data_type = prm->datatype;
480
        wrkld_per_rank.u.send.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
481
	    assert(wrkld_per_rank.u.send.num_bytes >= 0);
482 483
        wrkld_per_rank.u.send.dest_rank = prm->dest;
        wrkld_per_rank.u.send.source_rank = myctx->my_rank;
484
         wrkld_per_rank.u.send.req_id = -1;
485

486 487
        
         update_times_and_insert(&wrkld_per_rank, wall, myctx);
488 489 490 491 492 493 494
        return 0;
}

int handleDUMPIRecv(const dumpi_recv *prm, uint16_t thread,
                      const dumpi_time *cpu, const dumpi_time *wall,
                      const dumpi_perfinfo *perf, void *uarg)
{
495 496 497 498 499 500
     (void)prm;
     (void)thread;
     (void)cpu;
     (void)wall;
     (void)perf;

501 502 503 504
	rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	struct codes_workload_op wrkld_per_rank;

	wrkld_per_rank.op_type = CODES_WK_RECV;
505 506 507
    wrkld_per_rank.u.recv.tag = prm->tag;
    wrkld_per_rank.u.recv.count = prm->count;
    wrkld_per_rank.u.recv.data_type = prm->datatype;
508
    wrkld_per_rank.u.recv.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
509
	assert(wrkld_per_rank.u.recv.num_bytes >= 0);
510
	wrkld_per_rank.u.recv.req_id = -1;
511 512
    wrkld_per_rank.u.recv.source_rank = prm->source;
    wrkld_per_rank.u.recv.dest_rank = -1;
513

514
	//printf("\n recv source %d count %d data type %d bytes %lld ", prm->source, prm->count, prm->datatype, wrkld_per_rank.u.recv.num_bytes);
515 516
    update_times_and_insert(&wrkld_per_rank, wall, myctx);
    return 0;
517 518 519

}

520 521 522 523
int handleDUMPISendrecv(const dumpi_sendrecv* prm, uint16_t thread,
			const dumpi_time *cpu, const dumpi_time *wall,
			const dumpi_perfinfo *perf, void *uarg)
{
524 525 526 527 528 529 530
     (void)prm;
     (void)thread;
     (void)cpu;
     (void)wall;
     (void)perf;
	
     rank_mpi_context* myctx = (rank_mpi_context*)uarg;
531

532
     /* Issue a non-blocking send */
533 534
	{
		struct codes_workload_op wrkld_per_rank;
535
		wrkld_per_rank.op_type = CODES_WK_ISEND;
536 537 538 539
		wrkld_per_rank.u.send.tag = prm->sendtag;
		wrkld_per_rank.u.send.count = prm->sendcount;
		wrkld_per_rank.u.send.data_type = prm->sendtype;
		wrkld_per_rank.u.send.num_bytes = prm->sendcount * get_num_bytes(myctx,prm->sendtype);
540

541 542
		
        assert(wrkld_per_rank.u.send.num_bytes >= 0);
543 544
		wrkld_per_rank.u.send.dest_rank = prm->dest;
		wrkld_per_rank.u.send.source_rank = myctx->my_rank;
545
		wrkld_per_rank.u.send.req_id = myctx->num_reqs;
546
		update_times_and_insert(&wrkld_per_rank, wall, myctx);
547

548
	}
549
    /* issue a blocking receive */
550 551 552 553 554 555 556
	{
		struct codes_workload_op wrkld_per_rank;
		wrkld_per_rank.op_type = CODES_WK_RECV;
		wrkld_per_rank.u.recv.tag = prm->recvtag;
		wrkld_per_rank.u.recv.count = prm->recvcount;
		wrkld_per_rank.u.recv.data_type = prm->recvtype;
		wrkld_per_rank.u.recv.num_bytes = prm->recvcount * get_num_bytes(myctx,prm->recvtype);
557 558

        assert(wrkld_per_rank.u.recv.num_bytes >= 0);
559 560
		wrkld_per_rank.u.recv.source_rank = prm->source;
		wrkld_per_rank.u.recv.dest_rank = -1;
561
	    wrkld_per_rank.u.recv.req_id = -1;
562 563
		update_times_and_insert(&wrkld_per_rank, wall, myctx);
	}
564 565 566 567 568 569 570 571 572 573 574 575 576
    
    /* Issue a wait operation */
    {
        struct codes_workload_op wrkld_per_rank;

        wrkld_per_rank.op_type = CODES_WK_WAIT;
        wrkld_per_rank.u.wait.req_id = myctx->num_reqs;

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
    
        myctx->num_reqs++;
    }

577

578 579 580
	return 0;
}

581 582 583 584
int handleDUMPIBcast(const dumpi_bcast *prm, uint16_t thread,
                       const dumpi_time *cpu, const dumpi_time *wall,
                       const dumpi_perfinfo *perf, void *uarg)
{
585 586 587 588 589 590 591
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
        struct codes_workload_op wrkld_per_rank;
592 593

        wrkld_per_rank.op_type = CODES_WK_BCAST;
594
        wrkld_per_rank.u.collective.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
595
	    assert(wrkld_per_rank.u.collective.num_bytes >= 0);
596 597

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
598 599 600 601 602 603 604
        return 0;
}

int handleDUMPIAllgather(const dumpi_allgather *prm, uint16_t thread,
                           const dumpi_time *cpu, const dumpi_time *wall,
                           const dumpi_perfinfo *perf, void *uarg)
{
605 606 607 608 609
    (void)prm;
    (void)thread;
    (void)cpu;
    (void)wall;
    (void)perf;
610 611 612
	rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	struct codes_workload_op wrkld_per_rank;

613 614
    wrkld_per_rank.op_type = CODES_WK_ALLGATHER;
    wrkld_per_rank.u.collective.num_bytes = prm->sendcount * get_num_bytes(myctx,prm->sendtype);
615
	assert(wrkld_per_rank.u.collective.num_bytes > 0);
616

617 618
    update_times_and_insert(&wrkld_per_rank, wall, myctx);
    return 0;
619 620 621 622 623 624
}

int handleDUMPIAllgatherv(const dumpi_allgatherv *prm, uint16_t thread,
                            const dumpi_time *cpu, const dumpi_time *wall,
                            const dumpi_perfinfo *perf, void *uarg)
{
625 626 627 628 629 630 631
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	    rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
632 633

        wrkld_per_rank.op_type = CODES_WK_ALLGATHERV;
634
        wrkld_per_rank.u.collective.num_bytes = prm->sendcount * get_num_bytes(myctx,prm->sendtype);
635
	    assert(wrkld_per_rank.u.collective.num_bytes > 0);
636 637

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
638 639 640 641 642 643 644
        return 0;
}

int handleDUMPIAlltoall(const dumpi_alltoall *prm, uint16_t thread,
                          const dumpi_time *cpu, const dumpi_time *wall,
                          const dumpi_perfinfo *perf, void *uarg)
{
645 646 647 648 649 650 651
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	    rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
652 653

        wrkld_per_rank.op_type = CODES_WK_ALLTOALL;
654
        wrkld_per_rank.u.collective.num_bytes = prm->sendcount * get_num_bytes(myctx,prm->sendtype);
655
	    assert(wrkld_per_rank.u.collective.num_bytes > 0);
656 657

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
658 659 660 661 662 663 664
        return 0;
}

int handleDUMPIAlltoallv(const dumpi_alltoallv *prm, uint16_t thread,
                           const dumpi_time *cpu, const dumpi_time *wall,
                           const dumpi_perfinfo *perf, void *uarg)
{
665 666 667 668 669 670 671 672
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
673 674

        wrkld_per_rank.op_type = CODES_WK_ALLTOALLV;
675
        wrkld_per_rank.u.collective.num_bytes = prm->sendcounts[0] * get_num_bytes(myctx,prm->sendtype);
676
	    assert(wrkld_per_rank.u.collective.num_bytes > 0);
677 678

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
679 680 681 682 683 684 685
        return 0;
}

int handleDUMPIReduce(const dumpi_reduce *prm, uint16_t thread,
                        const dumpi_time *cpu, const dumpi_time *wall,
                        const dumpi_perfinfo *perf, void *uarg)
{
686 687 688 689 690 691 692 693
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
694 695

        wrkld_per_rank.op_type = CODES_WK_REDUCE;
696
        wrkld_per_rank.u.collective.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
697
	    assert(wrkld_per_rank.u.collective.num_bytes > 0);
698 699

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
700 701 702 703 704 705 706
        return 0;
}

int handleDUMPIAllreduce(const dumpi_allreduce *prm, uint16_t thread,
                           const dumpi_time *cpu, const dumpi_time *wall,
                           const dumpi_perfinfo *perf, void *uarg)
{
707 708 709 710 711 712 713 714
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
715 716

        wrkld_per_rank.op_type = CODES_WK_ALLREDUCE;
717
        wrkld_per_rank.u.collective.num_bytes = prm->count * get_num_bytes(myctx,prm->datatype);
718
	    assert(wrkld_per_rank.u.collective.num_bytes > 0);
719 720

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
721 722 723 724 725
        return 0;
}

int handleDUMPIFinalize(const dumpi_finalize *prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *uarg)
{
726 727 728 729 730 731 732 733
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
	
        rank_mpi_context* myctx = (rank_mpi_context*)uarg;
	    struct codes_workload_op wrkld_per_rank;
734 735

        wrkld_per_rank.op_type = CODES_WK_END;
736 737

        update_times_and_insert(&wrkld_per_rank, wall, myctx);
738 739 740
        return 0;
}

741 742
int handleDUMPIReqFree(const dumpi_request_free *prm, uint16_t thread, const dumpi_time *cpu, const dumpi_time *wall, const dumpi_perfinfo *perf, void *userarg)
{
743 744 745 746 747 748 749 750
        (void)prm;
        (void)thread;
        (void)cpu;
        (void)wall;
        (void)perf;
    
        rank_mpi_context* myctx = (rank_mpi_context*)userarg;
        struct codes_workload_op wrkld_per_rank;
751

752 753
        wrkld_per_rank.op_type = CODES_WK_REQ_FREE;
        wrkld_per_rank.u.free.req_id = prm->request;
754

755 756
        update_times_and_insert(&wrkld_per_rank, wall, myctx);
        return 0;
757 758
}

759 760
static int hash_rank_compare(void *key, struct qhash_head *link)
{
761
    rank_mpi_compare *in = key;
762 763 764
    rank_mpi_context *tmp;

    tmp = qhash_entry(link, rank_mpi_context, hash_link);
765
    if (tmp->my_rank == in->rank && tmp->my_app_id == in->app)
766 767 768 769
        return 1;
    return 0;
}

770
int dumpi_trace_nw_workload_load(const char* params, int app_id, int rank)
771 772 773
{
	libundumpi_callbacks callbacks;
	libundumpi_cbpair callarr[DUMPI_END_OF_STREAM];
774 775 776 777
#ifdef ENABLE_CORTEX
	libundumpi_cbpair transarr[DUMPI_END_OF_STREAM];
#endif
	PROFILE_TYPE profile;
778
	dumpi_trace_params* dumpi_params = (dumpi_trace_params*)params;
779
	char file_name[MAX_LENGTH_FILE];
780 781 782 783 784 785

	if(rank >= dumpi_params->num_net_traces)
		return -1;

	if(!rank_tbl)
    	{
786
            rank_tbl = qhash_init(hash_rank_compare, quickhash_64bit_hash, RANK_HASH_TABLE_SIZE);
787 788 789 790 791 792 793 794
            if(!rank_tbl)
                  return -1;
    	}
	
	rank_mpi_context *my_ctx;
	my_ctx = malloc(sizeof(rank_mpi_context));
	assert(my_ctx);
	my_ctx->my_rank = rank;
795
    my_ctx->my_app_id = app_id;
796
	my_ctx->last_op_time = 0.0;
797 798
    my_ctx->is_init = 0;
    my_ctx->num_reqs = 0;
799
	my_ctx->dumpi_mpi_array = dumpi_init_op_data();
800
    my_ctx->num_ops = 0;
801 802 803 804 805 806 807 808 809

	if(rank < 10)
            sprintf(file_name, "%s000%d.bin", dumpi_params->file_name, rank);
         else if(rank >=10 && rank < 100)
            sprintf(file_name, "%s00%d.bin", dumpi_params->file_name, rank);
           else if(rank >=100 && rank < 1000)
             sprintf(file_name, "%s0%d.bin", dumpi_params->file_name, rank);
             else
              sprintf(file_name, "%s%d.bin", dumpi_params->file_name, rank);
810
#ifdef ENABLE_CORTEX
811 812 813 814 815
	if(strcmp(dumpi_params->file_name,"none") == 0) {
		profile = cortex_undumpi_open(NULL, app_id, dumpi_params->num_net_traces, rank);
	} else {
		profile = cortex_undumpi_open(file_name, app_id, dumpi_params->num_net_traces, rank);
	}
816
	
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
	{ int i;
	for(i=0; i < dumpi_params->num_net_traces; i++) {
		struct codes_jobmap_id id = {
			.job = app_id,
			.rank = i
		};
		uint32_t cn_id;
		if(jobmap_ctx) {
			cn_id = codes_jobmap_to_global_id(id, jobmap_ctx);
		} else {
			cn_id = i;
		}
		cortex_placement_set(profile, i, cn_id);
	}
	}
832
	
833
	cortex_topology_set(profile,&model_net_topology);
834
#else
835
	profile =  undumpi_open(file_name);
836
#endif
837
        my_ctx->profile = profile;
838 839 840 841 842 843 844
        if(NULL == profile) {
                printf("Error: unable to open DUMPI trace: %s", file_name);
                exit(-1);
        }
	
	memset(&callbacks, 0, sizeof(libundumpi_callbacks));
        memset(&callarr, 0, sizeof(libundumpi_cbpair) * DUMPI_END_OF_STREAM);
845 846 847
#ifdef ENABLE_CORTEX
	memset(&transarr, 0, sizeof(libundumpi_cbpair) * DUMPI_END_OF_STREAM);
#endif
848 849

	/* handle MPI function calls */	        
850
        callbacks.on_init = handleDUMPIInit;
851 852 853 854 855 856
	callbacks.on_send = (dumpi_send_call)handleDUMPISend;
        callbacks.on_recv = (dumpi_recv_call)handleDUMPIRecv;
        callbacks.on_isend = (dumpi_isend_call)handleDUMPIISend;
        callbacks.on_irecv = (dumpi_irecv_call)handleDUMPIIRecv;
        callbacks.on_allreduce = (dumpi_allreduce_call)handleDUMPIAllreduce;
	callbacks.on_bcast = (dumpi_bcast_call)handleDUMPIBcast;
857 858 859 860 861 862 863 864 865
	callbacks.on_get_count = (dumpi_get_count_call)handleDUMPIIgnore;
	callbacks.on_bsend = (dumpi_bsend_call)handleDUMPIIgnore;
	callbacks.on_ssend = (dumpi_ssend_call)handleDUMPIIgnore;
	callbacks.on_rsend = (dumpi_rsend_call)handleDUMPIIgnore;
	callbacks.on_buffer_attach = (dumpi_buffer_attach_call)handleDUMPIIgnore;
	callbacks.on_buffer_detach = (dumpi_buffer_detach_call)handleDUMPIIgnore;
	callbacks.on_ibsend = (dumpi_ibsend_call)handleDUMPIIgnore;
	callbacks.on_issend = (dumpi_issend_call)handleDUMPIIgnore;
	callbacks.on_irsend = (dumpi_irsend_call)handleDUMPIIgnore;
866
	callbacks.on_wait = (dumpi_wait_call)handleDUMPIWait;
867
	callbacks.on_test = (dumpi_test_call)handleDUMPIIgnore;
868
	callbacks.on_request_free = (dumpi_request_free_call)handleDUMPIReqFree;
869
	callbacks.on_waitany = (dumpi_waitany_call)handleDUMPIWaitany;
870
	callbacks.on_testany = (dumpi_testany_call)handleDUMPIIgnore;
871
	callbacks.on_waitall = (dumpi_waitall_call)handleDUMPIWaitall;
872
	callbacks.on_testall = (dumpi_testall_call)handleDUMPIIgnore;
873
	callbacks.on_waitsome = (dumpi_waitsome_call)handleDUMPIWaitsome;
874 875 876 877 878 879 880 881 882 883 884 885
	callbacks.on_testsome = (dumpi_testsome_call)handleDUMPIIgnore;
	callbacks.on_iprobe = (dumpi_iprobe_call)handleDUMPIIgnore;
	callbacks.on_probe = (dumpi_probe_call)handleDUMPIIgnore;
	callbacks.on_cancel = (dumpi_cancel_call)handleDUMPIIgnore;
	callbacks.on_test_cancelled = (dumpi_test_cancelled_call)handleDUMPIIgnore;
	callbacks.on_send_init = (dumpi_send_init_call)handleDUMPIIgnore;
	callbacks.on_bsend_init = (dumpi_bsend_init_call)handleDUMPIIgnore;
	callbacks.on_ssend_init = (dumpi_ssend_init_call)handleDUMPIIgnore;
	callbacks.on_rsend_init = (dumpi_rsend_init_call)handleDUMPIIgnore;
	callbacks.on_recv_init = (dumpi_recv_init_call)handleDUMPIIgnore;
	callbacks.on_start = (dumpi_start_call)handleDUMPIIgnore;
	callbacks.on_startall = (dumpi_startall_call)handleDUMPIIgnore;
886
	callbacks.on_sendrecv = (dumpi_sendrecv_call)handleDUMPISendrecv;
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
	callbacks.on_sendrecv_replace = (dumpi_sendrecv_replace_call)handleDUMPIIgnore;
	callbacks.on_type_contiguous = (dumpi_type_contiguous_call)handleDUMPIIgnore;
	callbacks.on_barrier = (dumpi_barrier_call)handleDUMPIIgnore;
        callbacks.on_gather = (dumpi_gather_call)handleDUMPIIgnore;
        callbacks.on_gatherv = (dumpi_gatherv_call)handleDUMPIIgnore;
        callbacks.on_scatter = (dumpi_scatter_call)handleDUMPIIgnore;
        callbacks.on_scatterv = (dumpi_scatterv_call)handleDUMPIIgnore;
        callbacks.on_allgather = (dumpi_allgather_call)handleDUMPIIgnore;
        callbacks.on_allgatherv = (dumpi_allgatherv_call)handleDUMPIIgnore;
        callbacks.on_alltoall = (dumpi_alltoall_call)handleDUMPIIgnore;
        callbacks.on_alltoallv = (dumpi_alltoallv_call)handleDUMPIIgnore;
        callbacks.on_alltoallw = (dumpi_alltoallw_call)handleDUMPIIgnore;
        callbacks.on_reduce = (dumpi_reduce_call)handleDUMPIIgnore;
        callbacks.on_reduce_scatter = (dumpi_reduce_scatter_call)handleDUMPIIgnore;
        callbacks.on_group_size = (dumpi_group_size_call)handleDUMPIIgnore;
        callbacks.on_group_rank = (dumpi_group_rank_call)handleDUMPIIgnore;
        callbacks.on_comm_size = (dumpi_comm_size_call)handleDUMPIIgnore;
        callbacks.on_comm_rank = (dumpi_comm_rank_call)handleDUMPIIgnore;
        callbacks.on_comm_get_attr = (dumpi_comm_get_attr_call)handleDUMPIIgnore;
906 907
        callbacks.on_comm_dup = (dumpi_comm_dup_call)handleDUMPIError;
        callbacks.on_comm_create = (dumpi_comm_create_call)handleDUMPIError;
908
        callbacks.on_wtime = (dumpi_wtime_call)handleDUMPIIgnore;
909 910 911 912
        callbacks.on_finalize = (dumpi_finalize_call)handleDUMPIFinalize;

        libundumpi_populate_callbacks(&callbacks, callarr);

913
#ifdef ENABLE_CORTEX
914
#ifdef ENABLE_CORTEX_PYTHON
915 916 917 918 919
	if(dumpi_params->cortex_script[0] != 0) {
		libundumpi_populate_callbacks(CORTEX_PYTHON_TRANSLATION, transarr);
	} else {
		libundumpi_populate_callbacks(CORTEX_MPICH_TRANSLATION, transarr);
	}
920 921 922
#else
	libundumpi_populate_callbacks(CORTEX_MPICH_TRANSLATION, transarr);
#endif
923 924
#endif
        DUMPI_START_STREAM_READ(profile);
925 926
        //dumpi_header* trace_header = undumpi_read_header(profile);
        //dumpi_free_header(trace_header);
927

928
#ifdef ENABLE_CORTEX_PYTHON
929 930 931 932 933 934 935 936 937 938
	if(dumpi_params->cortex_script[0] != 0) {
		if(dumpi_params->cortex_class[0] != 0) {
			cortex_python_set_module(dumpi_params->cortex_script, dumpi_params->cortex_class);
		} else {
			cortex_python_set_module(dumpi_params->cortex_script, NULL);
		}
		if(dumpi_params->cortex_gen[0] != 0) {
			cortex_python_call_generator(profile, dumpi_params->cortex_gen);
		}
	}
939 940
#endif

941 942 943 944 945 946
        int finalize_reached = 0;
        int active = 1;
        int num_calls = 0;
        while(active && !finalize_reached)
        {
           num_calls++;
947
           my_ctx->num_ops++;
948
#ifdef ENABLE_CORTEX
949 950 951 952 953 954 955 956 957 958 959 960
           if(my_ctx->num_ops < max_threshold)
	        active = cortex_undumpi_read_single_call(profile, callarr, transarr, (void*)my_ctx, &finalize_reached);
           else
           {
                struct codes_workload_op op;
                op.op_type = CODES_WK_END;

                op.start_time = my_ctx->last_op_time;
                op.end_time = my_ctx->last_op_time + 1;
                dumpi_insert_next_op(my_ctx->dumpi_mpi_array, &op);
                break;
           }
961
#else
962
           active = undumpi_read_single_call(profile, callarr, (void*)my_ctx, &finalize_reached);
963
#endif
964
        }
965
	UNDUMPI_CLOSE(profile);
966 967
	dumpi_finalize_mpi_op_data(my_ctx->dumpi_mpi_array);
	/* add this rank context to hash table */	
968 969 970 971
        rank_mpi_compare cmp;
        cmp.app = my_ctx->my_app_id;
        cmp.rank = my_ctx->my_rank;
	qhash_add(rank_tbl, &cmp, &(my_ctx->hash_link));
972 973 974 975
	rank_tbl_pop++;

	return 0;
}
976 977 978
/* Data types are for 64-bit archs. Source:
 * https://www.tutorialspoint.com/cprogramming/c_data_types.htm 
 * */
979
static uint64_t get_num_bytes(rank_mpi_context* myctx, dumpi_datatype dt)
980
{
981 982
    (void)myctx;

983 984 985
#ifdef ENABLE_CORTEX
   return cortex_datatype_get_size(myctx->profile,dt);
#endif
986 987 988 989
   switch(dt)
   {
	case DUMPI_DATATYPE_ERROR:
	case DUMPI_DATATYPE_NULL:
990
		tw_error(TW_LOC, "\n data type error");
991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
	break;

	case DUMPI_CHAR:
	case DUMPI_UNSIGNED_CHAR:
	case DUMPI_SIGNED_CHAR:
	case DUMPI_BYTE:
		return 1; /* 1 byte for char */
	break;

	case DUMPI_WCHAR:
		return 4; /* 4 bytes for a 64-bit version */
	break;

	case DUMPI_SHORT:
	case DUMPI_SHORT_INT:
	case DUMPI_UNSIGNED_SHORT:
		return 2;
	break;

	 case DUMPI_INT:
1011 1012 1013
		return 4;
	 break;

1014
	 case DUMPI_UNSIGNED:
1015 1016 1017
     return 4;
     break;

1018 1019
	 case DUMPI_FLOAT:
	 case DUMPI_FLOAT_INT:
1020 1021
        return 4;
     break;
1022 1023

	case DUMPI_DOUBLE:
1024 1025 1026
     return 8;
    break;

1027
	case DUMPI_LONG:
1028 1029 1030
     return 8;
     break;

1031
	case DUMPI_LONG_INT:
1032 1033 1034
     return 8;
     break;

1035
	case DUMPI_UNSIGNED_LONG:
1036 1037 1038
     return 8;
     break;

1039
	case DUMPI_LONG_LONG_INT:
1040 1041 1042
     return 8;
     break;

1043
	case DUMPI_UNSIGNED_LONG_LONG:
1044 1045 1046
     return 8;
     break;

1047
	case DUMPI_LONG_LONG:
1048 1049 1050
     return 8;
     break;

1051 1052 1053 1054 1055
	case DUMPI_DOUBLE_INT:
		return 8;
	break;

	case DUMPI_LONG_DOUBLE_INT:
1056 1057 1058 1059
	case DUMPI_LONG_DOUBLE:
        return 10;
        break;

1060 1061
	default:
	  {
1062
        tw_error(TW_LOC, "\n undefined data type");
1063 1064 1065 1066 1067
		return 0;	
	  }	
   } 
}

1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
void dumpi_trace_nw_workload_get_next_rc2(int app_id, int rank)
{
    rank_mpi_context* temp_data; 
    struct qhash_head *hash_link = NULL;  
    rank_mpi_compare cmp;  
    cmp.rank = rank;
    cmp.app = app_id;

    hash_link = qhash_search(rank_tbl, &cmp);

    assert(hash_link);
    temp_data = qhash_entry(hash_link, rank_mpi_context, hash_link); 
    assert(temp_data);

    dumpi_roll_back_prev_op(temp_data->dumpi_mpi_array);
}
1084
void dumpi_trace_nw_workload_get_next(int app_id, int rank, struct codes_workload_op *op)
1085 1086 1087
{
   rank_mpi_context* temp_data;
   struct qhash_head *hash_link = NULL;
1088 1089 1090 1091
   rank_mpi_compare cmp;
   cmp.rank = rank;
   cmp.app = app_id;
   hash_link = qhash_search(rank_tbl, &cmp);
1092 1093
   if(!hash_link)
   {
yangxuserene's avatar
yangxuserene committed
1094
      printf("\n not found for rank id %d , %d", rank, app_id);
1095 1096 1097 1098 1099 1100
      op->op_type = CODES_WK_END;
      return;
   }
  temp_data = qhash_entry(hash_link, rank_mpi_context, hash_link);
  assert(temp_data);

1101 1102 1103 1104
  struct codes_workload_op mpi_op;
  dumpi_remove_next_op(temp_data->dumpi_mpi_array, &mpi_op, temp_data->last_op_time);
  *op = mpi_op;
  /*if( mpi_op.op_type == CODES_WK_END)
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
  {
	qhash_del(hash_link);
        free(temp_data);

        rank_tbl_pop--;
        if (!rank_tbl_pop)
        {
            qhash_finalize(rank_tbl);
            rank_tbl = NULL;
        }
1115
  }*/
1116 1117 1118 1119 1120 1121 1122
  return;
}

/* implements the codes workload method */
struct codes_workload_method dumpi_trace_workload_method =
{
    .method_name = "dumpi-trace-workload",
1123
    .codes_workload_read_config = NULL,
1124 1125
    .codes_workload_load = dumpi_trace_nw_workload_load,
    .codes_workload_get_next = dumpi_trace_nw_workload_get_next,
1126
    .codes_workload_get_next_rc2 = dumpi_trace_nw_workload_get_next_rc2,
1127
};
Jonathan Jenkins's avatar
Jonathan Jenkins committed
1128 1129 1130 1131 1132

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
1133
 *  indent-tabs-mode: nil
Jonathan Jenkins's avatar
Jonathan Jenkins committed
1134 1135 1136 1137
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */