darshan-dxt.c 24.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37 38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40 41 42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44 45 46
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
47 48 49
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
50
 * associate different types of handles with this dxt_file_record_ref struct.
51 52 53
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
54
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
55 56
 * there could be multiple open file descriptors that index it.
 */
57
struct dxt_file_record_ref
58
{
59
    struct dxt_file_record *file_rec;
60

61 62
    int64_t write_available_buf;
    int64_t read_available_buf;
63 64 65

    segment_info *write_traces;
    segment_info *read_traces;
66 67
};

68 69
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
70 71
 * shutdown time.
 */
72
struct dxt_posix_runtime
73 74 75
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
76 77
    char *record_buf;
    int record_buf_size;
78 79
};

80
struct dxt_mpiio_runtime
81 82 83
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
84 85
    char *record_buf;
    int record_buf_size;
86 87
};

88 89 90 91 92
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
93 94 95 96
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
97 98 99 100 101 102

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
103
    void);
104
static void dxt_mpiio_runtime_initialize(
105
    void);
106 107
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
108 109
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
110
static void dxt_posix_cleanup_runtime(
111
    void);
112
static void dxt_mpiio_cleanup_runtime(
113 114
    void);

115
static void dxt_posix_shutdown(
116
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
117 118
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
119
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
120
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
121

122 123
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
124
static pthread_mutex_t dxt_runtime_mutex =
125 126 127 128 129 130
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;
131
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
132

133 134
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
135 136 137


/**********************************************************
138
 *      Wrappers for DXT I/O functions of interest      *
139 140
 **********************************************************/

141
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
142
{
143
    struct dxt_file_record *file_rec = rec_ref->file_rec;
144

145
    int write_count = file_rec->write_count;
146
    int write_available_buf = rec_ref->write_available_buf;
147 148

    if (write_count >= write_available_buf) {
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
165 166
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
167 168 169 170
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
171
    }
172 173 174 175 176
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
177 178

    int read_count = file_rec->read_count;
179
    int read_available_buf = rec_ref->read_available_buf;
180 181

    if (read_count >= read_available_buf) {
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
198 199
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
200 201 202 203
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
204 205 206
    }
}

207 208
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
209
{
210 211
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
212

213 214 215 216 217 218
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
219 220
    }

221 222 223 224 225 226 227
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
228 229
    }

230 231 232 233 234
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

235 236 237 238
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
239 240 241
    file_rec->write_count += 1;
}

242 243
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
244
{
245 246
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
247

248 249 250 251 252 253 254 255 256 257
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
258
    if (!rec_ref) {
259 260 261
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
262 263 264
    }

    file_rec = rec_ref->file_rec;
265 266 267
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
268

269 270 271 272
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
273 274 275
    file_rec->read_count += 1;
}

276
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
277 278
        double start_time, double end_time)
{
279 280
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
281

282 283 284 285 286 287
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
288 289
    }

290 291 292 293 294 295 296
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
297 298
    }

299 300 301 302 303
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

304 305 306
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
307 308 309
    file_rec->write_count += 1;
}

310
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
311 312
        double start_time, double end_time)
{
313 314
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
315

316 317 318 319 320 321
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
322 323
    }

324 325 326 327 328 329 330
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
331 332
    }

333 334 335 336 337
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */

338 339 340
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
341 342 343 344 345
    file_rec->read_count += 1;
}


/**********************************************************
346
 * Internal functions for manipulating DXT module state *
347 348
 **********************************************************/

349
/* initialize internal DXT module data structures and register with darshan-core */
350
static void dxt_posix_runtime_initialize()
351
{
352 353 354 355 356
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
357

358
    /* register the DXT module with darshan core */
359
    darshan_core_register_module(
360 361
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
362
        &dxt_psx_buf_size,
363 364 365
        &posix_my_rank,
        &darshan_mem_alignment);

366 367
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
368
    {
369
        darshan_core_unregister_module(DXT_POSIX_MOD);
370 371 372
        return;
    }

373 374
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
375
    {
376
        darshan_core_unregister_module(DXT_POSIX_MOD);
377 378
        return;
    }
379
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
380 381 382 383

    return;
}

384
void dxt_mpiio_runtime_initialize()
385
{
386 387 388 389 390
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
391

392
    /* register the DXT module with darshan core */
393
    darshan_core_register_module(
394 395
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
396
        &dxt_mpiio_buf_size,
397 398 399
        &mpiio_my_rank,
        &darshan_mem_alignment);

400 401
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
402
    {
403
        darshan_core_unregister_module(DXT_MPIIO_MOD);
404 405 406
        return;
    }

407 408
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
409
    {
410
        darshan_core_unregister_module(DXT_MPIIO_MOD);
411 412
        return;
    }
413
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
414 415 416 417

    return;
}

418 419
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
420
{
421 422
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
423 424
    int ret;

425 426 427 428 429 430 431 432
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

433 434
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
435 436 437 438
    {
        DXT_UNLOCK();
        return(NULL);
    }
439 440
    memset(rec_ref, 0, sizeof(*rec_ref));

441 442 443 444 445 446 447 448 449
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

450
    /* add a reference to this file record based on record id */
451
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
452 453 454
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
455
        free(file_rec);
456
        free(rec_ref);
457 458
        DXT_UNLOCK();
        return(NULL);
459 460
    }

461 462
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
463

464
    /* initialize record and record reference fields */
465 466
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;
Cong Xu's avatar
Cong Xu committed
467
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
468 469

    rec_ref->file_rec = file_rec;
470
    dxt_posix_runtime->file_rec_count++;
471 472

    return(rec_ref);
473 474
}

475 476
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
477
{
478 479
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
480 481
    int ret;

482 483 484 485 486 487 488 489
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

490 491
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
492 493 494 495
    {
        DXT_UNLOCK();
        return(NULL);
    }
496 497
    memset(rec_ref, 0, sizeof(*rec_ref));

498 499 500 501 502 503 504 505 506
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

507
    /* add a reference to this file record based on record id */
508
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
509 510 511
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
512
        free(file_rec);
513
        free(rec_ref);
514
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
515
        return(NULL);
516 517
    }

518 519
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
520

521
    /* initialize record and record reference fields */
522 523
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;
Cong Xu's avatar
Cong Xu committed
524
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
525 526

    rec_ref->file_rec = file_rec;
527
    dxt_mpiio_runtime->file_rec_count++;
528

529
    return(rec_ref);
530 531
}

Shane Snyder's avatar
Shane Snyder committed
532
static void dxt_free_record_data(void *rec_ref_p)
533
{
534
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
535

536
    /* TODO: update these pointer addresses once {write/read}_traces are moved to rec_ref structure */
537 538
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
539
    free(dxt_rec_ref->file_rec);
540 541
}

542
static void dxt_posix_cleanup_runtime()
543
{
Shane Snyder's avatar
Shane Snyder committed
544
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
545
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
546

547 548
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
549 550 551 552

    return;
}

553
static void dxt_mpiio_cleanup_runtime()
554
{
Shane Snyder's avatar
Shane Snyder committed
555
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
556
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
557

558 559
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
560 561 562 563 564 565 566 567 568

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
569
static void dxt_serialize_posix_records(void *rec_ref_p)
570
{
Shane Snyder's avatar
Shane Snyder committed
571
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
572
    struct dxt_file_record *file_rec;
573 574 575
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
576
    void *tmp_buf_ptr;
577

Shane Snyder's avatar
Shane Snyder committed
578 579 580
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
581

Shane Snyder's avatar
Shane Snyder committed
582 583 584 585
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
586

Shane Snyder's avatar
Shane Snyder committed
587 588
    /*
     * Buffer format:
589
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
590 591 592
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
593

Shane Snyder's avatar
Shane Snyder committed
594 595
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
596

Shane Snyder's avatar
Shane Snyder committed
597 598 599
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
600

Shane Snyder's avatar
Shane Snyder committed
601
    /*Copy write record */
602
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
603 604 605
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
606

Shane Snyder's avatar
Shane Snyder committed
607
    /*Copy read record */
608
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
609 610 611
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
612

Shane Snyder's avatar
Shane Snyder committed
613
    dxt_posix_runtime->record_buf_size += record_size;
614

Shane Snyder's avatar
Shane Snyder committed
615 616
#if 0
    int i;
Cong Xu's avatar
Cong Xu committed
617 618
    int64_t rank;
    char *hostname;
Shane Snyder's avatar
Shane Snyder committed
619 620 621 622
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
623

Cong Xu's avatar
Cong Xu committed
624 625 626 627 628
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
Shane Snyder's avatar
Shane Snyder committed
629
                file_rec->write_count, file_rec->read_count);
Cong Xu's avatar
Cong Xu committed
630
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
631

Shane Snyder's avatar
Shane Snyder committed
632
    for (i = 0; i < file_rec->write_count; i++) {
633 634 635 636
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
637

Cong Xu's avatar
Cong Xu committed
638
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
639
    }
640

Shane Snyder's avatar
Shane Snyder committed
641
    for (i = 0; i < file_rec->read_count; i++) {
642 643 644 645
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
646

Cong Xu's avatar
Cong Xu committed
647
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
648 649 650
    }
#endif
}
651

Shane Snyder's avatar
Shane Snyder committed
652 653 654 655 656 657 658 659
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
660

Shane Snyder's avatar
Shane Snyder committed
661 662 663 664 665 666 667 668 669 670 671 672 673 674
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
675 676

    /* shutdown internal structures used for instrumenting */
677
    dxt_posix_cleanup_runtime();
678 679 680 681 682 683 684

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

Shane Snyder's avatar
Shane Snyder committed
685
static void dxt_serialize_mpiio_records(void *rec_ref_p)
686
{
Shane Snyder's avatar
Shane Snyder committed
687
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
688
    struct dxt_file_record *file_rec;
689 690 691
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
692
    void *tmp_buf_ptr;
693

Shane Snyder's avatar
Shane Snyder committed
694 695 696
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
697

Shane Snyder's avatar
Shane Snyder committed
698 699 700 701
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
702

Shane Snyder's avatar
Shane Snyder committed
703 704
    /*
     * Buffer format:
705
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
706 707 708
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
709

Shane Snyder's avatar
Shane Snyder committed
710 711
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
712

Shane Snyder's avatar
Shane Snyder committed
713 714 715
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
716

Shane Snyder's avatar
Shane Snyder committed
717
    /*Copy write record */
718
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
719 720
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
721 722
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
723
    /*Copy read record */
724
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
725 726
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
727 728
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
729
    dxt_mpiio_runtime->record_buf_size += record_size;
730 731

#if 0
Shane Snyder's avatar
Shane Snyder committed
732 733
    int i;
    int64_t rank;
Cong Xu's avatar
Cong Xu committed
734 735
    char *hostname;
    int64_t length;
Shane Snyder's avatar
Shane Snyder committed
736 737
    double start_time;
    double end_time;
738

Cong Xu's avatar
Cong Xu committed
739 740 741 742 743 744 745
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
746

Shane Snyder's avatar
Shane Snyder committed
747
    for (i = 0; i < file_rec->write_count; i++) {
748 749 750
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
751

Cong Xu's avatar
Cong Xu committed
752
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
753 754
    }

Shane Snyder's avatar
Shane Snyder committed
755
    for (i = 0; i < file_rec->read_count; i++) {
756 757 758
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
759

Cong Xu's avatar
Cong Xu committed
760
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

789
    /* shutdown internal structures used for instrumenting */
790
    dxt_mpiio_cleanup_runtime();
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */