darshan-dxt.c 25.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37 38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40 41 42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44
struct dxt_record_ref_tracker
45 46 47 48 49
{
    void *rec_ref_p;
    UT_hash_handle hlink;
};

50 51 52
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
53 54 55
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
56
 * associate different types of handles with this dxt_file_record_ref struct.
57 58 59
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
60
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
61 62
 * there could be multiple open file descriptors that index it.
 */
63
struct dxt_file_record_ref
64
{
65
    struct dxt_file_record *file_rec;
66 67
    int64_t write_available_buf;
    int64_t read_available_buf;
68 69 70
    int fs_type; /* same as darshan_fs_info->fs_type */
};

71 72
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
73 74
 * shutdown time.
 */
75
struct dxt_posix_runtime
76 77 78 79 80
{
    void *rec_id_hash;
    int file_rec_count;
};

81
struct dxt_mpiio_runtime
82 83 84 85 86 87
{
    void *rec_id_hash;
    void *fh_hash;
    int file_rec_count;
};

88 89 90 91 92 93 94 95 96 97 98
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
99
    void);
100
void dxt_mpiio_runtime_initialize(
101
    void);
102 103
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
104
void dxt_mpiio_track_new_file_record(
105
    darshan_record_id rec_id, const char *path);
106 107
void dxt_mpiio_add_record_ref(darshan_record_id rec_id, MPI_File fh);
static void dxt_posix_cleanup_runtime(
108
    void);
109
static void dxt_mpiio_cleanup_runtime(
110 111
    void);

112
static void dxt_posix_shutdown(
113
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
114 115
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
116
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
117
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
118

119 120
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
121
static pthread_mutex_t dxt_runtime_mutex =
122 123 124 125 126 127
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;
128
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
129

130 131
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
132 133 134


/**********************************************************
135
 *      Wrappers for DXT I/O functions of interest      *
136 137
 **********************************************************/

138
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
139
{
140
    struct dxt_file_record *file_rec = rec_ref->file_rec;
141

142
    int write_count = file_rec->write_count;
143
    int write_available_buf = rec_ref->write_available_buf;
144 145

    if (write_count >= write_available_buf) {
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
            file_rec->write_traces =
                (segment_info *)realloc(file_rec->write_traces,
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
168
    }
169 170 171 172 173
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
174 175

    int read_count = file_rec->read_count;
176
    int read_available_buf = rec_ref->read_available_buf;
177 178

    if (read_count >= read_available_buf) {
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
            file_rec->read_traces =
                (segment_info *)realloc(file_rec->read_traces,
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
201 202 203
    }
}

204 205
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
206
{
207 208
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
209

210 211 212 213 214 215
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
216 217
    }

218 219 220 221 222 223 224
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
225 226
    }

227 228 229 230 231
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

232 233 234 235 236 237 238
    file_rec->write_traces[file_rec->write_count].offset = offset;
    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

239 240
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
241
{
242 243
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
244

245 246 247 248 249 250 251 252 253 254
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
255
    if (!rec_ref) {
256 257 258
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
259 260 261
    }

    file_rec = rec_ref->file_rec;
262 263 264
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
265 266 267 268 269 270 271 272

    file_rec->read_traces[file_rec->read_count].offset = offset;
    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}

273
void dxt_mpiio_write(MPI_File fh, int64_t length,
274 275
        double start_time, double end_time)
{
276 277
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
278

279
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->fh_hash,
280 281
                &fh, sizeof(MPI_File));
    if (!rec_ref) {
282
        fprintf(stderr, "Error: dxt_mpiio_write unable to find rec_ref.\n");
283 284 285 286
        return;
    }

    file_rec = rec_ref->file_rec;
287
    if (dxt_mpiio_runtime) {
288
        check_wr_trace_buf(rec_ref);
289 290 291 292 293 294 295 296
    }

    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

297
void dxt_mpiio_read(MPI_File fh, int64_t length,
298 299
        double start_time, double end_time)
{
300 301
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
302

303
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->fh_hash,
304 305
                &fh, sizeof(MPI_File));
    if (!rec_ref) {
306
        fprintf(stderr, "Error: dxt_mpiio_read unable to find rec_ref.\n");
307 308 309 310
        return;
    }

    file_rec = rec_ref->file_rec;
311
    if (dxt_mpiio_runtime) {
312
        check_rd_trace_buf(rec_ref);
313 314 315 316 317 318 319 320 321 322
    }

    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}


/**********************************************************
323
 * Internal functions for manipulating DXT module state *
324 325
 **********************************************************/

326
/* initialize internal DXT module data structures and register with darshan-core */
327
static void dxt_posix_runtime_initialize()
328
{
329 330 331 332 333
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
334

335
    /* register the DXT module with darshan core */
336
    darshan_core_register_module(
337 338
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
339
        &dxt_psx_buf_size,
340 341 342
        &posix_my_rank,
        &darshan_mem_alignment);

343 344
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
345
    {
346
        darshan_core_unregister_module(DXT_POSIX_MOD);
347 348 349
        return;
    }

350 351
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
352
    {
353
        darshan_core_unregister_module(DXT_POSIX_MOD);
354 355
        return;
    }
356
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
357 358 359 360

    return;
}

361
void dxt_mpiio_runtime_initialize()
362
{
363 364 365 366 367
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
368

369
    /* register the DXT module with darshan core */
370
    darshan_core_register_module(
371 372
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
373
        &dxt_mpiio_buf_size,
374 375 376
        &mpiio_my_rank,
        &darshan_mem_alignment);

377 378
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
379
    {
380
        darshan_core_unregister_module(DXT_MPIIO_MOD);
381 382 383
        return;
    }

384 385
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
386
    {
387
        darshan_core_unregister_module(DXT_MPIIO_MOD);
388 389
        return;
    }
390
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
391 392 393 394

    return;
}

395 396
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
397
{
398 399
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
400 401
    int ret;

402 403 404 405 406 407 408 409
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

410 411
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
412 413 414 415
    {
        DXT_UNLOCK();
        return(NULL);
    }
416 417
    memset(rec_ref, 0, sizeof(*rec_ref));

418 419 420 421 422 423 424 425 426
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

427
    /* add a reference to this file record based on record id */
428
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
429 430 431
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
432
        free(file_rec);
433
        free(rec_ref);
434 435
        DXT_UNLOCK();
        return(NULL);
436 437
    }

438 439
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
440

441
    /* initialize record and record reference fields */
442 443 444 445
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;

    rec_ref->file_rec = file_rec;
446
    dxt_posix_runtime->file_rec_count++;
447 448

    return(rec_ref);
449 450
}

451
void dxt_mpiio_track_new_file_record(
452 453
    darshan_record_id rec_id, const char *path)
{
454 455
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
456 457 458 459 460 461 462 463 464
    struct darshan_fs_info fs_info;
    int ret;

    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
        return;
    memset(rec_ref, 0, sizeof(*rec_ref));

    /* add a reference to this file record based on record id */
465
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
466 467 468 469 470 471 472 473 474 475 476 477 478
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
        free(rec_ref);
        return;
    }

    /* register the actual file record with darshan-core so it is persisted
     * in the log file
     */
    file_rec = darshan_core_register_record(
            rec_id,
            path,
479 480
            DXT_MPIIO_MOD,
            sizeof(struct dxt_file_record),
481 482 483 484
            &fs_info);

    if(!file_rec)
    {
485
        darshan_delete_record_ref(&(dxt_mpiio_runtime->rec_id_hash),
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
                &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return;
    }

    /* registering this file record was successful, so initialize
     * some fields
     */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;

    file_rec->write_count = 0;
    file_rec->write_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    file_rec->read_count = 0;
    file_rec->read_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    rec_ref->file_rec = file_rec;
506 507
    rec_ref->write_available_buf = IO_TRACE_BUF_SIZE;
    rec_ref->read_available_buf = IO_TRACE_BUF_SIZE;
508 509
    rec_ref->fs_type = fs_info.fs_type;

510
    dxt_mpiio_runtime->file_rec_count++;
511 512
}

513
void dxt_mpiio_add_record_ref(darshan_record_id rec_id, MPI_File fh)
514
{
515
    struct dxt_file_record_ref *rec_ref = NULL;
516

517
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash, &rec_id,
518 519 520
                sizeof(darshan_record_id));
    assert(rec_ref);

521
    darshan_add_record_ref(&(dxt_mpiio_runtime->fh_hash), &fh,
522 523 524
            sizeof(MPI_File), rec_ref);
}

525
void dxt_clear_record_refs(void **hash_head_p, int free_flag)
526
{
527 528 529 530 531
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        *(struct dxt_record_ref_tracker **)hash_head_p;
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
532

533
#if 0    
534 535 536 537 538
    /* iterate the hash table and remove/free all reference trackers */
    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp)
    {
        HASH_DELETE(hlink, ref_tracker_head, ref_tracker);
        if (free_flag) {
539
            rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
540 541 542 543 544 545 546 547 548 549 550 551 552
            file_rec = rec_ref->file_rec;

            if (file_rec->write_traces)
                free(file_rec->write_traces);

            if (file_rec->read_traces)
                free(file_rec->read_traces);

            free(rec_ref);
        }
        free(ref_tracker);
    }
    *hash_head_p = ref_tracker_head;
553
#endif
554 555 556 557

    return;
}

558
static void dxt_posix_cleanup_runtime()
559
{
560
    dxt_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
561

562 563
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
564 565 566 567

    return;
}

568
static void dxt_mpiio_cleanup_runtime()
569
{
570 571
    dxt_clear_record_refs(&(dxt_mpiio_runtime->fh_hash), 0);
    dxt_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
572

573 574
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
575 576 577 578 579 580 581 582 583

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

584
static void dxt_posix_shutdown(
585 586 587
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
588 589
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
590
{
591 592
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
593 594 595 596 597 598 599 600 601 602 603
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
604
    void *tmp_buf_ptr = *dxt_posix_buf;
605

606
    assert(dxt_posix_runtime);
607

608 609 610 611
    int dxt_rec_count = dxt_posix_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_posix_runtime->rec_id_hash);
612

613
    *dxt_posix_buf_sz = 0;
614 615

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
616
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
617 618 619 620 621 622 623 624 625 626 627 628
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;

        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
629
         * dxt_file_record + ost_ids + write_traces + read_traces
630
         */
631
        record_size = sizeof(struct dxt_file_record) +
632 633
                (record_write_count + record_read_count) * sizeof(segment_info);

634 635
        if (*dxt_posix_buf_sz == 0) {
            *dxt_posix_buf = (void *)malloc(record_size);
636
        } else {
637 638
            *dxt_posix_buf = (void *)realloc((*dxt_posix_buf),
                            *dxt_posix_buf_sz + record_size);
639
        }
640
        tmp_buf_ptr = (void *)(*dxt_posix_buf) + *dxt_posix_buf_sz;
641

642 643 644
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
645 646 647 648 649 650 651 652 653 654 655 656 657

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

658
        *dxt_posix_buf_sz += record_size;
659 660

#if 0
661 662
        printf("DXT, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
        printf("DXT, write_count is: %d read_count is: %d\n",
663
                    file_rec->write_count, file_rec->read_count);
664 665 666 667 668 669 670 671

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

672
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
673 674 675 676 677 678 679 680 681
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

682
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
683 684 685 686 687
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
688
    dxt_posix_cleanup_runtime();
689 690 691 692 693 694 695

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

696
static void dxt_mpiio_shutdown(
697 698 699
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
700 701
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
702
{
703 704
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
705 706 707 708 709 710 711 712 713 714 715
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
716
    void *tmp_buf_ptr = *dxt_mpiio_buf;
717

718
    assert(dxt_mpiio_runtime);
719

720 721 722 723
    int dxt_rec_count = dxt_mpiio_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_mpiio_runtime->rec_id_hash);
724

725
    *dxt_mpiio_buf_sz = 0;
726 727

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
728
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
729 730 731 732 733 734 735 736 737 738 739
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;
        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
740
         * dxt_file_record + ost_ids + write_traces + read_traces
741
         */
742
        record_size = sizeof(struct dxt_file_record) +
743 744
                (record_write_count + record_read_count) * sizeof(segment_info);

745 746
        if (*dxt_mpiio_buf_sz == 0) {
            *dxt_mpiio_buf = (void *)malloc(record_size);
747
        } else {
748 749
            *dxt_mpiio_buf = (void *)realloc((*dxt_mpiio_buf),
                            *dxt_mpiio_buf_sz + record_size);
750
        }
751
        tmp_buf_ptr = (void *)(*dxt_mpiio_buf) + *dxt_mpiio_buf_sz;
752

753 754 755
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
756 757 758 759 760 761 762 763 764 765 766 767 768

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

769
        *dxt_mpiio_buf_sz += record_size;
770 771 772

#if 0
        printf("Cong, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
773
        printf("DXT, file_rec->write_count is: %d\n",
774 775 776 777 778 779 780 781 782
                    file_rec->write_count);

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

783
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
784 785 786 787 788 789 790 791 792
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

793
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
794 795 796 797 798
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
799
    dxt_mpiio_cleanup_runtime();
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */