darshan-dxt.c 24.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

#ifdef DARSHAN_LUSTRE
#include <lustre/lustre_user.h>
#endif

#define IO_TRACE_BUF_SIZE       1024

43
struct dxt_record_ref_tracker
44 45 46 47 48
{
    void *rec_ref_p;
    UT_hash_handle hlink;
};

49 50 51
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
52 53 54
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
55
 * associate different types of handles with this dxt_file_record_ref struct.
56 57 58
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
59
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
60 61
 * there could be multiple open file descriptors that index it.
 */
62
struct dxt_file_record_ref
63
{
64
    struct dxt_file_record *file_rec;
65 66
    int64_t write_available_buf;
    int64_t read_available_buf;
67 68 69
    int fs_type; /* same as darshan_fs_info->fs_type */
};

70 71
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
72 73
 * shutdown time.
 */
74
struct dxt_posix_runtime
75 76 77 78 79 80
{
    void *rec_id_hash;
    void *fd_hash;
    int file_rec_count;
};

81
struct dxt_mpiio_runtime
82 83 84 85 86 87
{
    void *rec_id_hash;
    void *fh_hash;
    int file_rec_count;
};

88
void dxt_posix_runtime_initialize(
89
    void);
90
void dxt_mpiio_runtime_initialize(
91
    void);
92
void dxt_posix_track_new_file_record(
93
    darshan_record_id rec_id, const char *path);
94
void dxt_mpiio_track_new_file_record(
95
    darshan_record_id rec_id, const char *path);
96 97
void dxt_posix_add_record_ref(darshan_record_id rec_id, int fd);
void dxt_mpiio_add_record_ref(darshan_record_id rec_id, MPI_File fh);
98
#if 0
99 100
static void dxt_instrument_fs_data(
    darshan_record_id rec_id, int fs_type, struct dxt_file_record *file_rec);
101
#endif
102
static void dxt_posix_cleanup_runtime(
103
    void);
104
static void dxt_mpiio_cleanup_runtime(
105 106
    void);

107
static void dxt_posix_shutdown(
108
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
109 110
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
111
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
112
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
113 114 115 116 117 118


#ifdef DARSHAN_LUSTRE
/* XXX modules don't expose an API for other modules, so use extern to get
 * Lustre instrumentation function
 */
119 120
extern void dxt_get_lustre_stripe_info(
    darshan_record_id rec_id, struct dxt_file_record *file_rec);
121 122
#endif

123 124 125
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
static pthread_mutex_t dxt_posix_runtime_mutex =
126
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
127
static pthread_mutex_t dxt_mpiio_runtime_mutex =
128 129 130 131 132 133 134
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;

135 136
#define DXT_POSIX_LOCK() pthread_mutex_lock(&dxt_posix_runtime_mutex)
#define DXT_POSIX_UNLOCK() pthread_mutex_unlock(&dxt_posix_runtime_mutex)
137

138 139
#define DXT_MPIIO_LOCK() pthread_mutex_lock(&dxt_mpiio_runtime_mutex)
#define DXT_MPIIO_UNLOCK() pthread_mutex_unlock(&dxt_mpiio_runtime_mutex)
140 141 142


/**********************************************************
143
 *      Wrappers for DXT I/O functions of interest      *
144 145
 **********************************************************/

146
void check_io_trace_buf(struct dxt_file_record_ref *rec_ref)
147
{
148
    struct dxt_file_record *file_rec = rec_ref->file_rec;
149

150
    int write_count = file_rec->write_count;
151
    int write_available_buf = rec_ref->write_available_buf;
152 153 154 155 156 157 158 159

    if (write_count >= write_available_buf) {
        write_available_buf += IO_TRACE_BUF_SIZE;

        file_rec->write_traces =
            (segment_info *)realloc(file_rec->write_traces,
                    write_available_buf * sizeof(segment_info));

160
        rec_ref->write_available_buf = write_available_buf;
161 162 163
    }

    int read_count = file_rec->read_count;
164
    int read_available_buf = rec_ref->read_available_buf;
165 166 167 168 169 170 171 172

    if (read_count >= read_available_buf) {
        read_available_buf += IO_TRACE_BUF_SIZE;

        file_rec->read_traces =
            (segment_info *)realloc(file_rec->read_traces,
                    read_available_buf * sizeof(segment_info));

173
        rec_ref->read_available_buf = read_available_buf;
174 175 176
    }
}

177
void dxt_posix_write(int fd, int64_t offset, int64_t length,
178 179
        double start_time, double end_time)
{
180 181
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
182

183
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->fd_hash,
184 185
                &fd, sizeof(int));
    if (!rec_ref) {
186
        fprintf(stderr, "Error: dxt_posix_write unable to find rec_ref.\n");
187 188 189 190
        return;
    }

    file_rec = rec_ref->file_rec;
191
    if (dxt_posix_runtime) {
192
        check_io_trace_buf(rec_ref);
193 194 195 196 197 198 199 200 201
    }

    file_rec->write_traces[file_rec->write_count].offset = offset;
    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

202
void dxt_posix_read(int fd, int64_t offset, int64_t length,
203 204
        double start_time, double end_time)
{
205 206
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
207

208
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->fd_hash,
209 210
                &fd, sizeof(int));
    if (!rec_ref) {
211
        fprintf(stderr, "Error: dxt_posix_read unable to find rec_ref.\n");
212 213 214 215
        return;
    }

    file_rec = rec_ref->file_rec;
216
    if (dxt_posix_runtime) {
217
        check_io_trace_buf(rec_ref);
218 219 220 221 222 223 224 225 226
    }

    file_rec->read_traces[file_rec->read_count].offset = offset;
    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}

227
void dxt_mpiio_write(MPI_File fh, int64_t length,
228 229
        double start_time, double end_time)
{
230 231
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
232

233
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->fh_hash,
234 235
                &fh, sizeof(MPI_File));
    if (!rec_ref) {
236
        fprintf(stderr, "Error: dxt_mpiio_write unable to find rec_ref.\n");
237 238 239 240
        return;
    }

    file_rec = rec_ref->file_rec;
241
    if (dxt_mpiio_runtime) {
242
        check_io_trace_buf(rec_ref);
243 244 245 246 247 248 249 250
    }

    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

251
void dxt_mpiio_read(MPI_File fh, int64_t length,
252 253
        double start_time, double end_time)
{
254 255
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
256

257
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->fh_hash,
258 259
                &fh, sizeof(MPI_File));
    if (!rec_ref) {
260
        fprintf(stderr, "Error: dxt_mpiio_read unable to find rec_ref.\n");
261 262 263 264
        return;
    }

    file_rec = rec_ref->file_rec;
265
    if (dxt_mpiio_runtime) {
266
        check_io_trace_buf(rec_ref);
267 268 269 270 271 272 273 274 275 276
    }

    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}


/**********************************************************
277
 * Internal functions for manipulating DXT module state *
278 279
 **********************************************************/

280 281
/* initialize internal DXT module data structures and register with darshan-core */
void dxt_posix_runtime_initialize()
282
{
283 284 285 286 287
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
288

289
    /* register the DXT module with darshan core */
290
    darshan_core_register_module(
291 292
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
293
        &dxt_psx_buf_size,
294 295 296
        &posix_my_rank,
        &darshan_mem_alignment);

297 298
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
299
    {
300
        darshan_core_unregister_module(DXT_POSIX_MOD);
301 302 303
        return;
    }

304 305
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
306
    {
307
        darshan_core_unregister_module(DXT_POSIX_MOD);
308 309
        return;
    }
310
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
311 312 313 314

    return;
}

315
void dxt_mpiio_runtime_initialize()
316
{
317 318 319 320 321
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
322

323
    /* register the DXT module with darshan core */
324
    darshan_core_register_module(
325 326
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
327
        &dxt_mpiio_buf_size,
328 329 330
        &mpiio_my_rank,
        &darshan_mem_alignment);

331 332
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
333
    {
334
        darshan_core_unregister_module(DXT_MPIIO_MOD);
335 336 337
        return;
    }

338 339
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
340
    {
341
        darshan_core_unregister_module(DXT_MPIIO_MOD);
342 343
        return;
    }
344
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
345 346 347 348

    return;
}

349
void dxt_posix_track_new_file_record(
350 351
    darshan_record_id rec_id, const char *path)
{
352 353
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
354 355 356 357 358 359 360 361 362
    struct darshan_fs_info fs_info;
    int ret;

    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
        return;
    memset(rec_ref, 0, sizeof(*rec_ref));

    /* add a reference to this file record based on record id */
363
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
364 365 366 367 368 369 370 371 372 373 374 375 376
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
        free(rec_ref);
        return;
    }

    /* register the actual file record with darshan-core so it is persisted
     * in the log file
     */
    file_rec = darshan_core_register_record(
            rec_id,
            path,
377 378
            DXT_POSIX_MOD,
            sizeof(struct dxt_file_record),
379 380 381 382
            &fs_info);

    if(!file_rec)
    {
383
        darshan_delete_record_ref(&(dxt_posix_runtime->rec_id_hash),
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
                &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return;
    }

    /*
     * Registering this file record was successful, so initialize
     * some fields */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;

    file_rec->write_count = 0;
    file_rec->write_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    file_rec->read_count = 0;
    file_rec->read_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    rec_ref->file_rec = file_rec;
404 405
    rec_ref->write_available_buf = IO_TRACE_BUF_SIZE;
    rec_ref->read_available_buf = IO_TRACE_BUF_SIZE;
406 407
    rec_ref->fs_type = fs_info.fs_type;

408
    dxt_posix_runtime->file_rec_count++;
409 410
}

411
void dxt_mpiio_track_new_file_record(
412 413
    darshan_record_id rec_id, const char *path)
{
414 415
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
416 417 418 419 420 421 422 423 424
    struct darshan_fs_info fs_info;
    int ret;

    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
        return;
    memset(rec_ref, 0, sizeof(*rec_ref));

    /* add a reference to this file record based on record id */
425
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
426 427 428 429 430 431 432 433 434 435 436 437 438
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
        free(rec_ref);
        return;
    }

    /* register the actual file record with darshan-core so it is persisted
     * in the log file
     */
    file_rec = darshan_core_register_record(
            rec_id,
            path,
439 440
            DXT_MPIIO_MOD,
            sizeof(struct dxt_file_record),
441 442 443 444
            &fs_info);

    if(!file_rec)
    {
445
        darshan_delete_record_ref(&(dxt_mpiio_runtime->rec_id_hash),
446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
                &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return;
    }

    /* registering this file record was successful, so initialize
     * some fields
     */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;

    file_rec->write_count = 0;
    file_rec->write_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    file_rec->read_count = 0;
    file_rec->read_traces = malloc(IO_TRACE_BUF_SIZE *
            sizeof(segment_info));

    rec_ref->file_rec = file_rec;
466 467
    rec_ref->write_available_buf = IO_TRACE_BUF_SIZE;
    rec_ref->read_available_buf = IO_TRACE_BUF_SIZE;
468 469
    rec_ref->fs_type = fs_info.fs_type;

470
    dxt_mpiio_runtime->file_rec_count++;
471 472
}

473
void dxt_posix_add_record_ref(darshan_record_id rec_id, int fd)
474
{
475 476
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec;
477 478
    int i;

479
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash, &rec_id,
480 481 482
                sizeof(darshan_record_id));
    assert(rec_ref);

483
    darshan_add_record_ref(&(dxt_posix_runtime->fd_hash), &fd,
484 485
            sizeof(int), rec_ref);

486
#if 0
487 488
    /* get Lustre stripe information */
    file_rec = rec_ref->file_rec;
489
    dxt_instrument_fs_data(rec_id, rec_ref->fs_type, file_rec);
490
#endif
491 492
}

493
void dxt_mpiio_add_record_ref(darshan_record_id rec_id, MPI_File fh)
494
{
495
    struct dxt_file_record_ref *rec_ref = NULL;
496

497
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash, &rec_id,
498 499 500
                sizeof(darshan_record_id));
    assert(rec_ref);

501
    darshan_add_record_ref(&(dxt_mpiio_runtime->fh_hash), &fh,
502 503 504
            sizeof(MPI_File), rec_ref);
}

505
#if 0
506 507
static void dxt_instrument_fs_data(
        darshan_record_id rec_id, int fs_type, struct dxt_file_record *file_rec)
508 509 510 511 512
{
#ifdef DARSHAN_LUSTRE
    /* allow lustre to generate a record if we configured with lustre support */
    if(fs_type == LL_SUPER_MAGIC)
    {
513
        dxt_get_lustre_stripe_info(rec_id, file_rec);
514 515 516 517 518
        return;
    }
#endif
    return;
}
519
#endif
520

521
void dxt_clear_record_refs(void **hash_head_p, int free_flag)
522
{
523 524 525 526 527
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        *(struct dxt_record_ref_tracker **)hash_head_p;
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
528

529
#if 0    
530 531 532 533 534
    /* iterate the hash table and remove/free all reference trackers */
    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp)
    {
        HASH_DELETE(hlink, ref_tracker_head, ref_tracker);
        if (free_flag) {
535
            rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
536 537 538 539 540 541 542 543 544 545 546 547 548
            file_rec = rec_ref->file_rec;

            if (file_rec->write_traces)
                free(file_rec->write_traces);

            if (file_rec->read_traces)
                free(file_rec->read_traces);

            free(rec_ref);
        }
        free(ref_tracker);
    }
    *hash_head_p = ref_tracker_head;
549
#endif
550 551 552 553

    return;
}

554
static void dxt_posix_cleanup_runtime()
555
{
556 557
    dxt_clear_record_refs(&(dxt_posix_runtime->fd_hash), 0);
    dxt_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
558

559 560
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
561 562 563 564

    return;
}

565
static void dxt_mpiio_cleanup_runtime()
566
{
567 568
    dxt_clear_record_refs(&(dxt_mpiio_runtime->fh_hash), 0);
    dxt_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
569

570 571
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
572 573 574 575 576 577 578 579 580

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

581
static void dxt_posix_shutdown(
582 583 584
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
585 586
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
587
{
588 589
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
590 591 592 593 594 595 596 597 598 599 600
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
601
    void *tmp_buf_ptr = *dxt_posix_buf;
602

603
    assert(dxt_posix_runtime);
604

605 606 607 608 609
    DXT_POSIX_LOCK();
    int dxt_rec_count = dxt_posix_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_posix_runtime->rec_id_hash);
610

611
    *dxt_posix_buf_sz = 0;
612 613

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
614
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
615 616 617 618 619 620 621 622 623 624 625 626
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;

        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
627
         * dxt_file_record + ost_ids + write_traces + read_traces
628
         */
629
        record_size = sizeof(struct dxt_file_record) +
630 631
                (record_write_count + record_read_count) * sizeof(segment_info);

632 633
        if (*dxt_posix_buf_sz == 0) {
            *dxt_posix_buf = (void *)malloc(record_size);
634
        } else {
635 636
            *dxt_posix_buf = (void *)realloc((*dxt_posix_buf),
                            *dxt_posix_buf_sz + record_size);
637
        }
638
        tmp_buf_ptr = (void *)(*dxt_posix_buf) + *dxt_posix_buf_sz;
639

640 641 642
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
643 644 645 646 647 648 649 650 651 652 653 654 655

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

656
        *dxt_posix_buf_sz += record_size;
657 658

#if 0
659 660
        printf("DXT, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
        printf("DXT, write_count is: %d read_count is: %d\n",
661
                    file_rec->write_count, file_rec->read_count);
662 663 664 665 666 667 668 669

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

670
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
671 672 673 674 675 676 677 678 679
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

680
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
681 682 683 684 685
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
686
    dxt_posix_cleanup_runtime();
687 688 689 690

    /* disable further instrumentation */
    instrumentation_disabled = 1;

691
    DXT_POSIX_UNLOCK();
692 693 694 695

    return;
}

696
static void dxt_mpiio_shutdown(
697 698 699
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
700 701
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
702
{
703 704
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
705 706 707 708 709 710 711 712 713 714 715
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
716
    void *tmp_buf_ptr = *dxt_mpiio_buf;
717

718
    assert(dxt_mpiio_runtime);
719

720 721 722 723 724
    DXT_MPIIO_LOCK();
    int dxt_rec_count = dxt_mpiio_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_mpiio_runtime->rec_id_hash);
725

726
    *dxt_mpiio_buf_sz = 0;
727 728

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
729
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
730 731 732 733 734 735 736 737 738 739 740
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;
        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
741
         * dxt_file_record + ost_ids + write_traces + read_traces
742
         */
743
        record_size = sizeof(struct dxt_file_record) +
744 745
                (record_write_count + record_read_count) * sizeof(segment_info);

746 747
        if (*dxt_mpiio_buf_sz == 0) {
            *dxt_mpiio_buf = (void *)malloc(record_size);
748
        } else {
749 750
            *dxt_mpiio_buf = (void *)realloc((*dxt_mpiio_buf),
                            *dxt_mpiio_buf_sz + record_size);
751
        }
752
        tmp_buf_ptr = (void *)(*dxt_mpiio_buf) + *dxt_mpiio_buf_sz;
753

754 755 756
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
757 758 759 760 761 762 763 764 765 766 767 768 769

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

770
        *dxt_mpiio_buf_sz += record_size;
771 772 773

#if 0
        printf("Cong, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
774
        printf("DXT, file_rec->write_count is: %d\n",
775 776 777 778 779 780 781 782 783
                    file_rec->write_count);

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

784
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
785 786 787 788 789 790 791 792 793
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

794
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
795 796 797 798 799
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
800
    dxt_mpiio_cleanup_runtime();
801 802 803 804

    /* disable further instrumentation */
    instrumentation_disabled = 1;

805
    DXT_MPIIO_UNLOCK();
806 807 808 809 810 811 812 813 814 815 816 817

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */