darshan-dxt.c 24.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37 38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40 41 42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44 45 46
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
47 48 49
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
50
 * associate different types of handles with this dxt_file_record_ref struct.
51 52 53
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
54
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
55 56
 * there could be multiple open file descriptors that index it.
 */
57
struct dxt_file_record_ref
58
{
59
    struct dxt_file_record *file_rec;
60

61 62
    int64_t write_available_buf;
    int64_t read_available_buf;
63 64 65

    segment_info *write_traces;
    segment_info *read_traces;
66 67
};

68 69
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
70 71
 * shutdown time.
 */
72
struct dxt_posix_runtime
73 74 75
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
76 77
    char *record_buf;
    int record_buf_size;
78 79
};

80
struct dxt_mpiio_runtime
81 82 83
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
84 85
    char *record_buf;
    int record_buf_size;
86 87
};

88 89 90 91 92
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
93 94 95 96
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
97 98 99 100 101 102

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
103
    void);
104
static void dxt_mpiio_runtime_initialize(
105
    void);
106 107
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
108 109
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
110
static void dxt_posix_cleanup_runtime(
111
    void);
112
static void dxt_mpiio_cleanup_runtime(
113 114
    void);

115
static void dxt_posix_shutdown(
116
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
117 118
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
119
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
120
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
121

122 123
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
124
static pthread_mutex_t dxt_runtime_mutex =
125 126 127 128 129 130
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;
131
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
132

133 134
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
135 136 137


/**********************************************************
138
 *      Wrappers for DXT I/O functions of interest      *
139 140
 **********************************************************/

141
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
142
{
143
    struct dxt_file_record *file_rec = rec_ref->file_rec;
144

145
    int write_count = file_rec->write_count;
146
    int write_available_buf = rec_ref->write_available_buf;
147 148

    if (write_count >= write_available_buf) {
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
165 166
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
167 168 169 170
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
171
    }
172 173 174 175 176
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
177 178

    int read_count = file_rec->read_count;
179
    int read_available_buf = rec_ref->read_available_buf;
180 181

    if (read_count >= read_available_buf) {
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
198 199
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
200 201 202 203
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
204 205 206
    }
}

207 208
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
209
{
210 211
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
212

213 214 215 216 217 218
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
219 220
    }

221 222 223 224 225 226 227
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
228 229
    }

230 231 232 233 234
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

235 236 237 238
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
239 240 241
    file_rec->write_count += 1;
}

242 243
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
244
{
245 246
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
247

248 249 250 251 252 253 254 255 256 257
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
258
    if (!rec_ref) {
259 260 261
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
262 263 264
    }

    file_rec = rec_ref->file_rec;
265 266 267
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
268

269 270 271 272
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
273 274 275
    file_rec->read_count += 1;
}

276
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
277 278
        double start_time, double end_time)
{
279 280
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
281

282 283 284 285 286 287
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
288 289
    }

290 291 292 293 294 295 296
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
297 298
    }

299 300 301 302 303
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

304 305 306
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
307 308 309
    file_rec->write_count += 1;
}

310
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
311 312
        double start_time, double end_time)
{
313 314
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
315

316 317 318 319 320 321
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
322 323
    }

324 325 326 327 328 329 330
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
331 332
    }

333 334 335 336 337
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */

338 339 340
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
341 342 343 344 345
    file_rec->read_count += 1;
}


/**********************************************************
346
 * Internal functions for manipulating DXT module state *
347 348
 **********************************************************/

349
/* initialize internal DXT module data structures and register with darshan-core */
350
static void dxt_posix_runtime_initialize()
351
{
352 353 354 355 356
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
357

358
    /* register the DXT module with darshan core */
359
    darshan_core_register_module(
360 361
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
362
        &dxt_psx_buf_size,
363 364 365
        &posix_my_rank,
        &darshan_mem_alignment);

366 367
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
368
    {
369
        darshan_core_unregister_module(DXT_POSIX_MOD);
370 371 372
        return;
    }

373 374
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
375
    {
376
        darshan_core_unregister_module(DXT_POSIX_MOD);
377 378
        return;
    }
379
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
380 381 382 383

    return;
}

384
void dxt_mpiio_runtime_initialize()
385
{
386 387 388 389 390
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
391

392
    /* register the DXT module with darshan core */
393
    darshan_core_register_module(
394 395
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
396
        &dxt_mpiio_buf_size,
397 398 399
        &mpiio_my_rank,
        &darshan_mem_alignment);

400 401
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
402
    {
403
        darshan_core_unregister_module(DXT_MPIIO_MOD);
404 405 406
        return;
    }

407 408
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
409
    {
410
        darshan_core_unregister_module(DXT_MPIIO_MOD);
411 412
        return;
    }
413
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
414 415 416 417

    return;
}

418 419
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
420
{
421 422
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
423 424
    int ret;

425 426 427 428 429 430 431 432
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

433 434
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
435 436 437 438
    {
        DXT_UNLOCK();
        return(NULL);
    }
439 440
    memset(rec_ref, 0, sizeof(*rec_ref));

441 442 443 444 445 446 447 448 449
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

450
    /* add a reference to this file record based on record id */
451
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
452 453 454
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
455
        free(file_rec);
456
        free(rec_ref);
457 458
        DXT_UNLOCK();
        return(NULL);
459 460
    }

461 462
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
463

464
    /* initialize record and record reference fields */
465 466 467 468
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;

    rec_ref->file_rec = file_rec;
469
    dxt_posix_runtime->file_rec_count++;
470 471

    return(rec_ref);
472 473
}

474 475
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
476
{
477 478
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
479 480
    int ret;

481 482 483 484 485 486 487 488
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

489 490
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
491 492 493 494
    {
        DXT_UNLOCK();
        return(NULL);
    }
495 496
    memset(rec_ref, 0, sizeof(*rec_ref));

497 498 499 500 501 502 503 504 505
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

506
    /* add a reference to this file record based on record id */
507
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
508 509 510
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
511
        free(file_rec);
512
        free(rec_ref);
513
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
514
        return(NULL);
515 516
    }

517 518
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
519

520
    /* initialize record and record reference fields */
521 522 523 524
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;

    rec_ref->file_rec = file_rec;
525
    dxt_mpiio_runtime->file_rec_count++;
526

527
    return(rec_ref);
528 529
}

Shane Snyder's avatar
Shane Snyder committed
530
static void dxt_free_record_data(void *rec_ref_p)
531
{
532
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
533

534
    /* TODO: update these pointer addresses once {write/read}_traces are moved to rec_ref structure */
535 536
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
537
    free(dxt_rec_ref->file_rec);
538 539
}

540
static void dxt_posix_cleanup_runtime()
541
{
Shane Snyder's avatar
Shane Snyder committed
542
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
543
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
544

545 546
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
547 548 549 550

    return;
}

551
static void dxt_mpiio_cleanup_runtime()
552
{
Shane Snyder's avatar
Shane Snyder committed
553
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
554
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
555

556 557
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
558 559 560 561 562 563 564 565 566

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
567
static void dxt_serialize_posix_records(void *rec_ref_p)
568
{
Shane Snyder's avatar
Shane Snyder committed
569
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
570
    struct dxt_file_record *file_rec;
571 572 573
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
574
    void *tmp_buf_ptr;
575

Shane Snyder's avatar
Shane Snyder committed
576 577 578
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
579

Shane Snyder's avatar
Shane Snyder committed
580 581 582 583
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
584

Shane Snyder's avatar
Shane Snyder committed
585 586
    /*
     * Buffer format:
587
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
588 589 590
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
591

Shane Snyder's avatar
Shane Snyder committed
592 593
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
594

Shane Snyder's avatar
Shane Snyder committed
595 596 597
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
598

Shane Snyder's avatar
Shane Snyder committed
599
    /*Copy write record */
600
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
601 602 603
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
604

Shane Snyder's avatar
Shane Snyder committed
605
    /*Copy read record */
606
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
607 608 609
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
610

Shane Snyder's avatar
Shane Snyder committed
611
    dxt_posix_runtime->record_buf_size += record_size;
612

Shane Snyder's avatar
Shane Snyder committed
613 614 615 616 617 618 619
#if 0
    int i;
    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;
620

Shane Snyder's avatar
Shane Snyder committed
621 622 623
    printf("DXT, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("DXT, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
624

Shane Snyder's avatar
Shane Snyder committed
625 626
    for (i = 0; i < file_rec->write_count; i++) {
        rank = file_rec->base_rec.rank;
627 628 629 630
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
631

Shane Snyder's avatar
Shane Snyder committed
632 633
        printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
634

Shane Snyder's avatar
Shane Snyder committed
635 636
    for (i = 0; i < file_rec->read_count; i++) {
        rank = file_rec->base_rec.rank;
637 638 639 640
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
641

Shane Snyder's avatar
Shane Snyder committed
642 643 644 645
        printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
#endif
}
646

Shane Snyder's avatar
Shane Snyder committed
647 648 649 650 651 652 653 654
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
655

Shane Snyder's avatar
Shane Snyder committed
656 657 658 659 660 661 662 663 664 665 666 667 668 669
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
670 671

    /* shutdown internal structures used for instrumenting */
672
    dxt_posix_cleanup_runtime();
673 674 675 676 677 678 679

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

Shane Snyder's avatar
Shane Snyder committed
680
static void dxt_serialize_mpiio_records(void *rec_ref_p)
681
{
Shane Snyder's avatar
Shane Snyder committed
682
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
683
    struct dxt_file_record *file_rec;
684 685 686
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
687
    void *tmp_buf_ptr;
688

Shane Snyder's avatar
Shane Snyder committed
689 690 691
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
692

Shane Snyder's avatar
Shane Snyder committed
693 694 695 696
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
697

Shane Snyder's avatar
Shane Snyder committed
698 699
    /*
     * Buffer format:
700
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
701 702 703
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
704

Shane Snyder's avatar
Shane Snyder committed
705 706
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
707

Shane Snyder's avatar
Shane Snyder committed
708 709 710
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
711

Shane Snyder's avatar
Shane Snyder committed
712
    /*Copy write record */
713
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
714 715
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
716 717
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
718
    /*Copy read record */
719
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
720 721
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
722 723
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
724
    dxt_mpiio_runtime->record_buf_size += record_size;
725 726

#if 0
Shane Snyder's avatar
Shane Snyder committed
727 728 729 730 731 732
    int i;
    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;
733

Shane Snyder's avatar
Shane Snyder committed
734 735 736
    printf("Cong, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("DXT, file_rec->write_count is: %d\n",
                file_rec->write_count);
737

Shane Snyder's avatar
Shane Snyder committed
738 739
    for (i = 0; i < file_rec->write_count; i++) {
        rank = file_rec->base_rec.rank;
740 741 742 743
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
744 745

        printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
746 747
    }

Shane Snyder's avatar
Shane Snyder committed
748 749
    for (i = 0; i < file_rec->read_count; i++) {
        rank = file_rec->base_rec.rank;
750 751 752 753
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783

        printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

784
    /* shutdown internal structures used for instrumenting */
785
    dxt_mpiio_cleanup_runtime();
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */