darshan-dxt.c 26.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37
/* maximum amount of memory to use for storing DXT records */
38 39 40 41 42
#ifdef __DARSHAN_MOD_MEM_MAX
#define DXT_IO_TRACE_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
#else
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB default */
#endif
43

44 45 46
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
47

48 49 50 51 52 53 54 55 56 57 58 59
/* XXX: dirty hack -- If DXT runs out of memory to store trace data in,
 * we should set a flag so that log parsers know that the log has
 * incomplete data. This functionality is typically handled automatically
 * when registering records with Darshan, but DXT modules don't
 * register records and manage their own memory. Since DXT modules request
 * 0 memory when registering themselves, any attempt to register a record
 * will result in setting the partial flag for the module, which is
 * exactly what we do here.
 */
#define SET_DXT_MOD_PARTIAL_FLAG(mod_id) \
    darshan_core_register_record(0, NULL, mod_id, 1, NULL);

60 61 62
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
63 64 65
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
66
 * associate different types of handles with this dxt_file_record_ref struct.
67 68 69
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
70
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
71 72
 * there could be multiple open file descriptors that index it.
 */
73
struct dxt_file_record_ref
74
{
75
    struct dxt_file_record *file_rec;
76

77 78
    int64_t write_available_buf;
    int64_t read_available_buf;
79 80 81

    segment_info *write_traces;
    segment_info *read_traces;
82 83
};

84 85
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
86 87
 * shutdown time.
 */
88
struct dxt_posix_runtime
89 90 91
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
92 93
    char *record_buf;
    int record_buf_size;
94 95
};

96
struct dxt_mpiio_runtime
97 98 99
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
100 101
    char *record_buf;
    int record_buf_size;
102 103
};

104 105 106 107 108
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
109 110 111 112
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
113 114 115 116 117 118

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
119
    void);
120
static void dxt_mpiio_runtime_initialize(
121
    void);
122 123
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
124 125
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
126
static void dxt_posix_cleanup_runtime(
127
    void);
128
static void dxt_mpiio_cleanup_runtime(
129 130
    void);

131
static void dxt_posix_shutdown(
132
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
133 134
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
135
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
136
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
137

138 139
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
140
static pthread_mutex_t dxt_runtime_mutex =
141 142
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

143
static int dxt_my_rank = -1;
144
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
145

146 147
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
148 149 150


/**********************************************************
151
 *      Wrappers for DXT I/O functions of interest      *
152 153
 **********************************************************/

154
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
155
{
156
    struct dxt_file_record *file_rec = rec_ref->file_rec;
157

158
    int write_count = file_rec->write_count;
159
    int write_available_buf = rec_ref->write_available_buf;
160 161

    if (write_count >= write_available_buf) {
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
178 179
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
180 181 182 183
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
184
    }
185 186 187 188 189
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
190 191

    int read_count = file_rec->read_count;
192
    int read_available_buf = rec_ref->read_available_buf;
193 194

    if (read_count >= read_available_buf) {
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
211 212
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
213 214 215 216
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
217 218 219
    }
}

220 221
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
222
{
223 224
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
225

226 227 228 229 230
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
231 232
    }

233 234 235 236 237 238 239
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
240 241
    }

242 243 244
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
245 246 247 248 249
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
250

251 252 253 254
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
255 256 257
    file_rec->write_count += 1;
}

258 259
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
260
{
261 262
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
263

264 265 266 267 268 269 270 271 272
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
273
    if (!rec_ref) {
274 275 276
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
277 278 279
    }

    file_rec = rec_ref->file_rec;
280 281
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
282 283 284 285 286
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
287

288 289 290 291
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
292 293 294
    file_rec->read_count += 1;
}

295
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
296 297
        double start_time, double end_time)
{
298 299
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
300

301 302 303 304 305
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
306 307
    }

308 309 310 311 312 313 314
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
315 316
    }

317 318 319
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
320 321 322 323 324
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
325

326 327 328
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
329 330 331
    file_rec->write_count += 1;
}

332
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
333 334
        double start_time, double end_time)
{
335 336
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
337

338 339 340 341 342
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
343 344
    }

345 346 347 348 349 350 351
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
352 353
    }

354 355 356
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
357 358 359 360 361
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
362

363 364 365
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
366 367 368 369 370
    file_rec->read_count += 1;
}


/**********************************************************
371
 * Internal functions for manipulating DXT module state *
372 373
 **********************************************************/

374
/* initialize internal DXT module data structures and register with darshan-core */
375
static void dxt_posix_runtime_initialize()
376
{
377 378 379 380 381
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
382 383
    int ret;
    double tmpfloat;
384
    char *envstr;
385

386
    /* register the DXT module with darshan core */
387
    darshan_core_register_module(
388 389
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
390
        &dxt_psx_buf_size,
391 392
        &dxt_my_rank,
        NULL);
393

394 395
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
396
    {
397
        darshan_core_unregister_module(DXT_POSIX_MOD);
398 399 400
        return;
    }

401
    DXT_LOCK();
402 403
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
404
    {
405
        darshan_core_unregister_module(DXT_POSIX_MOD);
406
        DXT_UNLOCK();
407 408
        return;
    }
409
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
410

411 412 413 414
    /* set the memory quota for DXT, if it has not been initialized */
    envstr = getenv("ENABLE_DXT_IO_TRACE_MEM");
    if(envstr && dxt_mpiio_runtime == NULL)
    {
415
        ret = sscanf(envstr, "%lf", &tmpfloat);
416
        /* silently ignore if the env variable is set poorly */
417
        if(ret == 1 && tmpfloat > 0)
418
        {
419
            dxt_mem_remaining = tmpfloat * 1024 * 1024; /* convert from MiB */
420 421 422 423
        }
    }
    DXT_UNLOCK();

424 425 426
    return;
}

427
void dxt_mpiio_runtime_initialize()
428
{
429 430 431 432 433
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
434 435
    int ret;
    double tmpfloat;
436
    char *envstr;
437

438
    /* register the DXT module with darshan core */
439
    darshan_core_register_module(
440 441
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
442
        &dxt_mpiio_buf_size,
443 444
        &dxt_my_rank,
        NULL);
445

446 447
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
448
    {
449
        darshan_core_unregister_module(DXT_MPIIO_MOD);
450 451 452
        return;
    }

453
    DXT_LOCK();
454 455
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
456
    {
457
        darshan_core_unregister_module(DXT_MPIIO_MOD);
458
        DXT_UNLOCK();
459 460
        return;
    }
461
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
462

463 464 465 466
    /* set the memory quota for DXT, if it has not been initialized */
    envstr = getenv("ENABLE_DXT_IO_TRACE_MEM");
    if(envstr && dxt_posix_runtime == NULL)
    {
467
        ret = sscanf(envstr, "%lf", &tmpfloat);
468
        /* silently ignore if the env variable is set poorly */
469
        if(ret == 1 && tmpfloat > 0)
470
        {
471
            dxt_mem_remaining = tmpfloat * 1024 * 1024; /* convert from MiB */
472 473 474 475
        }
    }
    DXT_UNLOCK();

476 477 478
    return;
}

479 480
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
481
{
482 483
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
484 485
    int ret;

486 487 488 489
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
490
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
491 492 493 494
        DXT_UNLOCK();
        return(NULL);
    }

495 496
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
497 498 499 500
    {
        DXT_UNLOCK();
        return(NULL);
    }
501 502
    memset(rec_ref, 0, sizeof(*rec_ref));

503 504 505 506 507 508 509 510 511
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

512
    /* add a reference to this file record based on record id */
513
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
514 515 516
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
517
        free(file_rec);
518
        free(rec_ref);
519 520
        DXT_UNLOCK();
        return(NULL);
521 522
    }

523 524
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
525

526
    /* initialize record and record reference fields */
527
    file_rec->base_rec.id = rec_id;
528
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
529
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
530 531

    rec_ref->file_rec = file_rec;
532
    dxt_posix_runtime->file_rec_count++;
533 534

    return(rec_ref);
535 536
}

537 538
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
539
{
540 541
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
542 543
    int ret;

544 545 546 547
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
548
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
549 550 551 552
        DXT_UNLOCK();
        return(NULL);
    }

553 554
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
555 556 557 558
    {
        DXT_UNLOCK();
        return(NULL);
    }
559 560
    memset(rec_ref, 0, sizeof(*rec_ref));

561 562 563 564 565 566 567 568 569
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

570
    /* add a reference to this file record based on record id */
571
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
572 573 574
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
575
        free(file_rec);
576
        free(rec_ref);
577
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
578
        return(NULL);
579 580
    }

581 582
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
583

584
    /* initialize record and record reference fields */
585
    file_rec->base_rec.id = rec_id;
586
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
587
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
588 589

    rec_ref->file_rec = file_rec;
590
    dxt_mpiio_runtime->file_rec_count++;
591

592
    return(rec_ref);
593 594
}

Shane Snyder's avatar
Shane Snyder committed
595
static void dxt_free_record_data(void *rec_ref_p)
596
{
597
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
598

599 600
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
601
    free(dxt_rec_ref->file_rec);
602 603
}

604
static void dxt_posix_cleanup_runtime()
605
{
Shane Snyder's avatar
Shane Snyder committed
606
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
607
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
608

609 610
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
611 612 613 614

    return;
}

615
static void dxt_mpiio_cleanup_runtime()
616
{
Shane Snyder's avatar
Shane Snyder committed
617
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
618
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
619

620 621
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
622 623 624 625 626 627 628 629 630

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
631
static void dxt_serialize_posix_records(void *rec_ref_p)
632
{
Shane Snyder's avatar
Shane Snyder committed
633
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
634
    struct dxt_file_record *file_rec;
635 636 637
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
638
    void *tmp_buf_ptr;
639

Shane Snyder's avatar
Shane Snyder committed
640 641 642
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
643

Shane Snyder's avatar
Shane Snyder committed
644 645 646 647
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
648

Shane Snyder's avatar
Shane Snyder committed
649 650
    /*
     * Buffer format:
651
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
652 653 654
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
655

Shane Snyder's avatar
Shane Snyder committed
656 657
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
658

Shane Snyder's avatar
Shane Snyder committed
659 660 661
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
662

Shane Snyder's avatar
Shane Snyder committed
663
    /*Copy write record */
664
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
665 666 667
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
668

Shane Snyder's avatar
Shane Snyder committed
669
    /*Copy read record */
670
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
671 672 673
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
674

Shane Snyder's avatar
Shane Snyder committed
675
    dxt_posix_runtime->record_buf_size += record_size;
676

Shane Snyder's avatar
Shane Snyder committed
677 678
#if 0
    int i;
Cong Xu's avatar
Cong Xu committed
679 680
    int64_t rank;
    char *hostname;
Shane Snyder's avatar
Shane Snyder committed
681 682 683 684
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
685

Cong Xu's avatar
Cong Xu committed
686 687 688 689 690
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
Shane Snyder's avatar
Shane Snyder committed
691
                file_rec->write_count, file_rec->read_count);
Cong Xu's avatar
Cong Xu committed
692
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
693

Shane Snyder's avatar
Shane Snyder committed
694
    for (i = 0; i < file_rec->write_count; i++) {
695 696 697 698
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
699

Cong Xu's avatar
Cong Xu committed
700
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
701
    }
702

Shane Snyder's avatar
Shane Snyder committed
703
    for (i = 0; i < file_rec->read_count; i++) {
704 705 706 707
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
708

Cong Xu's avatar
Cong Xu committed
709
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
710 711 712
    }
#endif
}
713

Shane Snyder's avatar
Shane Snyder committed
714 715 716 717 718 719 720 721
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
722

Shane Snyder's avatar
Shane Snyder committed
723 724 725 726 727 728 729 730 731 732 733 734 735 736
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
737 738

    /* shutdown internal structures used for instrumenting */
739
    dxt_posix_cleanup_runtime();
740 741 742 743

    return;
}

Shane Snyder's avatar
Shane Snyder committed
744
static void dxt_serialize_mpiio_records(void *rec_ref_p)
745
{
Shane Snyder's avatar
Shane Snyder committed
746
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
747
    struct dxt_file_record *file_rec;
748 749 750
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
751
    void *tmp_buf_ptr;
752

Shane Snyder's avatar
Shane Snyder committed
753 754 755
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
756

Shane Snyder's avatar
Shane Snyder committed
757 758 759 760
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
761

Shane Snyder's avatar
Shane Snyder committed
762 763
    /*
     * Buffer format:
764
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
765 766 767
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
768

Shane Snyder's avatar
Shane Snyder committed
769 770
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
771

Shane Snyder's avatar
Shane Snyder committed
772 773 774
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
775

Shane Snyder's avatar
Shane Snyder committed
776
    /*Copy write record */
777
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
778 779
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
780 781
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
782
    /*Copy read record */
783
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
784 785
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
786 787
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
788
    dxt_mpiio_runtime->record_buf_size += record_size;
789 790

#if 0
Shane Snyder's avatar
Shane Snyder committed
791 792
    int i;
    int64_t rank;
Cong Xu's avatar
Cong Xu committed
793 794
    char *hostname;
    int64_t length;
Shane Snyder's avatar
Shane Snyder committed
795 796
    double start_time;
    double end_time;
797

Cong Xu's avatar
Cong Xu committed
798 799 800 801 802 803 804
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
805

Shane Snyder's avatar
Shane Snyder committed
806
    for (i = 0; i < file_rec->write_count; i++) {
807 808 809
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
810

Cong Xu's avatar
Cong Xu committed
811
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
812 813
    }

Shane Snyder's avatar
Shane Snyder committed
814
    for (i = 0; i < file_rec->read_count; i++) {
815 816 817
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
818

Cong Xu's avatar
Cong Xu committed
819
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

848
    /* shutdown internal structures used for instrumenting */
849
    dxt_mpiio_cleanup_runtime();
850 851 852 853 854 855 856 857 858 859 860 861

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */