darshan-dxt.c 26.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37
/* maximum amount of memory to use for storing DXT records */
38 39 40 41 42
#ifdef __DARSHAN_MOD_MEM_MAX
#define DXT_IO_TRACE_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
#else
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB default */
#endif
43

44 45 46
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
47

48 49 50 51 52 53 54 55 56 57 58 59
/* XXX: dirty hack -- If DXT runs out of memory to store trace data in,
 * we should set a flag so that log parsers know that the log has
 * incomplete data. This functionality is typically handled automatically
 * when registering records with Darshan, but DXT modules don't
 * register records and manage their own memory. Since DXT modules request
 * 0 memory when registering themselves, any attempt to register a record
 * will result in setting the partial flag for the module, which is
 * exactly what we do here.
 */
#define SET_DXT_MOD_PARTIAL_FLAG(mod_id) \
    darshan_core_register_record(0, NULL, mod_id, 1, NULL);

60 61 62
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
63 64 65
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
66
 * associate different types of handles with this dxt_file_record_ref struct.
67 68 69
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
70
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
71 72
 * there could be multiple open file descriptors that index it.
 */
73
struct dxt_file_record_ref
74
{
75
    struct dxt_file_record *file_rec;
76

77 78
    int64_t write_available_buf;
    int64_t read_available_buf;
79 80 81

    segment_info *write_traces;
    segment_info *read_traces;
82 83
};

84 85
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
86 87
 * shutdown time.
 */
88
struct dxt_posix_runtime
89 90 91
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
92 93
    char *record_buf;
    int record_buf_size;
94 95
};

96
struct dxt_mpiio_runtime
97 98 99
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
100 101
    char *record_buf;
    int record_buf_size;
102 103
};

104 105 106 107 108
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
109 110 111 112
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
113 114 115 116 117 118

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
119
    void);
120
static void dxt_mpiio_runtime_initialize(
121
    void);
122 123
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
124 125
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
126
static void dxt_posix_cleanup_runtime(
127
    void);
128
static void dxt_mpiio_cleanup_runtime(
129 130
    void);

131
static void dxt_posix_shutdown(
132
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
133 134
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
135
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
136
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
137

138 139
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
140
static pthread_mutex_t dxt_runtime_mutex =
141 142
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

143
static int dxt_my_rank = -1;
144 145
static int dxt_total_mem = DXT_IO_TRACE_MEM_MAX;
static int dxt_mem_remaining = 0;
146

147 148
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
149 150 151


/**********************************************************
152
 *      Wrappers for DXT I/O functions of interest      *
153 154
 **********************************************************/

155
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
156
{
157
    struct dxt_file_record *file_rec = rec_ref->file_rec;
158

159
    int write_count = file_rec->write_count;
160
    int write_available_buf = rec_ref->write_available_buf;
161 162

    if (write_count >= write_available_buf) {
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
179 180
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
181 182 183 184
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
185
    }
186 187 188 189 190
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
191 192

    int read_count = file_rec->read_count;
193
    int read_available_buf = rec_ref->read_available_buf;
194 195

    if (read_count >= read_available_buf) {
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
212 213
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
214 215 216 217
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
218 219 220
    }
}

221 222
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
223
{
224 225
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
226

227 228 229 230 231
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
232 233
    }

234 235 236 237 238 239 240
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
241 242
    }

243 244 245
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
246 247 248 249 250
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
251

252 253 254 255
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
256 257 258
    file_rec->write_count += 1;
}

259 260
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
261
{
262 263
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
264

265 266 267 268 269 270 271 272 273
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
274
    if (!rec_ref) {
275 276 277
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
278 279 280
    }

    file_rec = rec_ref->file_rec;
281 282
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
283 284 285 286 287
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
288

289 290 291 292
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
293 294 295
    file_rec->read_count += 1;
}

296
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
297 298
        double start_time, double end_time)
{
299 300
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
301

302 303 304 305 306
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
307 308
    }

309 310 311 312 313 314 315
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
316 317
    }

318 319 320
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
321 322 323 324 325
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
326

327 328 329
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
330 331 332
    file_rec->write_count += 1;
}

333
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
334 335
        double start_time, double end_time)
{
336 337
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
338

339 340 341 342 343
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
344 345
    }

346 347 348 349 350 351 352
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
353 354
    }

355 356 357
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
358 359 360 361 362
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
363

364 365 366
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
367 368 369 370 371
    file_rec->read_count += 1;
}


/**********************************************************
372
 * Internal functions for manipulating DXT module state *
373 374
 **********************************************************/

375
/* initialize internal DXT module data structures and register with darshan-core */
376
static void dxt_posix_runtime_initialize()
377
{
378 379 380 381 382
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
383 384
    int ret;
    double tmpfloat;
385
    char *envstr;
386

387
    /* register the DXT module with darshan core */
388
    darshan_core_register_module(
389 390
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
391
        &dxt_psx_buf_size,
392 393
        &dxt_my_rank,
        NULL);
394

395 396
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
397
    {
398
        darshan_core_unregister_module(DXT_POSIX_MOD);
399 400 401
        return;
    }

402
    DXT_LOCK();
403 404
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
405
    {
406
        darshan_core_unregister_module(DXT_POSIX_MOD);
407
        DXT_UNLOCK();
408 409
        return;
    }
410
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
411

412
    /* set the memory quota for DXT, if it has not been initialized */
413
    envstr = getenv("DXT_ENABLE_IO_TRACE");
414 415
    if(envstr && dxt_mpiio_runtime == NULL)
    {
416
        ret = sscanf(envstr, "%lf", &tmpfloat);
417
        /* silently ignore if the env variable is set poorly */
418
        if(ret == 1 && tmpfloat > 0)
419
        {
420
            dxt_total_mem = tmpfloat * 1024 * 1024; /* convert from MiB */
421 422
        }
    }
423
    dxt_mem_remaining = dxt_total_mem;
424 425
    DXT_UNLOCK();

426 427 428
    return;
}

429
void dxt_mpiio_runtime_initialize()
430
{
431 432 433 434 435
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
436 437
    int ret;
    double tmpfloat;
438
    char *envstr;
439

440
    /* register the DXT module with darshan core */
441
    darshan_core_register_module(
442 443
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
444
        &dxt_mpiio_buf_size,
445 446
        &dxt_my_rank,
        NULL);
447

448 449
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
450
    {
451
        darshan_core_unregister_module(DXT_MPIIO_MOD);
452 453 454
        return;
    }

455
    DXT_LOCK();
456 457
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
458
    {
459
        darshan_core_unregister_module(DXT_MPIIO_MOD);
460
        DXT_UNLOCK();
461 462
        return;
    }
463
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
464

465
    /* set the memory quota for DXT, if it has not been initialized */
466
    envstr = getenv("DXT_ENABLE_IO_TRACE");
467 468
    if(envstr && dxt_posix_runtime == NULL)
    {
469
        ret = sscanf(envstr, "%lf", &tmpfloat);
470
        /* silently ignore if the env variable is set poorly */
471
        if(ret == 1 && tmpfloat > 0)
472
        {
473
            dxt_total_mem = tmpfloat * 1024 * 1024; /* convert from MiB */
474 475
        }
    }
476
    dxt_mem_remaining = dxt_total_mem;
477 478
    DXT_UNLOCK();

479 480 481
    return;
}

482 483
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
484
{
485 486
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
487 488
    int ret;

489 490 491 492
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
493
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
494 495 496 497
        DXT_UNLOCK();
        return(NULL);
    }

498 499
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
500 501 502 503
    {
        DXT_UNLOCK();
        return(NULL);
    }
504 505
    memset(rec_ref, 0, sizeof(*rec_ref));

506 507 508 509 510 511 512 513 514
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

515
    /* add a reference to this file record based on record id */
516
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
517 518 519
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
520
        free(file_rec);
521
        free(rec_ref);
522 523
        DXT_UNLOCK();
        return(NULL);
524 525
    }

526 527
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
528

529
    /* initialize record and record reference fields */
530
    file_rec->base_rec.id = rec_id;
531
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
532
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
533 534

    rec_ref->file_rec = file_rec;
535
    dxt_posix_runtime->file_rec_count++;
536 537

    return(rec_ref);
538 539
}

540 541
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
542
{
543 544
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
545 546
    int ret;

547 548 549 550
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
551
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
552 553 554 555
        DXT_UNLOCK();
        return(NULL);
    }

556 557
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
558 559 560 561
    {
        DXT_UNLOCK();
        return(NULL);
    }
562 563
    memset(rec_ref, 0, sizeof(*rec_ref));

564 565 566 567 568 569 570 571 572
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

573
    /* add a reference to this file record based on record id */
574
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
575 576 577
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
578
        free(file_rec);
579
        free(rec_ref);
580
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
581
        return(NULL);
582 583
    }

584 585
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
586

587
    /* initialize record and record reference fields */
588
    file_rec->base_rec.id = rec_id;
589
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
590
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
591 592

    rec_ref->file_rec = file_rec;
593
    dxt_mpiio_runtime->file_rec_count++;
594

595
    return(rec_ref);
596 597
}

Shane Snyder's avatar
Shane Snyder committed
598
static void dxt_free_record_data(void *rec_ref_p)
599
{
600
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
601

602 603
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
604
    free(dxt_rec_ref->file_rec);
605 606
}

607
static void dxt_posix_cleanup_runtime()
608
{
Shane Snyder's avatar
Shane Snyder committed
609
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
610
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
611

612 613
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
614 615 616 617

    return;
}

618
static void dxt_mpiio_cleanup_runtime()
619
{
Shane Snyder's avatar
Shane Snyder committed
620
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
621
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
622

623 624
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
625 626 627 628 629 630 631 632 633

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
634
static void dxt_serialize_posix_records(void *rec_ref_p)
635
{
Shane Snyder's avatar
Shane Snyder committed
636
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
637
    struct dxt_file_record *file_rec;
638 639 640
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
641
    void *tmp_buf_ptr;
642

Shane Snyder's avatar
Shane Snyder committed
643 644 645
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
646

Shane Snyder's avatar
Shane Snyder committed
647 648 649 650
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
651

Shane Snyder's avatar
Shane Snyder committed
652 653
    /*
     * Buffer format:
654
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
655 656 657
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
658

Shane Snyder's avatar
Shane Snyder committed
659 660
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
661

Shane Snyder's avatar
Shane Snyder committed
662 663 664
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
665

Shane Snyder's avatar
Shane Snyder committed
666
    /*Copy write record */
667
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
668 669 670
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
671

Shane Snyder's avatar
Shane Snyder committed
672
    /*Copy read record */
673
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
674 675 676
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
677

Shane Snyder's avatar
Shane Snyder committed
678
    dxt_posix_runtime->record_buf_size += record_size;
679

Shane Snyder's avatar
Shane Snyder committed
680 681
#if 0
    int i;
Cong Xu's avatar
Cong Xu committed
682 683
    int64_t rank;
    char *hostname;
Shane Snyder's avatar
Shane Snyder committed
684 685 686 687
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
688

Cong Xu's avatar
Cong Xu committed
689 690 691 692 693
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
Shane Snyder's avatar
Shane Snyder committed
694
                file_rec->write_count, file_rec->read_count);
Cong Xu's avatar
Cong Xu committed
695
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
696

Shane Snyder's avatar
Shane Snyder committed
697
    for (i = 0; i < file_rec->write_count; i++) {
698 699 700 701
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
702

Cong Xu's avatar
Cong Xu committed
703
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
704
    }
705

Shane Snyder's avatar
Shane Snyder committed
706
    for (i = 0; i < file_rec->read_count; i++) {
707 708 709 710
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
711

Cong Xu's avatar
Cong Xu committed
712
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
713 714 715
    }
#endif
}
716

Shane Snyder's avatar
Shane Snyder committed
717 718 719 720 721 722 723 724
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
725

Shane Snyder's avatar
Shane Snyder committed
726 727
    *dxt_posix_buf_sz = 0;

728
    dxt_posix_runtime->record_buf = malloc(dxt_total_mem);
Shane Snyder's avatar
Shane Snyder committed
729 730
    if(!(dxt_posix_runtime->record_buf))
        return;
731
    memset(dxt_posix_runtime->record_buf, 0, dxt_total_mem);
Shane Snyder's avatar
Shane Snyder committed
732 733 734 735 736 737 738 739
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
740 741

    /* shutdown internal structures used for instrumenting */
742
    dxt_posix_cleanup_runtime();
743 744 745 746

    return;
}

Shane Snyder's avatar
Shane Snyder committed
747
static void dxt_serialize_mpiio_records(void *rec_ref_p)
748
{
Shane Snyder's avatar
Shane Snyder committed
749
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
750
    struct dxt_file_record *file_rec;
751 752 753
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
754
    void *tmp_buf_ptr;
755

Shane Snyder's avatar
Shane Snyder committed
756 757 758
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
759

Shane Snyder's avatar
Shane Snyder committed
760 761 762 763
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
764

Shane Snyder's avatar
Shane Snyder committed
765 766
    /*
     * Buffer format:
767
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
768 769 770
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
771

Shane Snyder's avatar
Shane Snyder committed
772 773
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
774

Shane Snyder's avatar
Shane Snyder committed
775 776 777
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
778

Shane Snyder's avatar
Shane Snyder committed
779
    /*Copy write record */
780
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
781 782
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
783 784
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
785
    /*Copy read record */
786
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
787 788
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
789 790
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
791
    dxt_mpiio_runtime->record_buf_size += record_size;
792 793

#if 0
Shane Snyder's avatar
Shane Snyder committed
794 795
    int i;
    int64_t rank;
Cong Xu's avatar
Cong Xu committed
796 797
    char *hostname;
    int64_t length;
Shane Snyder's avatar
Shane Snyder committed
798 799
    double start_time;
    double end_time;
800

Cong Xu's avatar
Cong Xu committed
801 802 803 804 805 806 807
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
808

Shane Snyder's avatar
Shane Snyder committed
809
    for (i = 0; i < file_rec->write_count; i++) {
810 811 812
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
813

Cong Xu's avatar
Cong Xu committed
814
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
815 816
    }

Shane Snyder's avatar
Shane Snyder committed
817
    for (i = 0; i < file_rec->read_count; i++) {
818 819 820
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
821

Cong Xu's avatar
Cong Xu committed
822
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

838
    dxt_mpiio_runtime->record_buf = malloc(dxt_total_mem);
Shane Snyder's avatar
Shane Snyder committed
839 840
    if(!(dxt_mpiio_runtime->record_buf))
        return;
841
    memset(dxt_mpiio_runtime->record_buf, 0, dxt_total_mem);
Shane Snyder's avatar
Shane Snyder committed
842 843 844 845 846 847 848 849 850
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

851
    /* shutdown internal structures used for instrumenting */
852
    dxt_mpiio_cleanup_runtime();
853 854 855 856 857 858 859 860 861 862 863 864

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */