darshan-dxt.c 24.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37 38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40 41 42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44 45 46
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
47 48 49
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
50
 * associate different types of handles with this dxt_file_record_ref struct.
51 52 53
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
54
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
55 56
 * there could be multiple open file descriptors that index it.
 */
57
struct dxt_file_record_ref
58
{
59
    struct dxt_file_record *file_rec;
60

61 62
    int64_t write_available_buf;
    int64_t read_available_buf;
63 64 65

    segment_info *write_traces;
    segment_info *read_traces;
66 67
};

68 69
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
70 71
 * shutdown time.
 */
72
struct dxt_posix_runtime
73 74 75
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
76 77
    char *record_buf;
    int record_buf_size;
78 79
};

80
struct dxt_mpiio_runtime
81 82 83
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
84 85
    char *record_buf;
    int record_buf_size;
86 87
};

88 89 90 91 92
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
93 94 95 96
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
97 98 99 100 101 102

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
103
    void);
104
static void dxt_mpiio_runtime_initialize(
105
    void);
106 107
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
108 109
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
110
static void dxt_posix_cleanup_runtime(
111
    void);
112
static void dxt_mpiio_cleanup_runtime(
113 114
    void);

115
static void dxt_posix_shutdown(
116
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
117 118
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
119
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
120
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
121

122 123
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
124
static pthread_mutex_t dxt_runtime_mutex =
125 126
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

127
static int dxt_my_rank = -1;
128
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
129

130 131
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
132 133 134


/**********************************************************
135
 *      Wrappers for DXT I/O functions of interest      *
136 137
 **********************************************************/

138
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
139
{
140
    struct dxt_file_record *file_rec = rec_ref->file_rec;
141

142
    int write_count = file_rec->write_count;
143
    int write_available_buf = rec_ref->write_available_buf;
144 145

    if (write_count >= write_available_buf) {
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
162 163
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
164 165 166 167
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
168
    }
169 170 171 172 173
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
174 175

    int read_count = file_rec->read_count;
176
    int read_available_buf = rec_ref->read_available_buf;
177 178

    if (read_count >= read_available_buf) {
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
195 196
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
197 198 199 200
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
201 202 203
    }
}

204 205
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
206
{
207 208
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
209

210 211 212 213 214
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
215 216
    }

217 218 219 220 221 222 223
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
224 225
    }

226 227 228 229 230
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

231 232 233 234
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
235 236 237
    file_rec->write_count += 1;
}

238 239
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
240
{
241 242
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
243

244 245 246 247 248 249 250 251 252
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
253
    if (!rec_ref) {
254 255 256
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
257 258 259
    }

    file_rec = rec_ref->file_rec;
260 261 262
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
263

264 265 266 267
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
268 269 270
    file_rec->read_count += 1;
}

271
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
272 273
        double start_time, double end_time)
{
274 275
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
276

277 278 279 280 281
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
282 283
    }

284 285 286 287 288 289 290
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
291 292
    }

293 294 295 296 297
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

298 299 300
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
301 302 303
    file_rec->write_count += 1;
}

304
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
305 306
        double start_time, double end_time)
{
307 308
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
309

310 311 312 313 314
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
315 316
    }

317 318 319 320 321 322 323
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
324 325
    }

326 327 328 329 330
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */

331 332 333
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
334 335 336 337 338
    file_rec->read_count += 1;
}


/**********************************************************
339
 * Internal functions for manipulating DXT module state *
340 341
 **********************************************************/

342
/* initialize internal DXT module data structures and register with darshan-core */
343
static void dxt_posix_runtime_initialize()
344
{
345 346 347 348 349
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
350

351
    /* register the DXT module with darshan core */
352
    darshan_core_register_module(
353 354
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
355
        &dxt_psx_buf_size,
356 357
        &dxt_my_rank,
        NULL);
358

359 360
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
361
    {
362
        darshan_core_unregister_module(DXT_POSIX_MOD);
363 364 365
        return;
    }

366 367
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
368
    {
369
        darshan_core_unregister_module(DXT_POSIX_MOD);
370 371
        return;
    }
372
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
373 374 375 376

    return;
}

377
void dxt_mpiio_runtime_initialize()
378
{
379 380 381 382 383
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
384

385
    /* register the DXT module with darshan core */
386
    darshan_core_register_module(
387 388
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
389
        &dxt_mpiio_buf_size,
390 391
        &dxt_my_rank,
        NULL);
392

393 394
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
395
    {
396
        darshan_core_unregister_module(DXT_MPIIO_MOD);
397 398 399
        return;
    }

400 401
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
402
    {
403
        darshan_core_unregister_module(DXT_MPIIO_MOD);
404 405
        return;
    }
406
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
407 408 409 410

    return;
}

411 412
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
413
{
414 415
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
416 417
    int ret;

418 419 420 421 422 423 424 425
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

426 427
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
428 429 430 431
    {
        DXT_UNLOCK();
        return(NULL);
    }
432 433
    memset(rec_ref, 0, sizeof(*rec_ref));

434 435 436 437 438 439 440 441 442
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

443
    /* add a reference to this file record based on record id */
444
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
445 446 447
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
448
        free(file_rec);
449
        free(rec_ref);
450 451
        DXT_UNLOCK();
        return(NULL);
452 453
    }

454 455
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
456

457
    /* initialize record and record reference fields */
458
    file_rec->base_rec.id = rec_id;
459
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
460
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
461 462

    rec_ref->file_rec = file_rec;
463
    dxt_posix_runtime->file_rec_count++;
464 465

    return(rec_ref);
466 467
}

468 469
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
470
{
471 472
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
473 474
    int ret;

475 476 477 478 479 480 481 482
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

483 484
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
485 486 487 488
    {
        DXT_UNLOCK();
        return(NULL);
    }
489 490
    memset(rec_ref, 0, sizeof(*rec_ref));

491 492 493 494 495 496 497 498 499
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

500
    /* add a reference to this file record based on record id */
501
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
502 503 504
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
505
        free(file_rec);
506
        free(rec_ref);
507
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
508
        return(NULL);
509 510
    }

511 512
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
513

514
    /* initialize record and record reference fields */
515
    file_rec->base_rec.id = rec_id;
516
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
517
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
518 519

    rec_ref->file_rec = file_rec;
520
    dxt_mpiio_runtime->file_rec_count++;
521

522
    return(rec_ref);
523 524
}

Shane Snyder's avatar
Shane Snyder committed
525
static void dxt_free_record_data(void *rec_ref_p)
526
{
527
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
528

529 530
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
531
    free(dxt_rec_ref->file_rec);
532 533
}

534
static void dxt_posix_cleanup_runtime()
535
{
Shane Snyder's avatar
Shane Snyder committed
536
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
537
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
538

539 540
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
541 542 543 544

    return;
}

545
static void dxt_mpiio_cleanup_runtime()
546
{
Shane Snyder's avatar
Shane Snyder committed
547
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
548
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
549

550 551
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
552 553 554 555 556 557 558 559 560

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
561
static void dxt_serialize_posix_records(void *rec_ref_p)
562
{
Shane Snyder's avatar
Shane Snyder committed
563
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
564
    struct dxt_file_record *file_rec;
565 566 567
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
568
    void *tmp_buf_ptr;
569

Shane Snyder's avatar
Shane Snyder committed
570 571 572
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
573

Shane Snyder's avatar
Shane Snyder committed
574 575 576 577
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
578

Shane Snyder's avatar
Shane Snyder committed
579 580
    /*
     * Buffer format:
581
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
582 583 584
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
585

Shane Snyder's avatar
Shane Snyder committed
586 587
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
588

Shane Snyder's avatar
Shane Snyder committed
589 590 591
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
592

Shane Snyder's avatar
Shane Snyder committed
593
    /*Copy write record */
594
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
595 596 597
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
598

Shane Snyder's avatar
Shane Snyder committed
599
    /*Copy read record */
600
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
601 602 603
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
604

Shane Snyder's avatar
Shane Snyder committed
605
    dxt_posix_runtime->record_buf_size += record_size;
606

Shane Snyder's avatar
Shane Snyder committed
607 608
#if 0
    int i;
Cong Xu's avatar
Cong Xu committed
609 610
    int64_t rank;
    char *hostname;
Shane Snyder's avatar
Shane Snyder committed
611 612 613 614
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
615

Cong Xu's avatar
Cong Xu committed
616 617 618 619 620
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
Shane Snyder's avatar
Shane Snyder committed
621
                file_rec->write_count, file_rec->read_count);
Cong Xu's avatar
Cong Xu committed
622
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
623

Shane Snyder's avatar
Shane Snyder committed
624
    for (i = 0; i < file_rec->write_count; i++) {
625 626 627 628
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
629

Cong Xu's avatar
Cong Xu committed
630
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
631
    }
632

Shane Snyder's avatar
Shane Snyder committed
633
    for (i = 0; i < file_rec->read_count; i++) {
634 635 636 637
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
638

Cong Xu's avatar
Cong Xu committed
639
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
640 641 642
    }
#endif
}
643

Shane Snyder's avatar
Shane Snyder committed
644 645 646 647 648 649 650 651
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
652

Shane Snyder's avatar
Shane Snyder committed
653 654 655 656 657 658 659 660 661 662 663 664 665 666
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
667 668

    /* shutdown internal structures used for instrumenting */
669
    dxt_posix_cleanup_runtime();
670 671 672 673

    return;
}

Shane Snyder's avatar
Shane Snyder committed
674
static void dxt_serialize_mpiio_records(void *rec_ref_p)
675
{
Shane Snyder's avatar
Shane Snyder committed
676
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
677
    struct dxt_file_record *file_rec;
678 679 680
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
681
    void *tmp_buf_ptr;
682

Shane Snyder's avatar
Shane Snyder committed
683 684 685
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
686

Shane Snyder's avatar
Shane Snyder committed
687 688 689 690
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
691

Shane Snyder's avatar
Shane Snyder committed
692 693
    /*
     * Buffer format:
694
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
695 696 697
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
698

Shane Snyder's avatar
Shane Snyder committed
699 700
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
701

Shane Snyder's avatar
Shane Snyder committed
702 703 704
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
705

Shane Snyder's avatar
Shane Snyder committed
706
    /*Copy write record */
707
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
708 709
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
710 711
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
712
    /*Copy read record */
713
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
714 715
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
716 717
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
718
    dxt_mpiio_runtime->record_buf_size += record_size;
719 720

#if 0
Shane Snyder's avatar
Shane Snyder committed
721 722
    int i;
    int64_t rank;
Cong Xu's avatar
Cong Xu committed
723 724
    char *hostname;
    int64_t length;
Shane Snyder's avatar
Shane Snyder committed
725 726
    double start_time;
    double end_time;
727

Cong Xu's avatar
Cong Xu committed
728 729 730 731 732 733 734
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
735

Shane Snyder's avatar
Shane Snyder committed
736
    for (i = 0; i < file_rec->write_count; i++) {
737 738 739
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
740

Cong Xu's avatar
Cong Xu committed
741
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
742 743
    }

Shane Snyder's avatar
Shane Snyder committed
744
    for (i = 0; i < file_rec->read_count; i++) {
745 746 747
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
748

Cong Xu's avatar
Cong Xu committed
749
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

778
    /* shutdown internal structures used for instrumenting */
779
    dxt_mpiio_cleanup_runtime();
780 781 782 783 784 785 786 787 788 789 790 791

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */