darshan-dxt.c 26.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37
/* maximum amount of memory to use for storing DXT records */
38 39 40 41 42
#ifdef __DARSHAN_MOD_MEM_MAX
#define DXT_IO_TRACE_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
#else
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB default */
#endif
43

44 45 46
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
47

48 49 50 51 52 53 54 55 56 57 58 59
/* XXX: dirty hack -- If DXT runs out of memory to store trace data in,
 * we should set a flag so that log parsers know that the log has
 * incomplete data. This functionality is typically handled automatically
 * when registering records with Darshan, but DXT modules don't
 * register records and manage their own memory. Since DXT modules request
 * 0 memory when registering themselves, any attempt to register a record
 * will result in setting the partial flag for the module, which is
 * exactly what we do here.
 */
#define SET_DXT_MOD_PARTIAL_FLAG(mod_id) \
    darshan_core_register_record(0, NULL, mod_id, 1, NULL);

60 61 62
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
63 64 65
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
66
 * associate different types of handles with this dxt_file_record_ref struct.
67 68 69
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
70
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
71 72
 * there could be multiple open file descriptors that index it.
 */
73
struct dxt_file_record_ref
74
{
75
    struct dxt_file_record *file_rec;
76

77 78
    int64_t write_available_buf;
    int64_t read_available_buf;
79 80 81

    segment_info *write_traces;
    segment_info *read_traces;
82 83
};

84 85
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
86 87
 * shutdown time.
 */
88
struct dxt_posix_runtime
89 90 91
{
    void *rec_id_hash;
    int file_rec_count;
92 93
    char *record_buf;
    int record_buf_size;
94 95
};

96
struct dxt_mpiio_runtime
97 98 99
{
    void *rec_id_hash;
    int file_rec_count;
100 101
    char *record_buf;
    int record_buf_size;
102 103
};

104 105 106 107 108
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
109 110 111 112
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
113 114 115 116 117 118

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
119
    void);
120
static void dxt_mpiio_runtime_initialize(
121
    void);
122 123
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
124 125
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
126
static void dxt_posix_cleanup_runtime(
127
    void);
128
static void dxt_mpiio_cleanup_runtime(
129 130
    void);

131
static void dxt_posix_shutdown(
132
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
133 134
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
135
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
136
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
137

138 139
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
140
static pthread_mutex_t dxt_runtime_mutex =
141 142
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

143
static int dxt_my_rank = -1;
144
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
145

146 147
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
148 149 150


/**********************************************************
151
 *      Wrappers for DXT I/O functions of interest      *
152 153
 **********************************************************/

154
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
155
{
156
    struct dxt_file_record *file_rec = rec_ref->file_rec;
157

158
    int write_count = file_rec->write_count;
159
    int write_available_buf = rec_ref->write_available_buf;
160 161

    if (write_count >= write_available_buf) {
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
178 179
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
180 181 182 183
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
184
    }
185 186 187 188 189
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
190 191

    int read_count = file_rec->read_count;
192
    int read_available_buf = rec_ref->read_available_buf;
193 194

    if (read_count >= read_available_buf) {
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
211 212
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
213 214 215 216
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
217 218 219
    }
}

220 221
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
222
{
223 224
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
225

226 227 228 229 230
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
231 232
    }

233 234 235 236 237 238 239
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
240 241
    }

242 243 244
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
245 246 247 248 249
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
250

251 252 253 254
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
255 256 257
    file_rec->write_count += 1;
}

258 259
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
260
{
261 262
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
263

264 265 266 267 268 269 270 271 272
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
273
    if (!rec_ref) {
274 275 276
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
277 278 279
    }

    file_rec = rec_ref->file_rec;
280 281
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
282 283 284 285 286
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
287

288 289 290 291
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
292 293 294
    file_rec->read_count += 1;
}

295
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
296 297
        double start_time, double end_time)
{
298 299
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
300

301 302 303 304 305
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
306 307
    }

308 309 310 311 312 313 314
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
315 316
    }

317 318 319
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
320 321 322 323 324
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
325

326 327 328
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
329 330 331
    file_rec->write_count += 1;
}

332
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
333 334
        double start_time, double end_time)
{
335 336
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
337

338 339 340 341 342
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
343 344
    }

345 346 347 348 349 350 351
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
352 353
    }

354 355 356
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
357 358 359 360 361
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
362

363 364 365
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
366 367 368 369 370
    file_rec->read_count += 1;
}


/**********************************************************
371
 * Internal functions for manipulating DXT module state *
372 373
 **********************************************************/

374
/* initialize internal DXT module data structures and register with darshan-core */
375
static void dxt_posix_runtime_initialize()
376
{
377 378 379 380 381
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
382 383
    int ret, tmpval;
    char *envstr;
384

385
    /* register the DXT module with darshan core */
386
    darshan_core_register_module(
387 388
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
389
        &dxt_psx_buf_size,
390 391
        &dxt_my_rank,
        NULL);
392

393 394
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
395
    {
396
        darshan_core_unregister_module(DXT_POSIX_MOD);
397 398 399
        return;
    }

400
    DXT_LOCK();
401 402
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
403
    {
404
        darshan_core_unregister_module(DXT_POSIX_MOD);
405
        DXT_UNLOCK();
406 407
        return;
    }
408
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
409

410 411 412 413 414 415 416 417 418 419 420 421 422
    /* set the memory quota for DXT, if it has not been initialized */
    envstr = getenv("ENABLE_DXT_IO_TRACE_MEM");
    if(envstr && dxt_mpiio_runtime == NULL)
    {
        ret = sscanf(envstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            dxt_mem_remaining = tmpval * 1024 * 1024; /* convert from MiB */
        }
    }
    DXT_UNLOCK();

423 424 425
    return;
}

426
void dxt_mpiio_runtime_initialize()
427
{
428 429 430 431 432
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
433 434
    int ret, tmpval;
    char *envstr;
435

436
    /* register the DXT module with darshan core */
437
    darshan_core_register_module(
438 439
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
440
        &dxt_mpiio_buf_size,
441 442
        &dxt_my_rank,
        NULL);
443

444 445
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
446
    {
447
        darshan_core_unregister_module(DXT_MPIIO_MOD);
448 449 450
        return;
    }

451
    DXT_LOCK();
452 453
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
454
    {
455
        darshan_core_unregister_module(DXT_MPIIO_MOD);
456
        DXT_UNLOCK();
457 458
        return;
    }
459
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
460

461 462 463 464 465 466 467 468 469 470 471 472 473
    /* set the memory quota for DXT, if it has not been initialized */
    envstr = getenv("ENABLE_DXT_IO_TRACE_MEM");
    if(envstr && dxt_posix_runtime == NULL)
    {
        ret = sscanf(envstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            dxt_mem_remaining = tmpval * 1024 * 1024; /* convert from MiB */
        }
    }
    DXT_UNLOCK();

474 475 476
    return;
}

477 478
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
479
{
480 481
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
482 483
    int ret;

484 485 486 487
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
488
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
489 490 491 492
        DXT_UNLOCK();
        return(NULL);
    }

493 494
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
495 496 497 498
    {
        DXT_UNLOCK();
        return(NULL);
    }
499 500
    memset(rec_ref, 0, sizeof(*rec_ref));

501 502 503 504 505 506 507 508 509
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

510
    /* add a reference to this file record based on record id */
511
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
512 513 514
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
515
        free(file_rec);
516
        free(rec_ref);
517 518
        DXT_UNLOCK();
        return(NULL);
519 520
    }

521 522
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
523

524
    /* initialize record and record reference fields */
525
    file_rec->base_rec.id = rec_id;
526
    file_rec->base_rec.rank = dxt_my_rank;
527
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
528 529

    rec_ref->file_rec = file_rec;
530
    dxt_posix_runtime->file_rec_count++;
531 532

    return(rec_ref);
533 534
}

535 536
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
537
{
538 539
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
540 541
    int ret;

542 543 544 545
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
546
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
547 548 549 550
        DXT_UNLOCK();
        return(NULL);
    }

551 552
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
553 554 555 556
    {
        DXT_UNLOCK();
        return(NULL);
    }
557 558
    memset(rec_ref, 0, sizeof(*rec_ref));

559 560 561 562 563 564 565 566 567
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

568
    /* add a reference to this file record based on record id */
569
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
570 571 572
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
573
        free(file_rec);
574
        free(rec_ref);
575
        DXT_UNLOCK();
576
        return(NULL);
577 578
    }

579 580
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
581

582
    /* initialize record and record reference fields */
583
    file_rec->base_rec.id = rec_id;
584
    file_rec->base_rec.rank = dxt_my_rank;
585
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
586 587

    rec_ref->file_rec = file_rec;
588
    dxt_mpiio_runtime->file_rec_count++;
589

590
    return(rec_ref);
591 592
}

593
static void dxt_free_record_data(void *rec_ref_p)
594
{
595
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
596

597 598
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
599
    free(dxt_rec_ref->file_rec);
600 601
}

602
static void dxt_posix_cleanup_runtime()
603
{
604
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
605
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
606

607 608
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
609 610 611 612

    return;
}

613
static void dxt_mpiio_cleanup_runtime()
614
{
615
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
616
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
617

618 619
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
620 621 622 623 624 625 626 627 628

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

629
static void dxt_serialize_posix_records(void *rec_ref_p)
630
{
631
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
632
    struct dxt_file_record *file_rec;
633 634 635
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
636
    void *tmp_buf_ptr;
637

638 639 640
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
641

642 643 644 645
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
646

647 648
    /*
     * Buffer format:
649
     * dxt_file_record + write_traces + read_traces
650 651 652
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
653

654 655
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
656

657 658 659
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
660

661
    /*Copy write record */
662
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
663 664 665
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
666

667
    /*Copy read record */
668
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
669 670 671
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
672

673
    dxt_posix_runtime->record_buf_size += record_size;
674

675 676
#if 0
    int i;
677 678
    int64_t rank;
    char *hostname;
679 680 681 682
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
683

684 685 686 687 688
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
689
                file_rec->write_count, file_rec->read_count);
690
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
691

692
    for (i = 0; i < file_rec->write_count; i++) {
693 694 695 696
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
697

698
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
699
    }
700

701
    for (i = 0; i < file_rec->read_count; i++) {
702 703 704 705
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
706

707
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
708 709 710
    }
#endif
}
711

712 713 714 715 716 717 718 719
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
720

721 722 723 724 725 726 727 728 729 730 731 732 733 734
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
735 736

    /* shutdown internal structures used for instrumenting */
737
    dxt_posix_cleanup_runtime();
738 739 740 741

    return;
}

742
static void dxt_serialize_mpiio_records(void *rec_ref_p)
743
{
744
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
745
    struct dxt_file_record *file_rec;
746 747 748
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
749
    void *tmp_buf_ptr;
750

751 752 753
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
754

755 756 757 758
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
759

760 761
    /*
     * Buffer format:
762
     * dxt_file_record + write_traces + read_traces
763 764 765
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
766

767 768
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
769

770 771 772
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
773

774
    /*Copy write record */
775
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
776 777
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
778 779
                record_write_count * sizeof(segment_info));

780
    /*Copy read record */
781
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
782 783
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
784 785
                record_read_count * sizeof(segment_info));

786
    dxt_mpiio_runtime->record_buf_size += record_size;
787 788

#if 0
789 790
    int i;
    int64_t rank;
791 792
    char *hostname;
    int64_t length;
793 794
    double start_time;
    double end_time;
795

796 797 798 799 800 801 802
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
803

804
    for (i = 0; i < file_rec->write_count; i++) {
805 806 807
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
808

809
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
810 811
    }

812
    for (i = 0; i < file_rec->read_count; i++) {
813 814 815
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
816

817
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

846
    /* shutdown internal structures used for instrumenting */
847
    dxt_mpiio_cleanup_runtime();
848 849 850 851 852 853 854 855 856 857 858 859

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */