darshan-dxt.c 25.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37 38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40 41 42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44 45 46 47 48 49 50 51 52 53 54 55
/* XXX: dirty hack -- If DXT runs out of memory to store trace data in,
 * we should set a flag so that log parsers know that the log has
 * incomplete data. This functionality is typically handled automatically
 * when registering records with Darshan, but DXT modules don't
 * register records and manage their own memory. Since DXT modules request
 * 0 memory when registering themselves, any attempt to register a record
 * will result in setting the partial flag for the module, which is
 * exactly what we do here.
 */
#define SET_DXT_MOD_PARTIAL_FLAG(mod_id) \
    darshan_core_register_record(0, NULL, mod_id, 1, NULL);

56 57 58
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
59 60 61
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
62
 * associate different types of handles with this dxt_file_record_ref struct.
63 64 65
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
66
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
67 68
 * there could be multiple open file descriptors that index it.
 */
69
struct dxt_file_record_ref
70
{
71
    struct dxt_file_record *file_rec;
72

73 74
    int64_t write_available_buf;
    int64_t read_available_buf;
75 76 77

    segment_info *write_traces;
    segment_info *read_traces;
78 79
};

80 81
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
82 83
 * shutdown time.
 */
84
struct dxt_posix_runtime
85 86 87
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
88 89
    char *record_buf;
    int record_buf_size;
90 91
};

92
struct dxt_mpiio_runtime
93 94 95
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
96 97
    char *record_buf;
    int record_buf_size;
98 99
};

100 101 102 103 104
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
105 106 107 108
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
109 110 111 112 113 114

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
115
    void);
116
static void dxt_mpiio_runtime_initialize(
117
    void);
118 119
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
120 121
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
122
static void dxt_posix_cleanup_runtime(
123
    void);
124
static void dxt_mpiio_cleanup_runtime(
125 126
    void);

127
static void dxt_posix_shutdown(
128
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
129 130
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
131
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
132
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
133

134 135
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
136
static pthread_mutex_t dxt_runtime_mutex =
137 138
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

139
static int dxt_my_rank = -1;
140
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
141

142 143
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
144 145 146


/**********************************************************
147
 *      Wrappers for DXT I/O functions of interest      *
148 149
 **********************************************************/

150
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
151
{
152
    struct dxt_file_record *file_rec = rec_ref->file_rec;
153

154
    int write_count = file_rec->write_count;
155
    int write_available_buf = rec_ref->write_available_buf;
156 157

    if (write_count >= write_available_buf) {
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
174 175
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
176 177 178 179
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
180
    }
181 182 183 184 185
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
186 187

    int read_count = file_rec->read_count;
188
    int read_available_buf = rec_ref->read_available_buf;
189 190

    if (read_count >= read_available_buf) {
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
207 208
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
209 210 211 212
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
213 214 215
    }
}

216 217
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
218
{
219 220
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
221

222 223 224 225 226
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
227 228
    }

229 230 231 232 233 234 235
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
236 237
    }

238 239 240
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
241 242 243 244 245
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
246

247 248 249 250
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
251 252 253
    file_rec->write_count += 1;
}

254 255
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
256
{
257 258
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
259

260 261 262 263 264 265 266 267 268
    /* make sure dxt posix runtime is initialized properly */
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
269
    if (!rec_ref) {
270 271 272
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
273 274 275
    }

    file_rec = rec_ref->file_rec;
276 277
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
278 279 280 281 282
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
        return;
    }
283

284 285 286 287
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
288 289 290
    file_rec->read_count += 1;
}

291
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
292 293
        double start_time, double end_time)
{
294 295
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
296

297 298 299 300 301
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
302 303
    }

304 305 306 307 308 309 310
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
311 312
    }

313 314 315
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
316 317 318 319 320
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
321

322 323 324
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
325 326 327
    file_rec->write_count += 1;
}

328
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
329 330
        double start_time, double end_time)
{
331 332
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
333

334 335 336 337 338
    /* make sure dxt mpiio runtime is initialized properly */
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
339 340
    }

341 342 343 344 345 346 347
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
348 349
    }

350 351 352
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
353 354 355 356 357
    {
        /* no more memory for i/o segments ... back out */
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
        return;
    }
358

359 360 361
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
362 363 364 365 366
    file_rec->read_count += 1;
}


/**********************************************************
367
 * Internal functions for manipulating DXT module state *
368 369
 **********************************************************/

370
/* initialize internal DXT module data structures and register with darshan-core */
371
static void dxt_posix_runtime_initialize()
372
{
373 374 375 376 377
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
378

379
    /* register the DXT module with darshan core */
380
    darshan_core_register_module(
381 382
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
383
        &dxt_psx_buf_size,
384 385
        &dxt_my_rank,
        NULL);
386

387 388
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
389
    {
390
        darshan_core_unregister_module(DXT_POSIX_MOD);
391 392 393
        return;
    }

394 395
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
396
    {
397
        darshan_core_unregister_module(DXT_POSIX_MOD);
398 399
        return;
    }
400
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
401 402 403 404

    return;
}

405
void dxt_mpiio_runtime_initialize()
406
{
407 408 409 410 411
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
412

413
    /* register the DXT module with darshan core */
414
    darshan_core_register_module(
415 416
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
417
        &dxt_mpiio_buf_size,
418 419
        &dxt_my_rank,
        NULL);
420

421 422
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
423
    {
424
        darshan_core_unregister_module(DXT_MPIIO_MOD);
425 426 427
        return;
    }

428 429
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
430
    {
431
        darshan_core_unregister_module(DXT_MPIIO_MOD);
432 433
        return;
    }
434
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
435 436 437 438

    return;
}

439 440
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
441
{
442 443
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
444 445
    int ret;

446 447 448 449
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
450
        SET_DXT_MOD_PARTIAL_FLAG(DXT_POSIX_MOD);
451 452 453 454
        DXT_UNLOCK();
        return(NULL);
    }

455 456
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
457 458 459 460
    {
        DXT_UNLOCK();
        return(NULL);
    }
461 462
    memset(rec_ref, 0, sizeof(*rec_ref));

463 464 465 466 467 468 469 470 471
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

472
    /* add a reference to this file record based on record id */
473
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
474 475 476
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
477
        free(file_rec);
478
        free(rec_ref);
479 480
        DXT_UNLOCK();
        return(NULL);
481 482
    }

483 484
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
485

486
    /* initialize record and record reference fields */
487
    file_rec->base_rec.id = rec_id;
488
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
489
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
490 491

    rec_ref->file_rec = file_rec;
492
    dxt_posix_runtime->file_rec_count++;
493 494

    return(rec_ref);
495 496
}

497 498
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
499
{
500 501
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
502 503
    int ret;

504 505 506 507
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
508
        SET_DXT_MOD_PARTIAL_FLAG(DXT_MPIIO_MOD);
509 510 511 512
        DXT_UNLOCK();
        return(NULL);
    }

513 514
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
515 516 517 518
    {
        DXT_UNLOCK();
        return(NULL);
    }
519 520
    memset(rec_ref, 0, sizeof(*rec_ref));

521 522 523 524 525 526 527 528 529
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

530
    /* add a reference to this file record based on record id */
531
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
532 533 534
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
535
        free(file_rec);
536
        free(rec_ref);
537
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
538
        return(NULL);
539 540
    }

541 542
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
543

544
    /* initialize record and record reference fields */
545
    file_rec->base_rec.id = rec_id;
546
    file_rec->base_rec.rank = dxt_my_rank;
Cong Xu's avatar
Cong Xu committed
547
    gethostname(file_rec->hostname, HOSTNAME_SIZE);
548 549

    rec_ref->file_rec = file_rec;
550
    dxt_mpiio_runtime->file_rec_count++;
551

552
    return(rec_ref);
553 554
}

Shane Snyder's avatar
Shane Snyder committed
555
static void dxt_free_record_data(void *rec_ref_p)
556
{
557
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
558

559 560
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
561
    free(dxt_rec_ref->file_rec);
562 563
}

564
static void dxt_posix_cleanup_runtime()
565
{
Shane Snyder's avatar
Shane Snyder committed
566
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
567
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
568

569 570
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
571 572 573 574

    return;
}

575
static void dxt_mpiio_cleanup_runtime()
576
{
Shane Snyder's avatar
Shane Snyder committed
577
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
578
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
579

580 581
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
582 583 584 585 586 587 588 589 590

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
591
static void dxt_serialize_posix_records(void *rec_ref_p)
592
{
Shane Snyder's avatar
Shane Snyder committed
593
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
594
    struct dxt_file_record *file_rec;
595 596 597
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
598
    void *tmp_buf_ptr;
599

Shane Snyder's avatar
Shane Snyder committed
600 601 602
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
603

Shane Snyder's avatar
Shane Snyder committed
604 605 606 607
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
608

Shane Snyder's avatar
Shane Snyder committed
609 610
    /*
     * Buffer format:
611
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
612 613 614
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
615

Shane Snyder's avatar
Shane Snyder committed
616 617
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
618

Shane Snyder's avatar
Shane Snyder committed
619 620 621
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
622

Shane Snyder's avatar
Shane Snyder committed
623
    /*Copy write record */
624
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
625 626 627
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
628

Shane Snyder's avatar
Shane Snyder committed
629
    /*Copy read record */
630
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
631 632 633
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
634

Shane Snyder's avatar
Shane Snyder committed
635
    dxt_posix_runtime->record_buf_size += record_size;
636

Shane Snyder's avatar
Shane Snyder committed
637 638
#if 0
    int i;
Cong Xu's avatar
Cong Xu committed
639 640
    int64_t rank;
    char *hostname;
Shane Snyder's avatar
Shane Snyder committed
641 642 643 644
    int64_t offset;
    int64_t length;
    double start_time;
    double end_time;
645

Cong Xu's avatar
Cong Xu committed
646 647 648 649 650
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_POSIX, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_POSIX, write_count is: %d read_count is: %d\n",
Shane Snyder's avatar
Shane Snyder committed
651
                file_rec->write_count, file_rec->read_count);
Cong Xu's avatar
Cong Xu committed
652
    printf("X_POSIX, rank: %d hostname: %s\n", rank, hostname);
653

Shane Snyder's avatar
Shane Snyder committed
654
    for (i = 0; i < file_rec->write_count; i++) {
655 656 657 658
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
659

Cong Xu's avatar
Cong Xu committed
660
        printf("X_POSIX, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
661
    }
662

Shane Snyder's avatar
Shane Snyder committed
663
    for (i = 0; i < file_rec->read_count; i++) {
664 665 666 667
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
668

Cong Xu's avatar
Cong Xu committed
669
        printf("X_POSIX, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
670 671 672
    }
#endif
}
673

Shane Snyder's avatar
Shane Snyder committed
674 675 676 677 678 679 680 681
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
682

Shane Snyder's avatar
Shane Snyder committed
683 684 685 686 687 688 689 690 691 692 693 694 695 696
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
697 698

    /* shutdown internal structures used for instrumenting */
699
    dxt_posix_cleanup_runtime();
700 701 702 703

    return;
}

Shane Snyder's avatar
Shane Snyder committed
704
static void dxt_serialize_mpiio_records(void *rec_ref_p)
705
{
Shane Snyder's avatar
Shane Snyder committed
706
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
707
    struct dxt_file_record *file_rec;
708 709 710
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
711
    void *tmp_buf_ptr;
712

Shane Snyder's avatar
Shane Snyder committed
713 714 715
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
716

Shane Snyder's avatar
Shane Snyder committed
717 718 719 720
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
721

Shane Snyder's avatar
Shane Snyder committed
722 723
    /*
     * Buffer format:
724
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
725 726 727
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
728

Shane Snyder's avatar
Shane Snyder committed
729 730
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
731

Shane Snyder's avatar
Shane Snyder committed
732 733 734
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
735

Shane Snyder's avatar
Shane Snyder committed
736
    /*Copy write record */
737
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
738 739
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
740 741
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
742
    /*Copy read record */
743
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
744 745
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
746 747
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
748
    dxt_mpiio_runtime->record_buf_size += record_size;
749 750

#if 0
Shane Snyder's avatar
Shane Snyder committed
751 752
    int i;
    int64_t rank;
Cong Xu's avatar
Cong Xu committed
753 754
    char *hostname;
    int64_t length;
Shane Snyder's avatar
Shane Snyder committed
755 756
    double start_time;
    double end_time;
757

Cong Xu's avatar
Cong Xu committed
758 759 760 761 762 763 764
    rank = file_rec->base_rec.rank;
    hostname = file_rec->hostname;

    printf("X_MPIIO, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("X_MPIIO, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
    printf("X_MPIIO, rank: %d hostname: %s\n", rank, hostname);
765

Shane Snyder's avatar
Shane Snyder committed
766
    for (i = 0; i < file_rec->write_count; i++) {
767 768 769
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
770

Cong Xu's avatar
Cong Xu committed
771
        printf("X_MPIIO, rank %d writes segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
772 773
    }

Shane Snyder's avatar
Shane Snyder committed
774
    for (i = 0; i < file_rec->read_count; i++) {
775 776 777
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
778

Cong Xu's avatar
Cong Xu committed
779
        printf("X_MPIIO, rank %d reads segment %lld [length: %lld start_time: %fs end_time: %fs]\n", rank, i, length, start_time, end_time);
Shane Snyder's avatar
Shane Snyder committed
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

808
    /* shutdown internal structures used for instrumenting */
809
    dxt_mpiio_cleanup_runtime();
810 811 812 813 814 815 816 817 818 819 820 821

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */