darshan-dxt.c 25.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37
38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40
41
42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44
struct dxt_record_ref_tracker
45
46
47
48
49
{
    void *rec_ref_p;
    UT_hash_handle hlink;
};

50
51
52
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
53
54
55
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
56
 * associate different types of handles with this dxt_file_record_ref struct.
57
58
59
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
60
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
61
62
 * there could be multiple open file descriptors that index it.
 */
63
struct dxt_file_record_ref
64
{
65
    struct dxt_file_record *file_rec;
66
67
    int64_t write_available_buf;
    int64_t read_available_buf;
68
69
};

70
71
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
72
73
 * shutdown time.
 */
74
struct dxt_posix_runtime
75
76
77
78
79
{
    void *rec_id_hash;
    int file_rec_count;
};

80
struct dxt_mpiio_runtime
81
82
83
84
85
{
    void *rec_id_hash;
    int file_rec_count;
};

86
87
88
89
90
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
91
92
93
94
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
95
96
97
98
99
100

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
101
    void);
102
static void dxt_mpiio_runtime_initialize(
103
    void);
104
105
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
106
107
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
108
static void dxt_posix_cleanup_runtime(
109
    void);
110
static void dxt_mpiio_cleanup_runtime(
111
112
    void);

113
static void dxt_posix_shutdown(
114
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
115
116
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
117
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
118
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
119

120
121
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
122
static pthread_mutex_t dxt_runtime_mutex =
123
124
125
126
127
128
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;
129
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
130

131
132
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
133
134
135


/**********************************************************
136
 *      Wrappers for DXT I/O functions of interest      *
137
138
 **********************************************************/

139
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
140
{
141
    struct dxt_file_record *file_rec = rec_ref->file_rec;
142

143
    int write_count = file_rec->write_count;
144
    int write_available_buf = rec_ref->write_available_buf;
145
146

    if (write_count >= write_available_buf) {
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
            file_rec->write_traces =
                (segment_info *)realloc(file_rec->write_traces,
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
169
    }
170
171
172
173
174
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
175
176

    int read_count = file_rec->read_count;
177
    int read_available_buf = rec_ref->read_available_buf;
178
179

    if (read_count >= read_available_buf) {
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
            file_rec->read_traces =
                (segment_info *)realloc(file_rec->read_traces,
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
202
203
204
    }
}

205
206
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
207
{
208
209
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
210

211
212
213
214
215
216
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
217
218
    }

219
220
221
222
223
224
225
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
226
227
    }

228
229
230
231
232
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

233
234
235
236
237
238
239
    file_rec->write_traces[file_rec->write_count].offset = offset;
    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

240
241
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
242
{
243
244
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
245

246
247
248
249
250
251
252
253
254
255
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
256
    if (!rec_ref) {
257
258
259
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
260
261
262
    }

    file_rec = rec_ref->file_rec;
263
264
265
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
266
267
268
269
270
271
272
273

    file_rec->read_traces[file_rec->read_count].offset = offset;
    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}

274
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
275
276
        double start_time, double end_time)
{
277
278
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
279

280
281
282
283
284
285
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
286
287
    }

288
289
290
291
292
293
294
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
295
296
    }

297
298
299
300
301
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

302
303
304
305
306
307
    file_rec->write_traces[file_rec->write_count].length = length;
    file_rec->write_traces[file_rec->write_count].start_time = start_time;
    file_rec->write_traces[file_rec->write_count].end_time = end_time;
    file_rec->write_count += 1;
}

308
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
309
310
        double start_time, double end_time)
{
311
312
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
313

314
315
316
317
318
319
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
320
321
    }

322
323
324
325
326
327
328
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
329
330
    }

331
332
333
334
335
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */

336
337
338
339
340
341
342
343
    file_rec->read_traces[file_rec->read_count].length = length;
    file_rec->read_traces[file_rec->read_count].start_time = start_time;
    file_rec->read_traces[file_rec->read_count].end_time = end_time;
    file_rec->read_count += 1;
}


/**********************************************************
344
 * Internal functions for manipulating DXT module state *
345
346
 **********************************************************/

347
/* initialize internal DXT module data structures and register with darshan-core */
348
static void dxt_posix_runtime_initialize()
349
{
350
351
352
353
354
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
355

356
    /* register the DXT module with darshan core */
357
    darshan_core_register_module(
358
359
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
360
        &dxt_psx_buf_size,
361
362
363
        &posix_my_rank,
        &darshan_mem_alignment);

364
365
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
366
    {
367
        darshan_core_unregister_module(DXT_POSIX_MOD);
368
369
370
        return;
    }

371
372
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
373
    {
374
        darshan_core_unregister_module(DXT_POSIX_MOD);
375
376
        return;
    }
377
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
378
379
380
381

    return;
}

382
void dxt_mpiio_runtime_initialize()
383
{
384
385
386
387
388
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
389

390
    /* register the DXT module with darshan core */
391
    darshan_core_register_module(
392
393
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
394
        &dxt_mpiio_buf_size,
395
396
397
        &mpiio_my_rank,
        &darshan_mem_alignment);

398
399
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
400
    {
401
        darshan_core_unregister_module(DXT_MPIIO_MOD);
402
403
404
        return;
    }

405
406
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
407
    {
408
        darshan_core_unregister_module(DXT_MPIIO_MOD);
409
410
        return;
    }
411
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
412
413
414
415

    return;
}

416
417
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
418
{
419
420
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
421
422
    int ret;

423
424
425
426
427
428
429
430
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

431
432
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
433
434
435
436
    {
        DXT_UNLOCK();
        return(NULL);
    }
437
438
    memset(rec_ref, 0, sizeof(*rec_ref));

439
440
441
442
443
444
445
446
447
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

448
    /* add a reference to this file record based on record id */
449
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
450
451
452
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
453
        free(file_rec);
454
        free(rec_ref);
455
456
        DXT_UNLOCK();
        return(NULL);
457
458
    }

459
460
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
461

462
    /* initialize record and record reference fields */
463
464
465
466
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;

    rec_ref->file_rec = file_rec;
467
    dxt_posix_runtime->file_rec_count++;
468
469

    return(rec_ref);
470
471
}

472
473
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
474
{
475
476
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
477
478
    int ret;

479
480
481
482
483
484
485
486
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

487
488
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
489
490
491
492
    {
        DXT_UNLOCK();
        return(NULL);
    }
493
494
    memset(rec_ref, 0, sizeof(*rec_ref));

495
496
497
498
499
500
501
502
503
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

504
    /* add a reference to this file record based on record id */
505
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
506
507
508
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
509
        free(file_rec);
510
        free(rec_ref);
511
        DXT_UNLOCK();
512
513
514
        return;
    }

515
516
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
517

518
    /* initialize record and record reference fields */
519
520
521
522
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;

    rec_ref->file_rec = file_rec;
523
    dxt_mpiio_runtime->file_rec_count++;
524

525
    return(rec_ref);
526
527
}

528
void dxt_clear_record_refs(void **hash_head_p, int free_flag)
529
{
530
531
532
533
534
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        *(struct dxt_record_ref_tracker **)hash_head_p;
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
535

536
#if 0    
537
538
539
540
541
    /* iterate the hash table and remove/free all reference trackers */
    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp)
    {
        HASH_DELETE(hlink, ref_tracker_head, ref_tracker);
        if (free_flag) {
542
            rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
543
544
545
546
547
548
549
550
551
552
553
554
555
            file_rec = rec_ref->file_rec;

            if (file_rec->write_traces)
                free(file_rec->write_traces);

            if (file_rec->read_traces)
                free(file_rec->read_traces);

            free(rec_ref);
        }
        free(ref_tracker);
    }
    *hash_head_p = ref_tracker_head;
556
#endif
557
558
559
560

    return;
}

561
static void dxt_posix_cleanup_runtime()
562
{
563
    dxt_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
564

565
566
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
567
568
569
570

    return;
}

571
static void dxt_mpiio_cleanup_runtime()
572
{
573
    dxt_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
574

575
576
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
577
578
579
580
581
582
583
584
585

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

586
static void dxt_posix_shutdown(
587
588
589
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
590
591
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
592
{
593
594
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
595
596
597
598
599
600
601
602
603
604
605
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
606
    void *tmp_buf_ptr = *dxt_posix_buf;
607

608
    assert(dxt_posix_runtime);
609

610
611
612
613
    int dxt_rec_count = dxt_posix_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_posix_runtime->rec_id_hash);
614

615
    *dxt_posix_buf_sz = 0;
616
617

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
618
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
619
620
621
622
623
624
625
626
627
628
629
630
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;

        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
631
         * dxt_file_record + ost_ids + write_traces + read_traces
632
         */
633
        record_size = sizeof(struct dxt_file_record) +
634
635
                (record_write_count + record_read_count) * sizeof(segment_info);

636
637
        if (*dxt_posix_buf_sz == 0) {
            *dxt_posix_buf = (void *)malloc(record_size);
638
        } else {
639
640
            *dxt_posix_buf = (void *)realloc((*dxt_posix_buf),
                            *dxt_posix_buf_sz + record_size);
641
        }
642
        tmp_buf_ptr = (void *)(*dxt_posix_buf) + *dxt_posix_buf_sz;
643

644
645
646
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
647
648
649
650
651
652
653
654
655
656
657
658
659

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

660
        *dxt_posix_buf_sz += record_size;
661
662

#if 0
663
664
        printf("DXT, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
        printf("DXT, write_count is: %d read_count is: %d\n",
665
                    file_rec->write_count, file_rec->read_count);
666
667
668
669
670
671
672
673

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

674
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
675
676
677
678
679
680
681
682
683
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

684
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
685
686
687
688
689
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
690
    dxt_posix_cleanup_runtime();
691
692
693
694
695
696
697

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

698
static void dxt_mpiio_shutdown(
699
700
701
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
702
703
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
704
{
705
706
    struct dxt_file_record_ref *rec_ref;
    struct dxt_file_record *file_rec;
707
708
709
710
711
712
713
714
715
716
717
    int i, idx;

    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;

    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
718
    void *tmp_buf_ptr = *dxt_mpiio_buf;
719

720
    assert(dxt_mpiio_runtime);
721

722
723
724
725
    int dxt_rec_count = dxt_mpiio_runtime->file_rec_count;
    struct dxt_record_ref_tracker *ref_tracker, *tmp;
    struct dxt_record_ref_tracker *ref_tracker_head =
        (struct dxt_record_ref_tracker *)(dxt_mpiio_runtime->rec_id_hash);
726

727
    *dxt_mpiio_buf_sz = 0;
728
729

    HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp) {
730
        rec_ref = (struct dxt_file_record_ref *)ref_tracker->rec_ref_p;
731
732
733
734
735
736
737
738
739
740
741
        assert(rec_ref);

        file_rec = rec_ref->file_rec;

        record_write_count = file_rec->write_count;
        record_read_count = file_rec->read_count;
        if (record_write_count == 0 && record_read_count == 0)
            continue;

        /*
         * Buffer format:
742
         * dxt_file_record + ost_ids + write_traces + read_traces
743
         */
744
        record_size = sizeof(struct dxt_file_record) +
745
746
                (record_write_count + record_read_count) * sizeof(segment_info);

747
748
        if (*dxt_mpiio_buf_sz == 0) {
            *dxt_mpiio_buf = (void *)malloc(record_size);
749
        } else {
750
751
            *dxt_mpiio_buf = (void *)realloc((*dxt_mpiio_buf),
                            *dxt_mpiio_buf_sz + record_size);
752
        }
753
        tmp_buf_ptr = (void *)(*dxt_mpiio_buf) + *dxt_mpiio_buf_sz;
754

755
756
757
        /*Copy struct dxt_file_record */
        memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) + sizeof(struct dxt_file_record);
758
759
760
761
762
763
764
765
766
767
768
769
770

        /*Copy write record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->write_traces),
                record_write_count * sizeof(segment_info));
        tmp_buf_ptr = ((void *)tmp_buf_ptr) +
                    record_write_count * sizeof(segment_info);

        /*Copy read record */
        memcpy(tmp_buf_ptr, (void *)(file_rec->read_traces),
                record_read_count * sizeof(segment_info));
        tmp_buf_ptr = ((char *)tmp_buf_ptr) +
                    record_read_count * sizeof(segment_info);

771
        *dxt_mpiio_buf_sz += record_size;
772
773
774

#if 0
        printf("Cong, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
775
        printf("DXT, file_rec->write_count is: %d\n",
776
777
778
779
780
781
782
783
784
                    file_rec->write_count);

        for (i = 0; i < file_rec->write_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->write_traces[i].offset;
            length = file_rec->write_traces[i].length;
            start_time = file_rec->write_traces[i].start_time;
            end_time = file_rec->write_traces[i].end_time;

785
            printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
786
787
788
789
790
791
792
793
794
        }

        for (i = 0; i < file_rec->read_count; i++) {
            rank = file_rec->base_rec.rank;
            offset = file_rec->read_traces[i].offset;
            length = file_rec->read_traces[i].length;
            start_time = file_rec->read_traces[i].start_time;
            end_time = file_rec->read_traces[i].end_time;

795
            printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
796
797
798
799
800
        }
#endif
    }

    /* shutdown internal structures used for instrumenting */
801
    dxt_mpiio_cleanup_runtime();
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */