darshan-dxt.c 24.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/*
 * Copyright (C) 2016 Intel Corporation.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
#include <libgen.h>
#include <pthread.h>

#include "utlist.h"
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"

#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
#endif

37
38
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024) /* 4 MiB */
39

40
41
42
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
#define IO_TRACE_BUF_SIZE       64
43

44
45
46
/* The dxt_file_record_ref structure maintains necessary runtime metadata
 * for the DXT file record (dxt_file_record structure, defined in
 * darshan-dxt-log-format.h) pointed to by 'file_rec'. This metadata
47
48
49
 * assists with the instrumenting of specific statistics in the file record.
 *
 * NOTE: we use the 'darshan_record_ref' interface (in darshan-common) to
50
 * associate different types of handles with this dxt_file_record_ref struct.
51
52
53
 * This allows us to index this struct (and the underlying file record) by using
 * either the corresponding Darshan record identifier (derived from the filename)
 * or by a generated file descriptor, for instance. Note that, while there should
54
 * only be a single Darshan record identifier that indexes a dxt_file_record_ref,
55
56
 * there could be multiple open file descriptors that index it.
 */
57
struct dxt_file_record_ref
58
{
59
    struct dxt_file_record *file_rec;
60

61
62
    int64_t write_available_buf;
    int64_t read_available_buf;
63
64
65

    segment_info *write_traces;
    segment_info *read_traces;
66
67
};

68
69
/* The dxt_runtime structure maintains necessary state for storing
 * DXT file records and for coordinating with darshan-core at
70
71
 * shutdown time.
 */
72
struct dxt_posix_runtime
73
74
75
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
76
77
    char *record_buf;
    int record_buf_size;
78
79
};

80
struct dxt_mpiio_runtime
81
82
83
{
    void *rec_id_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
84
85
    char *record_buf;
    int record_buf_size;
86
87
};

88
89
90
91
92
/* dxt read/write instrumentation wrappers for POSIX and MPI-IO */
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time);
93
94
95
96
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
        double start_time, double end_time);
97
98
99
100
101
102

static void check_wr_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void check_rd_trace_buf(
    struct dxt_file_record_ref *rec_ref);
static void dxt_posix_runtime_initialize(
103
    void);
104
static void dxt_mpiio_runtime_initialize(
105
    void);
106
107
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id);
108
109
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id);
110
static void dxt_posix_cleanup_runtime(
111
    void);
112
static void dxt_mpiio_cleanup_runtime(
113
114
    void);

115
static void dxt_posix_shutdown(
116
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
117
118
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
static void dxt_mpiio_shutdown(
119
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
120
    int shared_rec_count, void **dxt_buf, int *dxt_buf_sz);
121

122
123
static struct dxt_posix_runtime *dxt_posix_runtime = NULL;
static struct dxt_mpiio_runtime *dxt_mpiio_runtime = NULL;
124
static pthread_mutex_t dxt_runtime_mutex =
125
126
127
128
129
130
            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

static int posix_my_rank = -1;
static int mpiio_my_rank = -1;
static int instrumentation_disabled = 0;
static int darshan_mem_alignment = 1;
131
static int dxt_mem_remaining = DXT_IO_TRACE_MEM_MAX;
132

133
134
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
135
136
137


/**********************************************************
138
 *      Wrappers for DXT I/O functions of interest      *
139
140
 **********************************************************/

141
static void check_wr_trace_buf(struct dxt_file_record_ref *rec_ref)
142
{
143
    struct dxt_file_record *file_rec = rec_ref->file_rec;
144

145
    int write_count = file_rec->write_count;
146
    int write_available_buf = rec_ref->write_available_buf;
147
148

    if (write_count >= write_available_buf) {
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        int write_count_inc;
        if(write_available_buf == 0)
            write_count_inc = IO_TRACE_BUF_SIZE;
        else
            write_count_inc = write_available_buf;

        DXT_LOCK();
        if((write_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            write_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (write_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(write_count_inc > 0)
        {
            write_available_buf += write_count_inc;
165
166
            rec_ref->write_traces =
                (segment_info *)realloc(rec_ref->write_traces,
167
168
169
170
                        write_available_buf * sizeof(segment_info));

            rec_ref->write_available_buf = write_available_buf;
        }
171
    }
172
173
174
175
176
}

static void check_rd_trace_buf(struct dxt_file_record_ref *rec_ref)
{
    struct dxt_file_record *file_rec = rec_ref->file_rec;
177
178

    int read_count = file_rec->read_count;
179
    int read_available_buf = rec_ref->read_available_buf;
180
181

    if (read_count >= read_available_buf) {
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
        int read_count_inc;
        if(read_available_buf == 0)
            read_count_inc = IO_TRACE_BUF_SIZE;
        else
            read_count_inc = read_available_buf;

        DXT_LOCK();
        if((read_count_inc * sizeof(segment_info)) > dxt_mem_remaining)
            read_count_inc = dxt_mem_remaining / sizeof(segment_info);

        dxt_mem_remaining -= (read_count_inc * sizeof(segment_info));
        DXT_UNLOCK();

        if(read_count_inc > 0)
        {
            read_available_buf += read_count_inc;
198
199
            rec_ref->read_traces =
                (segment_info *)realloc(rec_ref->read_traces,
200
201
202
203
                        read_available_buf * sizeof(segment_info));
            
            rec_ref->read_available_buf = read_available_buf;
        }
204
205
206
    }
}

207
208
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
209
{
210
211
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
212

213
214
215
216
217
218
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
219
220
    }

221
222
223
224
225
226
227
    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
        &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
228
229
    }

230
231
232
233
234
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

235
236
237
238
    rec_ref->write_traces[file_rec->write_count].offset = offset;
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
239
240
241
    file_rec->write_count += 1;
}

242
243
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
        int64_t length, double start_time, double end_time)
244
{
245
246
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
247

248
249
250
251
252
253
254
255
256
257
    /* make sure dxt posix runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_posix_runtime)
    {
        dxt_posix_runtime_initialize();
        if(!dxt_posix_runtime) return;
    }

    rec_ref = darshan_lookup_record_ref(dxt_posix_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
258
    if (!rec_ref) {
259
260
261
        /* track new dxt file record */
        rec_ref = dxt_posix_track_new_file_record(rec_id);
        if(!rec_ref) return;
262
263
264
    }

    file_rec = rec_ref->file_rec;
265
266
267
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */
268

269
270
271
272
    rec_ref->read_traces[file_rec->read_count].offset = offset;
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
273
274
275
    file_rec->read_count += 1;
}

276
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
277
278
        double start_time, double end_time)
{
279
280
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
281

282
283
284
285
286
287
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
288
289
    }

290
291
292
293
294
295
296
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
297
298
    }

299
300
301
302
303
    file_rec = rec_ref->file_rec;
    check_wr_trace_buf(rec_ref);
    if(file_rec->write_count == rec_ref->write_available_buf)
        return; /* no more memory for i/o segments ... back out */

304
305
306
    rec_ref->write_traces[file_rec->write_count].length = length;
    rec_ref->write_traces[file_rec->write_count].start_time = start_time;
    rec_ref->write_traces[file_rec->write_count].end_time = end_time;
307
308
309
    file_rec->write_count += 1;
}

310
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
311
312
        double start_time, double end_time)
{
313
314
    struct dxt_file_record_ref* rec_ref = NULL;
    struct dxt_file_record *file_rec;
315

316
317
318
319
320
321
    /* make sure dxt mpiio runtime is initialized properly */
    if(instrumentation_disabled) return;
    if(!dxt_mpiio_runtime)
    {
        dxt_mpiio_runtime_initialize();
        if(!dxt_mpiio_runtime) return;
322
323
    }

324
325
326
327
328
329
330
    rec_ref = darshan_lookup_record_ref(dxt_mpiio_runtime->rec_id_hash,
                &rec_id, sizeof(darshan_record_id));
    if(!rec_ref)
    {
        /* track new dxt file record */
        rec_ref = dxt_mpiio_track_new_file_record(rec_id);
        if(!rec_ref) return;
331
332
    }

333
334
335
336
337
    file_rec = rec_ref->file_rec;
    check_rd_trace_buf(rec_ref);
    if(file_rec->read_count == rec_ref->read_available_buf)
        return; /* no more memory for i/o segments ... back out */

338
339
340
    rec_ref->read_traces[file_rec->read_count].length = length;
    rec_ref->read_traces[file_rec->read_count].start_time = start_time;
    rec_ref->read_traces[file_rec->read_count].end_time = end_time;
341
342
343
344
345
    file_rec->read_count += 1;
}


/**********************************************************
346
 * Internal functions for manipulating DXT module state *
347
348
 **********************************************************/

349
/* initialize internal DXT module data structures and register with darshan-core */
350
static void dxt_posix_runtime_initialize()
351
{
352
353
354
355
356
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_psx_buf_size = 0;
357

358
    /* register the DXT module with darshan core */
359
    darshan_core_register_module(
360
361
        DXT_POSIX_MOD,
        &dxt_posix_shutdown,
362
        &dxt_psx_buf_size,
363
364
365
        &posix_my_rank,
        &darshan_mem_alignment);

366
367
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_psx_buf_size != 0)
368
    {
369
        darshan_core_unregister_module(DXT_POSIX_MOD);
370
371
372
        return;
    }

373
374
    dxt_posix_runtime = malloc(sizeof(*dxt_posix_runtime));
    if(!dxt_posix_runtime)
375
    {
376
        darshan_core_unregister_module(DXT_POSIX_MOD);
377
378
        return;
    }
379
    memset(dxt_posix_runtime, 0, sizeof(*dxt_posix_runtime));
380
381
382
383

    return;
}

384
void dxt_mpiio_runtime_initialize()
385
{
386
387
388
389
390
    /* DXT modules request 0 memory -- buffers will be managed internally by DXT
     * and passed back to darshan-core at shutdown time to allow DXT more control
     * over realloc'ing module memory as needed.
     */
    int dxt_mpiio_buf_size = 0;
391

392
    /* register the DXT module with darshan core */
393
    darshan_core_register_module(
394
395
        DXT_MPIIO_MOD,
        &dxt_mpiio_shutdown,
396
        &dxt_mpiio_buf_size,
397
398
399
        &mpiio_my_rank,
        &darshan_mem_alignment);

400
401
    /* return if darshan-core allocates an unexpected amount of memory */
    if(dxt_mpiio_buf_size != 0)
402
    {
403
        darshan_core_unregister_module(DXT_MPIIO_MOD);
404
405
406
        return;
    }

407
408
    dxt_mpiio_runtime = malloc(sizeof(*dxt_mpiio_runtime));
    if(!dxt_mpiio_runtime)
409
    {
410
        darshan_core_unregister_module(DXT_MPIIO_MOD);
411
412
        return;
    }
413
    memset(dxt_mpiio_runtime, 0, sizeof(*dxt_mpiio_runtime));
414
415
416
417

    return;
}

418
419
static struct dxt_file_record_ref *dxt_posix_track_new_file_record(
    darshan_record_id rec_id)
420
{
421
422
    struct dxt_file_record_ref *rec_ref = NULL;
    struct dxt_file_record *file_rec = NULL;
423
424
    int ret;

425
426
427
428
429
430
431
432
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

433
434
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
435
436
437
438
    {
        DXT_UNLOCK();
        return(NULL);
    }
439
440
    memset(rec_ref, 0, sizeof(*rec_ref));

441
442
443
444
445
446
447
448
449
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

450
    /* add a reference to this file record based on record id */
451
    ret = darshan_add_record_ref(&(dxt_posix_runtime->rec_id_hash), &rec_id,
452
453
454
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
455
        free(file_rec);
456
        free(rec_ref);
457
458
        DXT_UNLOCK();
        return(NULL);
459
460
    }

461
462
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
463

464
    /* initialize record and record reference fields */
465
466
467
468
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = posix_my_rank;

    rec_ref->file_rec = file_rec;
469
    dxt_posix_runtime->file_rec_count++;
470
471

    return(rec_ref);
472
473
}

474
475
static struct dxt_file_record_ref *dxt_mpiio_track_new_file_record(
    darshan_record_id rec_id)
476
{
477
478
    struct dxt_file_record *file_rec = NULL;
    struct dxt_file_record_ref *rec_ref = NULL;
479
480
    int ret;

481
482
483
484
485
486
487
488
    /* check if we have enough room for a new DXT record */
    DXT_LOCK();
    if(dxt_mem_remaining < sizeof(struct dxt_file_record))
    {
        DXT_UNLOCK();
        return(NULL);
    }

489
490
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
491
492
493
494
    {
        DXT_UNLOCK();
        return(NULL);
    }
495
496
    memset(rec_ref, 0, sizeof(*rec_ref));

497
498
499
500
501
502
503
504
505
    file_rec = malloc(sizeof(*file_rec));
    if(!file_rec)
    {
        free(rec_ref);
        DXT_UNLOCK();
        return(NULL);
    }
    memset(file_rec, 0, sizeof(*file_rec));

506
    /* add a reference to this file record based on record id */
507
    ret = darshan_add_record_ref(&(dxt_mpiio_runtime->rec_id_hash), &rec_id,
508
509
510
            sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
    {
511
        free(file_rec);
512
        free(rec_ref);
513
        DXT_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
514
        return(NULL);
515
516
    }

517
518
    dxt_mem_remaining -= sizeof(struct dxt_file_record);
    DXT_UNLOCK();
519

520
    /* initialize record and record reference fields */
521
522
523
524
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = mpiio_my_rank;

    rec_ref->file_rec = file_rec;
525
    dxt_mpiio_runtime->file_rec_count++;
526

527
    return(rec_ref);
528
529
}

Shane Snyder's avatar
Shane Snyder committed
530
static void dxt_free_record_data(void *rec_ref_p)
531
{
532
    struct dxt_file_record_ref *dxt_rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
533

534
    /* TODO: update these pointer addresses once {write/read}_traces are moved to rec_ref structure */
535
536
    free(dxt_rec_ref->write_traces);
    free(dxt_rec_ref->read_traces);
Shane Snyder's avatar
Shane Snyder committed
537
    free(dxt_rec_ref->file_rec);
538
539
}

540
static void dxt_posix_cleanup_runtime()
541
{
Shane Snyder's avatar
Shane Snyder committed
542
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_free_record_data);
543
    darshan_clear_record_refs(&(dxt_posix_runtime->rec_id_hash), 1);
544

545
546
    free(dxt_posix_runtime);
    dxt_posix_runtime = NULL;
547
548
549
550

    return;
}

551
static void dxt_mpiio_cleanup_runtime()
552
{
Shane Snyder's avatar
Shane Snyder committed
553
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_free_record_data);
554
    darshan_clear_record_refs(&(dxt_mpiio_runtime->rec_id_hash), 1);
555

556
557
    free(dxt_mpiio_runtime);
    dxt_mpiio_runtime = NULL;
558
559
560
561
562
563
564
565
566

    return;
}


/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

Shane Snyder's avatar
Shane Snyder committed
567
static void dxt_serialize_posix_records(void *rec_ref_p)
568
{
Shane Snyder's avatar
Shane Snyder committed
569
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
570
    struct dxt_file_record *file_rec;
571
572
573
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
574
    void *tmp_buf_ptr;
575

Shane Snyder's avatar
Shane Snyder committed
576
577
578
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
579

Shane Snyder's avatar
Shane Snyder committed
580
581
582
583
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
584

Shane Snyder's avatar
Shane Snyder committed
585
586
    /*
     * Buffer format:
587
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
588
589
590
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
591

Shane Snyder's avatar
Shane Snyder committed
592
593
    tmp_buf_ptr = (void *)(dxt_posix_runtime->record_buf +
        dxt_posix_runtime->record_buf_size);
594

Shane Snyder's avatar
Shane Snyder committed
595
596
597
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
598

Shane Snyder's avatar
Shane Snyder committed
599
    /*Copy write record */
600
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
601
602
603
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_write_count * sizeof(segment_info));
604

Shane Snyder's avatar
Shane Snyder committed
605
    /*Copy read record */
606
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
607
608
609
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
                record_read_count * sizeof(segment_info));
610

Shane Snyder's avatar
Shane Snyder committed
611
    dxt_posix_runtime->record_buf_size += record_size;
612

Shane Snyder's avatar
Shane Snyder committed
613
614
615
616
617
618
619
#if 0
    int i;
    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;
620

Shane Snyder's avatar
Shane Snyder committed
621
622
623
    printf("DXT, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("DXT, write_count is: %d read_count is: %d\n",
                file_rec->write_count, file_rec->read_count);
624

Shane Snyder's avatar
Shane Snyder committed
625
626
    for (i = 0; i < file_rec->write_count; i++) {
        rank = file_rec->base_rec.rank;
627
628
629
630
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
631

Shane Snyder's avatar
Shane Snyder committed
632
633
        printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
634

Shane Snyder's avatar
Shane Snyder committed
635
636
    for (i = 0; i < file_rec->read_count; i++) {
        rank = file_rec->base_rec.rank;
637
638
639
640
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
641

Shane Snyder's avatar
Shane Snyder committed
642
643
644
645
        printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
#endif
}
646

Shane Snyder's avatar
Shane Snyder committed
647
648
649
650
651
652
653
654
static void dxt_posix_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_posix_buf,
    int *dxt_posix_buf_sz)
{
    assert(dxt_posix_runtime);
655

Shane Snyder's avatar
Shane Snyder committed
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    *dxt_posix_buf_sz = 0;

    dxt_posix_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_posix_runtime->record_buf))
        return;
    memset(dxt_posix_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_posix_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_posix_runtime->rec_id_hash, dxt_serialize_posix_records);

    /* set output */
    *dxt_posix_buf = dxt_posix_runtime->record_buf;
    *dxt_posix_buf_sz = dxt_posix_runtime->record_buf_size;
670
671

    /* shutdown internal structures used for instrumenting */
672
    dxt_posix_cleanup_runtime();
673
674
675
676
677
678
679

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

Shane Snyder's avatar
Shane Snyder committed
680
static void dxt_serialize_mpiio_records(void *rec_ref_p)
681
{
Shane Snyder's avatar
Shane Snyder committed
682
    struct dxt_file_record_ref *rec_ref = (struct dxt_file_record_ref *)rec_ref_p;
683
    struct dxt_file_record *file_rec;
684
685
686
    int64_t record_size = 0;
    int64_t record_write_count = 0;
    int64_t record_read_count = 0;
Shane Snyder's avatar
Shane Snyder committed
687
    void *tmp_buf_ptr;
688

Shane Snyder's avatar
Shane Snyder committed
689
690
691
    assert(rec_ref);
    file_rec = rec_ref->file_rec;
    assert(file_rec);
692

Shane Snyder's avatar
Shane Snyder committed
693
694
695
696
    record_write_count = file_rec->write_count;
    record_read_count = file_rec->read_count;
    if (record_write_count == 0 && record_read_count == 0)
        return;
697

Shane Snyder's avatar
Shane Snyder committed
698
699
    /*
     * Buffer format:
700
     * dxt_file_record + write_traces + read_traces
Shane Snyder's avatar
Shane Snyder committed
701
702
703
     */
    record_size = sizeof(struct dxt_file_record) +
            (record_write_count + record_read_count) * sizeof(segment_info);
704

Shane Snyder's avatar
Shane Snyder committed
705
706
    tmp_buf_ptr = (void *)(dxt_mpiio_runtime->record_buf +
        dxt_mpiio_runtime->record_buf_size);
707

Shane Snyder's avatar
Shane Snyder committed
708
709
710
    /*Copy struct dxt_file_record */
    memcpy(tmp_buf_ptr, (void *)file_rec, sizeof(struct dxt_file_record));
    tmp_buf_ptr = (void *)(tmp_buf_ptr + sizeof(struct dxt_file_record));
711

Shane Snyder's avatar
Shane Snyder committed
712
    /*Copy write record */
713
    memcpy(tmp_buf_ptr, (void *)(rec_ref->write_traces),
Shane Snyder's avatar
Shane Snyder committed
714
715
            record_write_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
716
717
                record_write_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
718
    /*Copy read record */
719
    memcpy(tmp_buf_ptr, (void *)(rec_ref->read_traces),
Shane Snyder's avatar
Shane Snyder committed
720
721
            record_read_count * sizeof(segment_info));
    tmp_buf_ptr = (void *)(tmp_buf_ptr +
722
723
                record_read_count * sizeof(segment_info));

Shane Snyder's avatar
Shane Snyder committed
724
    dxt_mpiio_runtime->record_buf_size += record_size;
725
726

#if 0
Shane Snyder's avatar
Shane Snyder committed
727
728
729
730
731
732
    int i;
    int64_t offset;
    int64_t length;
    int64_t rank;
    double start_time;
    double end_time;
733

Shane Snyder's avatar
Shane Snyder committed
734
735
736
    printf("Cong, record_id: %" PRIu64 "\n", rec_ref->file_rec->base_rec.id);
    printf("DXT, file_rec->write_count is: %d\n",
                file_rec->write_count);
737

Shane Snyder's avatar
Shane Snyder committed
738
739
    for (i = 0; i < file_rec->write_count; i++) {
        rank = file_rec->base_rec.rank;
740
741
742
743
        offset = rec_ref->write_traces[i].offset;
        length = rec_ref->write_traces[i].length;
        start_time = rec_ref->write_traces[i].start_time;
        end_time = rec_ref->write_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
744
745

        printf("DXT, rank %d writes segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
746
747
    }

Shane Snyder's avatar
Shane Snyder committed
748
749
    for (i = 0; i < file_rec->read_count; i++) {
        rank = file_rec->base_rec.rank;
750
751
752
753
        offset = rec_ref->read_traces[i].offset;
        length = rec_ref->read_traces[i].length;
        start_time = rec_ref->read_traces[i].start_time;
        end_time = rec_ref->read_traces[i].end_time;
Shane Snyder's avatar
Shane Snyder committed
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783

        printf("DXT, rank %d reads segment %lld [offset: %lld length: %lld start_time: %fs end_time: %fs]\n", rank, i, offset, length, start_time, end_time);
    }
#endif
}

static void dxt_mpiio_shutdown(
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **dxt_mpiio_buf,
    int *dxt_mpiio_buf_sz)
{
    assert(dxt_mpiio_runtime);

    *dxt_mpiio_buf_sz = 0;

    dxt_mpiio_runtime->record_buf = malloc(DXT_IO_TRACE_MEM_MAX);
    if(!(dxt_mpiio_runtime->record_buf))
        return;
    memset(dxt_mpiio_runtime->record_buf, 0, DXT_IO_TRACE_MEM_MAX);
    dxt_mpiio_runtime->record_buf_size = 0;

    /* iterate all dxt posix records and serialize them to the output buffer */
    darshan_iter_record_refs(dxt_mpiio_runtime->rec_id_hash, dxt_serialize_mpiio_records);

    /* set output */ 
    *dxt_mpiio_buf = dxt_mpiio_runtime->record_buf;
    *dxt_mpiio_buf_sz = dxt_mpiio_runtime->record_buf_size;

784
    /* shutdown internal structures used for instrumenting */
785
    dxt_mpiio_cleanup_runtime();
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800

    /* disable further instrumentation */
    instrumentation_disabled = 1;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */