darshan-core.c 53.8 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _XOPEN_SOURCE 500
8
#define _GNU_SOURCE
9

10 11 12 13 14 15
#include "darshan-runtime-config.h"

#include <stdio.h>
#ifdef HAVE_MNTENT_H
#include <mntent.h>
#endif
16 17 18 19 20
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <pthread.h>
21 22 23
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/vfs.h>
24
#include <zlib.h>
25
#include <mpi.h>
26
#include <assert.h>
27

28
#include "uthash.h"
Shane Snyder's avatar
Shane Snyder committed
29
#include "darshan.h"
30
#include "darshan-core.h"
Shane Snyder's avatar
Shane Snyder committed
31
#include "darshan-dynamic.h"
32

33
extern char* __progname;
34
extern char* __progname_full;
35

36
/* internal variable delcarations */
37
static struct darshan_core_runtime *darshan_core = NULL;
38
static pthread_mutex_t darshan_core_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
39
static int my_rank = -1;
40
static int nprocs = -1;
41
static int darshan_mem_alignment = 1;
42

43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
/* paths prefixed with the following directories are not traced by darshan */
char* darshan_path_exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
#ifdef DARSHAN_BGQ
extern void bgq_runtime_initialize();
#endif

/* array of init functions for modules which need to be statically
 * initialized by darshan at startup time
 */
void (*mod_static_init_fns[])(void) =
{
#ifdef DARSHAN_BGQ
    &bgq_runtime_initialize,
#endif
    NULL
};

Shane Snyder's avatar
Shane Snyder committed
73 74 75
#define DARSHAN_CORE_LOCK() pthread_mutex_lock(&darshan_core_mutex)
#define DARSHAN_CORE_UNLOCK() pthread_mutex_unlock(&darshan_core_mutex)

76 77 78 79 80 81
/* FS mount information */
#define DARSHAN_MAX_MNTS 64
#define DARSHAN_MAX_MNT_PATH 256
#define DARSHAN_MAX_MNT_TYPE 32
struct mnt_data
{
82
    int block_size;
83 84 85 86 87 88
    char path[DARSHAN_MAX_MNT_PATH];
    char type[DARSHAN_MAX_MNT_TYPE];
};
static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;

89 90 91 92
/* prototypes for internal helper functions */
static void darshan_get_logfile_name(
    char* logfile_name, int jobid, struct tm* start_tm);
static void darshan_log_record_hints_and_ver(
93 94 95 96 97 98
    struct darshan_core_runtime* core);
static void darshan_get_exe_and_mounts_root(
    struct darshan_core_runtime *core, char* trailing_data,
    int space_left);
static char* darshan_get_exe_and_mounts(
    struct darshan_core_runtime *core);
99 100
static void darshan_block_size_from_path(
    const char *path, int *block_size);
101
static void darshan_get_shared_records(
102 103
    struct darshan_core_runtime *core, darshan_record_id **shared_recs,
    int *shared_rec_cnt);
104
static int darshan_log_open_all(
105
    char *logfile_name, MPI_File *log_fh);
106
static int darshan_deflate_buffer(
Shane Snyder's avatar
Shane Snyder committed
107 108
    void **pointers, int *lengths, int count, char *comp_buf,
    int *comp_buf_length);
109
static int darshan_log_write_record_hash(
110
    MPI_File log_fh, struct darshan_core_runtime *core,
111 112 113
    uint64_t *inout_off);
static int darshan_log_append_all(
    MPI_File log_fh, struct darshan_core_runtime *core, void *buf,
Shane Snyder's avatar
Shane Snyder committed
114
    int count, uint64_t *inout_off);
Shane Snyder's avatar
Shane Snyder committed
115 116
static void darshan_core_cleanup(
    struct darshan_core_runtime* core);
117

118 119
/* *********************************** */

Shane Snyder's avatar
Shane Snyder committed
120
void darshan_core_initialize(int argc, char **argv)
121
{
122
    struct darshan_core_runtime *init_core = NULL;
123 124 125
    int i;
    int internal_timing_flag = 0;
    double init_start, init_time, init_max;
126
    char *envstr;
127 128 129
    char* truncate_string = "<TRUNCATED>";
    int truncate_offset;
    int chars_left = 0;
130 131
    int ret;
    int tmpval;
132 133

    DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
134
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
135 136 137 138

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

139
    if(internal_timing_flag)
140 141 142
        init_start = DARSHAN_MPI_CALL(PMPI_Wtime)();

    /* setup darshan runtime if darshan is enabled and hasn't been initialized already */
143
    if(!getenv("DARSHAN_DISABLE") && !darshan_core)
144
    {
145
        #if (__DARSHAN_MEM_ALIGNMENT < 1)
146 147
            #error Darshan must be configured with a positive value for --with-mem-align
        #endif
148
        envstr = getenv(DARSHAN_MEM_ALIGNMENT_OVERRIDE);
149 150 151 152 153 154 155 156 157 158 159
        if(envstr)
        {
            ret = sscanf(envstr, "%d", &tmpval);
            /* silently ignore if the env variable is set poorly */
            if(ret == 1 && tmpval > 0)
            {
                darshan_mem_alignment = tmpval;
            }
        }
        else
        {
160
            darshan_mem_alignment = __DARSHAN_MEM_ALIGNMENT;
161 162 163 164 165 166 167
        }

        /* avoid floating point errors on faulty input */
        if (darshan_mem_alignment < 1)
        {
            darshan_mem_alignment = 1;
        }
168 169

        /* allocate structure to track darshan_core_runtime information */
170 171
        init_core = malloc(sizeof(*init_core));
        if(init_core)
172
        {
173
            memset(init_core, 0, sizeof(*init_core));
174

175 176 177 178
            init_core->log_job.uid = getuid();
            init_core->log_job.start_time = time(NULL);
            init_core->log_job.nprocs = nprocs;
            init_core->wtime_offset = DARSHAN_MPI_CALL(PMPI_Wtime)();
179 180

            /* record exe and arguments */
181
            for(i=0; i<argc; i++)
182
            {
183 184
                chars_left = DARSHAN_EXE_LEN-strlen(init_core->exe);
                strncat(init_core->exe, argv[i], chars_left);
185
                if(i < (argc-1))
186
                {
187 188
                    chars_left = DARSHAN_EXE_LEN-strlen(init_core->exe);
                    strncat(init_core->exe, " ", chars_left);
189 190 191 192 193 194 195 196
                }
            }

            /* if we don't see any arguments, then use glibc symbol to get
             * program name at least (this happens in fortran)
             */
            if(argc == 0)
            {
197 198 199 200
                chars_left = DARSHAN_EXE_LEN-strlen(init_core->exe);
                strncat(init_core->exe, __progname_full, chars_left);
                chars_left = DARSHAN_EXE_LEN-strlen(init_core->exe);
                strncat(init_core->exe, " <unknown args>", chars_left);
201 202 203 204 205
            }

            if(chars_left == 0)
            {
                /* we ran out of room; mark that string was truncated */
206
                truncate_offset = DARSHAN_EXE_LEN - strlen(truncate_string);
207
                sprintf(&init_core->exe[truncate_offset], "%s",
208 209
                    truncate_string);
            }
210 211

            /* collect information about command line and mounted file systems */
212
            init_core->trailing_data = darshan_get_exe_and_mounts(init_core);
213

Shane Snyder's avatar
Shane Snyder committed
214 215 216 217 218 219 220
            /* bootstrap any modules with static initialization routines */
            i = 0;
            while(mod_static_init_fns[i])
            {
                (*mod_static_init_fns[i])();
                i++;
            }
221
        }
222 223
    }

224 225 226 227 228
    if(internal_timing_flag)
    {
        init_time = DARSHAN_MPI_CALL(PMPI_Wtime)() - init_start;
        DARSHAN_MPI_CALL(PMPI_Reduce)(&init_time, &init_max, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
229
        if(my_rank == 0)
230
        {
231 232
            fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
            fprintf(stderr, "darshan:init\t%d\t%f\n", nprocs, init_max);
233 234 235
        }
    }

236 237 238 239
    /* if darshan was successfully initialized, set the global pointer */
    if(init_core)
        darshan_core = init_core;

240 241 242
    return;
}

Shane Snyder's avatar
Shane Snyder committed
243
void darshan_core_shutdown()
244
{
245
    int i;
246
    char *logfile_name;
247
    struct darshan_core_runtime *final_core;
248
    int internal_timing_flag = 0;
249 250
    char *envjobid;
    char *jobid_str;
251
    int jobid;
252
    struct tm *start_tm;
253
    time_t start_time_tmp;
254 255
    int ret = 0;
    int all_ret = 0;
256 257
    int64_t first_start_time;
    int64_t last_end_time;
258 259
    int local_mod_use[DARSHAN_MAX_MODS] = {0};
    int global_mod_use_count[DARSHAN_MAX_MODS] = {0};
260 261 262
    darshan_record_id *shared_recs;
    darshan_record_id *mod_shared_recs;
    int shared_rec_cnt = 0;
263
    double start_log_time;
264 265 266 267 268 269 270
    double open1, open2;
    double job1, job2;
    double rec1, rec2;
    double mod1[DARSHAN_MAX_MODS] = {0};
    double mod2[DARSHAN_MAX_MODS] = {0};
    double header1, header2;
    double tm_end;
271
    uint64_t gz_fp = 0;
272 273
    MPI_File log_fh;
    MPI_Status status;
274 275 276 277

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

278 279
    start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();

Shane Snyder's avatar
Shane Snyder committed
280
    /* disable darhan-core while we shutdown */
281
    DARSHAN_CORE_LOCK();
282
    if(!darshan_core)
283
    {
284
        DARSHAN_CORE_UNLOCK();
285 286
        return;
    }
287 288
    final_core = darshan_core;
    darshan_core = NULL;
Shane Snyder's avatar
Shane Snyder committed
289

290 291 292 293 294 295 296
    final_core->comp_buf = malloc(DARSHAN_COMP_BUF_SIZE);
    if(!(final_core->comp_buf))
    {
        darshan_core_cleanup(final_core);
        return;
    }

297
    /* we also need to set which modules were registered on this process and
298 299
     * call into those modules and give them a chance to perform any necessary
     * pre-shutdown steps.
Shane Snyder's avatar
Shane Snyder committed
300 301 302 303 304 305
     */
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
    {
        if(final_core->mod_array[i])
        {
            local_mod_use[i] = 1;
306
            final_core->mod_array[i]->mod_funcs.begin_shutdown();
Shane Snyder's avatar
Shane Snyder committed
307 308
        }
    }
309
    DARSHAN_CORE_UNLOCK();
310 311 312 313

    logfile_name = malloc(PATH_MAX);
    if(!logfile_name)
    {
314
        darshan_core_cleanup(final_core);
315 316 317
        return;
    }

318
    /* set darshan job id/metadata and constuct log file name on rank 0 */
319
    if(my_rank == 0)
320
    {
321
        /* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
322
        envjobid = getenv(DARSHAN_JOBID_OVERRIDE);
323
        if(!envjobid)
324
        {
325
            envjobid = __DARSHAN_JOBID;
326 327
        }

328
        /* find a job id */
329 330 331 332 333 334 335 336 337 338 339 340
        jobid_str = getenv(envjobid);
        if(jobid_str)
        {
            /* in cobalt we can find it in env var */
            ret = sscanf(jobid_str, "%d", &jobid);
        }
        if(!jobid_str || ret != 1)
        {
            /* use pid as fall back */
            jobid = getpid();
        }

341
        final_core->log_job.jobid = (int64_t)jobid;
342

343
        /* if we are using any hints to write the log file, then record those
344
         * hints with the darshan job information
345
         */
346
        darshan_log_record_hints_and_ver(final_core);
347

348
        /* use human readable start time format in log filename */
349
        start_time_tmp = final_core->log_job.start_time;
350
        start_tm = localtime(&start_time_tmp);
351

352 353
        /* construct log file name */
        darshan_get_logfile_name(logfile_name, jobid, start_tm);
354 355 356 357 358 359 360 361 362
    }

    /* broadcast log file name */
    DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
        MPI_COMM_WORLD);

    if(strlen(logfile_name) == 0)
    {
        /* failed to generate log file name */
363
        free(logfile_name);
364
        darshan_core_cleanup(final_core);
365 366 367
        return;
    }

368
    final_core->log_job.end_time = time(NULL);
369

370 371 372
    /* reduce to report first start time and last end time across all ranks
     * at rank 0
     */
373 374
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.start_time, &first_start_time, 1, MPI_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.end_time, &last_end_time, 1, MPI_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
375 376
    if(my_rank == 0)
    {
377 378
        final_core->log_job.start_time = first_start_time;
        final_core->log_job.end_time = last_end_time;
379
    }
380

381 382 383
    /* reduce the number of times a module was opened globally and bcast to everyone */   
    DARSHAN_MPI_CALL(PMPI_Allreduce)(local_mod_use, global_mod_use_count, DARSHAN_MAX_MODS, MPI_INT, MPI_SUM, MPI_COMM_WORLD);

384
    /* get a list of records which are shared across all processes */
385
    darshan_get_shared_records(final_core, &shared_recs, &shared_rec_cnt);
386

387 388
    if(internal_timing_flag)
        open1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
389
    /* collectively open the darshan log file */
390
    ret = darshan_log_open_all(logfile_name, &log_fh);
391 392
    if(internal_timing_flag)
        open2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
393 394 395 396 397 398 399 400

    /* error out if unable to open log file */
    DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
        MPI_LOR, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        if(my_rank == 0)
        {
401 402
            fprintf(stderr, "darshan library warning: unable to open log file %s\n",
                logfile_name);
403 404
        }
        free(logfile_name);
405
        darshan_core_cleanup(final_core);
406 407 408
        return;
    }

409 410
    if(internal_timing_flag)
        job1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
411
    /* rank 0 is responsible for writing the compressed darshan job information */
Shane Snyder's avatar
Shane Snyder committed
412
    if(my_rank == 0)
413
    {
414
        void *pointers[2] = {&final_core->log_job, final_core->trailing_data};
415
        int lengths[2] = {sizeof(struct darshan_job), strlen(final_core->trailing_data)};
416
        int comp_buf_sz = 0;
417

418
        /* compress the job info and the trailing mount/exe data */
Shane Snyder's avatar
Shane Snyder committed
419
        all_ret = darshan_deflate_buffer(pointers, lengths, 2,
420 421
            final_core->comp_buf, &comp_buf_sz);
        if(all_ret)
422
        {
423
            fprintf(stderr, "darshan library warning: unable to compress job data\n");
424
            unlink(logfile_name);
425
        }
426 427 428
        else
        {
            /* write the job information, preallocing space for the log header */
Shane Snyder's avatar
Shane Snyder committed
429
            gz_fp += sizeof(struct darshan_header);
430 431
            all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, gz_fp,
                final_core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
432 433 434 435 436
            if(all_ret != MPI_SUCCESS)
            {
                fprintf(stderr, "darshan library warning: unable to write job data to log file %s\n",
                        logfile_name);
                unlink(logfile_name);
Shane Snyder's avatar
Shane Snyder committed
437
                
438
            }
439
            gz_fp += comp_buf_sz;
440
        }
441 442
    }

443 444 445 446 447
    /* error out if unable to write job information */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        free(logfile_name);
448
        darshan_core_cleanup(final_core);
449 450
        return;
    }
451 452
    if(internal_timing_flag)
        job2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
453

454 455
    if(internal_timing_flag)
        rec1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
456
    /* write the record name->id hash to the log file */
Shane Snyder's avatar
Shane Snyder committed
457
    final_core->log_header.rec_map.off = gz_fp;
458
    ret = darshan_log_write_record_hash(log_fh, final_core, &gz_fp);
Shane Snyder's avatar
Shane Snyder committed
459
    final_core->log_header.rec_map.len = gz_fp - final_core->log_header.rec_map.off;
460

461
    /* error out if unable to write record hash */
462 463 464 465 466 467
    DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
        MPI_LOR, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        if(my_rank == 0)
        {
468
            fprintf(stderr, "darshan library warning: unable to write record hash to log file %s\n",
469
                logfile_name);
470
            unlink(logfile_name);
471 472
        }
        free(logfile_name);
473
        darshan_core_cleanup(final_core);
474 475
        return;
    }
Shane Snyder's avatar
Shane Snyder committed
476 477
    if(internal_timing_flag)
        rec2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
478

479 480 481
    mod_shared_recs = malloc(shared_rec_cnt * sizeof(darshan_record_id));
    assert(mod_shared_recs);

482
    /* loop over globally used darshan modules and:
483
     *      - perform shared file reductions, if possible
484
     *      - get final output buffer
485
     *      - compress (zlib) provided output buffer
Shane Snyder's avatar
Shane Snyder committed
486
     *      - append compressed buffer to log file
487 488
     *      - add module index info (file offset/length) to log header
     *      - shutdown the module
489
     */
490
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
491
    {
492
        struct darshan_core_module* this_mod = final_core->mod_array[i];
493
        struct darshan_core_record_ref *ref = NULL;
494
        int mod_shared_rec_cnt = 0;
495
        void* mod_buf = NULL;
496
        int mod_buf_sz = 0;
497
        int j;
498

499
        if(global_mod_use_count[i] == 0)
500 501
        {
            if(my_rank == 0)
502 503 504 505
            {
                final_core->log_header.mod_map[i].off = 0;
                final_core->log_header.mod_map[i].len = 0;
            }
506
            continue;
507
        }
508 509
 
        if(internal_timing_flag)
510
            mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
511

512
        /* set the shared file list for this module */
513 514
        memset(mod_shared_recs, 0, shared_rec_cnt * sizeof(darshan_record_id));
        for(j = 0; j < shared_rec_cnt; j++)
515 516 517 518
        {
            HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
                sizeof(darshan_record_id), ref);
            assert(ref);
519
            if(DARSHAN_MOD_FLAG_ISSET(ref->global_mod_flags, i))
520
            {
521
                mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
522
            }
523
        }
524

525 526 527 528 529
        /* if module is registered locally, get the corresponding output buffer
         * 
         * NOTE: this function can be used to run collective operations across
         * modules, if there are file records shared globally.
         */
530
        if(this_mod)
531
        {
532 533
            this_mod->mod_funcs.get_output_data(MPI_COMM_WORLD, mod_shared_recs,
                mod_shared_rec_cnt, &mod_buf, &mod_buf_sz);
534 535
        }

536
        /* append this module's data to the darshan log */
Shane Snyder's avatar
Shane Snyder committed
537 538 539 540
        final_core->log_header.mod_map[i].off = gz_fp;
        ret = darshan_log_append_all(log_fh, final_core, mod_buf, mod_buf_sz, &gz_fp);
        final_core->log_header.mod_map[i].len =
            gz_fp - final_core->log_header.mod_map[i].off;
541

542
        /* error out if the log append failed */
543 544 545
        DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
            MPI_LOR, MPI_COMM_WORLD);
        if(all_ret != 0)
546
        {
547 548 549 550 551 552 553 554
            if(my_rank == 0)
            {
                fprintf(stderr,
                    "darshan library warning: unable to write %s module data to log file %s\n",
                    darshan_module_names[i], logfile_name);
                unlink(logfile_name);
            }
            free(logfile_name);
555
            darshan_core_cleanup(final_core);
556
            return;
557 558 559
        }

        /* shutdown module if registered locally */
560
        if(this_mod)
561 562 563
        {
            this_mod->mod_funcs.shutdown();
        }
564 565
        if(internal_timing_flag)
            mod2[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
566 567
    }

568 569
    if(internal_timing_flag)
        header1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
570 571 572 573 574 575
    /* write out log header, after running 2 reduction on header variables:
     *  1) reduce 'partial_flag' variable to determine which modules ran out
     *     of memory for storing I/O data
     *  2) reduce 'mod_ver' array to determine which log format version each
     *     module used for this output log
     */
576 577
    if(my_rank == 0)
    {
578 579 580 581 582 583 584 585
        DARSHAN_MPI_CALL(PMPI_Reduce)(MPI_IN_PLACE,
            &(final_core->log_header.partial_flag), 1, MPI_UINT32_T,
            MPI_BOR, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(MPI_IN_PLACE,
            final_core->log_header.mod_ver, DARSHAN_MAX_MODS, MPI_UINT32_T,
            MPI_MAX, 0, MPI_COMM_WORLD);

        /* rank 0 is responsible for writing the log header */
586 587 588
        /* initialize the remaining header fields */
        strcpy(final_core->log_header.version_string, DARSHAN_LOG_VERSION);
        final_core->log_header.magic_nr = DARSHAN_MAGIC_NR;
589
        final_core->log_header.comp_type = DARSHAN_ZLIB_COMP;
590

Shane Snyder's avatar
Shane Snyder committed
591 592 593
        all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, 0, &(final_core->log_header),
            sizeof(struct darshan_header), MPI_BYTE, &status);
        if(all_ret != MPI_SUCCESS)
594
        {
Shane Snyder's avatar
Shane Snyder committed
595 596
            fprintf(stderr, "darshan library warning: unable to write header to log file %s\n",
                    logfile_name);
597
            unlink(logfile_name);
598
        }
599
    }
600 601 602 603 604 605 606 607 608
    else
    {
        DARSHAN_MPI_CALL(PMPI_Reduce)(&(final_core->log_header.partial_flag),
            &(final_core->log_header.partial_flag), 1, MPI_UINT32_T,
            MPI_BOR, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(final_core->log_header.mod_ver,
            final_core->log_header.mod_ver, DARSHAN_MAX_MODS, MPI_UINT32_T,
            MPI_MAX, 0, MPI_COMM_WORLD);
    }
609

610 611 612 613 614
    /* error out if unable to write log header */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        free(logfile_name);
615
        darshan_core_cleanup(final_core);
616 617
        return;
    }
618 619
    if(internal_timing_flag)
        header2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
620

621 622 623
    DARSHAN_MPI_CALL(PMPI_File_close)(&log_fh);

    /* if we got this far, there are no errors, so rename from *.darshan_partial
624
     * to *-<logwritetime>.darshan, which indicates that this log file is
625 626
     * complete and ready for analysis
     */
627 628
    if(my_rank == 0)
    {
Shane Snyder's avatar
Shane Snyder committed
629
        if(getenv("DARSHAN_LOGFILE"))
630
        {
631
#ifdef __DARSHAN_GROUP_READABLE_LOGS
Shane Snyder's avatar
Shane Snyder committed
632
            chmod(logfile_name, (S_IRUSR|S_IRGRP));
633
#else
Shane Snyder's avatar
Shane Snyder committed
634
            chmod(logfile_name, (S_IRUSR));
635
#endif
Shane Snyder's avatar
Shane Snyder committed
636 637 638 639 640 641 642 643 644 645 646 647 648 649
        }
        else
        {
            char* tmp_index;
            double end_log_time;
            char* new_logfile_name;

            new_logfile_name = malloc(PATH_MAX);
            if(new_logfile_name)
            {
                new_logfile_name[0] = '\0';
                end_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
                strcat(new_logfile_name, logfile_name);
                tmp_index = strstr(new_logfile_name, ".darshan_partial");
650
                sprintf(tmp_index, "_%d.darshan", (int)(end_log_time-start_log_time+1));
Shane Snyder's avatar
Shane Snyder committed
651 652
                rename(logfile_name, new_logfile_name);
                /* set permissions on log file */
653
#ifdef __DARSHAN_GROUP_READABLE_LOGS
Shane Snyder's avatar
Shane Snyder committed
654 655 656 657 658 659
                chmod(new_logfile_name, (S_IRUSR|S_IRGRP));
#else
                chmod(new_logfile_name, (S_IRUSR));
#endif
                free(new_logfile_name);
            }
660
        }
661
    }
662

663
    free(logfile_name);
664
    darshan_core_cleanup(final_core);
665

666
    if(internal_timing_flag)
667
    {
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
        double open_tm, open_slowest;
        double header_tm, header_slowest;
        double job_tm, job_slowest;
        double rec_tm, rec_slowest;
        double mod_tm[DARSHAN_MAX_MODS], mod_slowest[DARSHAN_MAX_MODS];
        double all_tm, all_slowest;

        tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)();

        open_tm = open2 - open1;
        header_tm = header2 - header1;
        job_tm = job2 - job1;
        rec_tm = rec2 - rec1;
        all_tm = tm_end - start_log_time;
        for(i = 0;i < DARSHAN_MAX_MODS; i++)
        {
            mod_tm[i] = mod2[i] - mod1[i];
        }

        DARSHAN_MPI_CALL(PMPI_Reduce)(&open_tm, &open_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&header_tm, &header_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&job_tm, &job_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&rec_tm, &rec_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&all_tm, &all_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(mod_tm, mod_slowest, DARSHAN_MAX_MODS,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);

        if(my_rank == 0)
        {
702 703 704 705 706
            fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
            fprintf(stderr, "darshan:log_open\t%d\t%f\n", nprocs, open_slowest);
            fprintf(stderr, "darshan:job_write\t%d\t%f\n", nprocs, job_slowest);
            fprintf(stderr, "darshan:hash_write\t%d\t%f\n", nprocs, rec_slowest);
            fprintf(stderr, "darshan:header_write\t%d\t%f\n", nprocs, header_slowest);
707 708 709
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                if(global_mod_use_count[i])
710
                    fprintf(stderr, "darshan:%s_shutdown\t%d\t%f\n", darshan_module_names[i],
Shane Snyder's avatar
Shane Snyder committed
711
                        nprocs, mod_slowest[i]);
712
            }
713
            fprintf(stderr, "darshan:core_shutdown\t%d\t%f\n", nprocs, all_slowest);
714
        }
715 716 717 718
    }
    
    return;
}
719

Shane Snyder's avatar
Shane Snyder committed
720
/* *********************************** */
721

722
/* construct the darshan log file name */
723
static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* start_tm)
724
{
Shane Snyder's avatar
Shane Snyder committed
725
    char* user_logfile_name;
726 727 728
    char* logpath;
    char* logname_string;
    char* logpath_override = NULL;
729
#ifdef __DARSHAN_LOG_ENV
730 731 732 733 734 735 736 737 738
    char env_check[256];
    char* env_tok;
#endif
    uint64_t hlevel;
    char hname[HOST_NAME_MAX];
    uint64_t logmod;
    char cuser[L_cuserid] = {0};
    int ret;

Shane Snyder's avatar
Shane Snyder committed
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
    /* first, check if user specifies a complete logpath to use */
    user_logfile_name = getenv("DARSHAN_LOGFILE");
    if(user_logfile_name)
    {
        if(strlen(user_logfile_name) >= (PATH_MAX-1))
        {
            fprintf(stderr, "darshan library warning: user log file name too long.\n");
            logfile_name[0] = '\0';
        }
        else
        {
            strcpy(logfile_name, user_logfile_name);
        }
    }
    else
754
    {
Shane Snyder's avatar
Shane Snyder committed
755 756
        /* otherwise, generate the log path automatically */

757 758
        /* Use DARSHAN_LOG_PATH_OVERRIDE for the value or __DARSHAN_LOG_PATH */
        logpath = getenv(DARSHAN_LOG_PATH_OVERRIDE);
Shane Snyder's avatar
Shane Snyder committed
759 760
        if(!logpath)
        {
761 762
#ifdef __DARSHAN_LOG_PATH
            logpath = __DARSHAN_LOG_PATH;
763
#endif
Shane Snyder's avatar
Shane Snyder committed
764
        }
765

Shane Snyder's avatar
Shane Snyder committed
766 767 768 769 770 771 772 773 774 775
        /* get the username for this job.  In order we will try each of the
         * following until one of them succeeds:
         *
         * - cuserid()
         * - getenv("LOGNAME")
         * - snprintf(..., geteuid());
         *
         * Note that we do not use getpwuid() because it generally will not
         * work in statically compiled binaries.
         */
776 777

#ifndef DARSHAN_DISABLE_CUSERID
Shane Snyder's avatar
Shane Snyder committed
778
        cuserid(cuser);
779 780
#endif

Shane Snyder's avatar
Shane Snyder committed
781 782
        /* if cuserid() didn't work, then check the environment */
        if(strcmp(cuser, "") == 0)
783
        {
Shane Snyder's avatar
Shane Snyder committed
784 785 786 787 788
            logname_string = getenv("LOGNAME");
            if(logname_string)
            {
                strncpy(cuser, logname_string, (L_cuserid-1));
            }
789 790
        }

Shane Snyder's avatar
Shane Snyder committed
791 792 793 794 795 796
        /* if cuserid() and environment both fail, then fall back to uid */
        if(strcmp(cuser, "") == 0)
        {
            uid_t uid = geteuid();
            snprintf(cuser, sizeof(cuser), "%u", uid);
        }
797

Shane Snyder's avatar
Shane Snyder committed
798 799 800 801
        /* generate a random number to help differentiate the log */
        hlevel=DARSHAN_MPI_CALL(PMPI_Wtime)() * 1000000;
        (void)gethostname(hname, sizeof(hname));
        logmod = darshan_hash((void*)hname,strlen(hname),hlevel);
802

Shane Snyder's avatar
Shane Snyder committed
803 804 805 806
        /* see if darshan was configured using the --with-logpath-by-env
         * argument, which allows the user to specify an absolute path to
         * place logs via an env variable.
         */
807
#ifdef __DARSHAN_LOG_ENV
Shane Snyder's avatar
Shane Snyder committed
808
        /* just silently skip if the environment variable list is too big */
809
        if(strlen(__DARSHAN_LOG_ENV) < 256)
810
        {
Shane Snyder's avatar
Shane Snyder committed
811
            /* copy env variable list to a temporary buffer */
812
            strcpy(env_check, __DARSHAN_LOG_ENV);
Shane Snyder's avatar
Shane Snyder committed
813 814 815
            /* tokenize the comma-separated list */
            env_tok = strtok(env_check, ",");
            if(env_tok)
816
            {
Shane Snyder's avatar
Shane Snyder committed
817
                do
818
                {
Shane Snyder's avatar
Shane Snyder committed
819 820 821 822 823 824 825 826 827
                    /* check each env variable in order */
                    logpath_override = getenv(env_tok);
                    if(logpath_override)
                    {
                        /* stop as soon as we find a match */
                        break;
                    }
                }while((env_tok = strtok(NULL, ",")));
            }
828 829 830
        }
#endif

Shane Snyder's avatar
Shane Snyder committed
831
        if(logpath_override)
832
        {
Shane Snyder's avatar
Shane Snyder committed
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
            ret = snprintf(logfile_name, PATH_MAX,
                "%s/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                logpath_override,
                cuser, __progname, jobid,
                (start_tm->tm_mon+1),
                start_tm->tm_mday,
                (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
                logmod);
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath_override, jobid);
            }
848
        }
Shane Snyder's avatar
Shane Snyder committed
849
        else if(logpath)
850
        {
Shane Snyder's avatar
Shane Snyder committed
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
            ret = snprintf(logfile_name, PATH_MAX,
                "%s/%d/%d/%d/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                logpath, (start_tm->tm_year+1900),
                (start_tm->tm_mon+1), start_tm->tm_mday,
                cuser, __progname, jobid,
                (start_tm->tm_mon+1),
                start_tm->tm_mday,
                (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
                logmod);
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath, jobid);
            }
        }
        else
        {
            logfile_name[0] = '\0';
871 872 873 874
        }
    }

    return;
875 876
}

877
/* record any hints used to write the darshan log in the log header */
878
static void darshan_log_record_hints_and_ver(struct darshan_core_runtime* core)
879 880 881 882 883 884 885 886 887
{
    char* hints;
    char* header_hints;
    int meta_remain = 0;
    char* m;

    /* check environment variable to see if the default MPI file hints have
     * been overridden
     */
888
    hints = getenv(DARSHAN_LOG_HINTS_OVERRIDE);
889 890
    if(!hints)
    {
891
        hints = __DARSHAN_LOG_HINTS;
892 893 894 895 896 897 898 899 900 901
    }

    if(!hints || strlen(hints) < 1)
        return;

    header_hints = strdup(hints);
    if(!header_hints)
        return;

    meta_remain = DARSHAN_JOB_METADATA_LEN -
902
        strlen(core->log_job.metadata) - 1;
903 904
    if(meta_remain >= (strlen(PACKAGE_VERSION) + 9))
    {
905
        sprintf(core->log_job.metadata, "lib_ver=%s\n", PACKAGE_VERSION);
906 907 908 909
        meta_remain -= (strlen(PACKAGE_VERSION) + 9);
    }
    if(meta_remain >= (3 + strlen(header_hints)))
    {
910
        m = core->log_job.metadata + strlen(core->log_job.metadata);
911 912 913 914 915 916 917 918 919 920 921 922
        /* We have room to store the hints in the metadata portion of
         * the job header.  We just prepend an h= to the hints list.  The
         * metadata parser will ignore = characters that appear in the value
         * portion of the metadata key/value pair.
         */
        sprintf(m, "h=%s\n", header_hints);
    }
    free(header_hints);

    return;
}

923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
static int mnt_data_cmp(const void* a, const void* b)
{
    const struct mnt_data *d_a = (const struct mnt_data*)a;
    const struct mnt_data *d_b = (const struct mnt_data*)b;

    if(strlen(d_a->path) > strlen(d_b->path))
        return(-1);
    else if(strlen(d_a->path) < strlen(d_b->path))
        return(1);
    else
        return(0);
}

/* adds an entry to table of mounted file systems */
static void add_entry(char* trailing_data, int* space_left, struct mntent *entry)
{
    int ret;
    char tmp_mnt[256];
    struct statfs statfsbuf;

    strncpy(mnt_data_array[mnt_data_count].path, entry->mnt_dir,
        DARSHAN_MAX_MNT_PATH-1);
    strncpy(mnt_data_array[mnt_data_count].type, entry->mnt_type,
        DARSHAN_MAX_MNT_TYPE-1);
    /* NOTE: we now try to detect the preferred block size for each file 
     * system using fstatfs().  On Lustre we assume a size of 1 MiB 
     * because fstatfs() reports 4 KiB. 
     */
#ifndef LL_SUPER_MAGIC
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
    ret = statfs(entry->mnt_dir, &statfsbuf);
    if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
        mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
    else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
        mnt_data_array[mnt_data_count].block_size = 1024*1024;
    else
        mnt_data_array[mnt_data_count].block_size = 4096;

    /* store mount information for use in header of darshan log */
963
    ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
        entry->mnt_type, entry->mnt_dir);
    if(ret < 256 && strlen(tmp_mnt) <= (*space_left))
    {
        strcat(trailing_data, tmp_mnt);
        (*space_left) -= strlen(tmp_mnt);
    }

    mnt_data_count++;
    return;
}

/* darshan_get_exe_and_mounts_root()
 *
 * collects command line and list of mounted file systems into a string that
 * will be stored with the job header
 */
static void darshan_get_exe_and_mounts_root(struct darshan_core_runtime *core,
    char* trailing_data, int space_left)
{
    FILE* tab;
    struct mntent *entry;
    char* exclude;
    int tmp_index = 0;
    int skip = 0;

    /* skip these fs types */
    static char* fs_exclusions[] = {
        "tmpfs",
        "proc",
        "sysfs",
        "devpts",
        "binfmt_misc",
        "fusectl",
        "debugfs",
        "securityfs",
        "nfsd",
        "none",
        "rpc_pipefs",
        "hugetlbfs",
        "cgroup",
        NULL
    };

Shane Snyder's avatar
Shane Snyder committed
1007
    /* length of exe has already been safety checked in darshan initialization */
1008
    strcat(trailing_data, core->exe);
1009
    space_left = DARSHAN_EXE_LEN - strlen(trailing_data);
1010 1011 1012 1013 1014

    /* we make two passes through mounted file systems; in the first pass we
     * grab any non-nfs mount points, then on the second pass we grab nfs
     * mount points
     */
1015
    mnt_data_count = 0;
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073

    tab = setmntent("/etc/mtab", "r");
    if(!tab)
        return;
    /* loop through list of mounted file systems */
    while(mnt_data_count<DARSHAN_MAX_MNTS && (entry = getmntent(tab)) != NULL)
    {
        /* filter out excluded fs types */
        tmp_index = 0;
        skip = 0;
        while((exclude = fs_exclusions[tmp_index]))
        {
            if(!(strcmp(exclude, entry->mnt_type)))
            {
                skip =1;
                break;
            }
            tmp_index++;
        }

        if(skip || (strcmp(entry->mnt_type, "nfs") == 0))
            continue;

        add_entry(trailing_data, &space_left, entry);
    }
    endmntent(tab);

    tab = setmntent("/etc/mtab", "r");
    if(!tab)
        return;
    /* loop through list of mounted file systems */
    while(mnt_data_count<DARSHAN_MAX_MNTS && (entry = getmntent(tab)) != NULL)
    {
        if(strcmp(entry->mnt_type, "nfs") != 0)
            continue;

        add_entry(trailing_data, &space_left, entry);
    }
    endmntent(tab);

    /* Sort mount points in order of longest path to shortest path.  This is
     * necessary so that if we try to match file paths to mount points later
     * we don't match on "/" every time.
     */
    qsort(mnt_data_array, mnt_data_count, sizeof(mnt_data_array[0]), mnt_data_cmp);
    return;
}

/* darshan_get_exe_and_mounts()
 *
 * collects command line and list of mounted file systems into a string that
 * will be stored with the job header
 */
static char* darshan_get_exe_and_mounts(struct darshan_core_runtime *core)
{
    char* trailing_data;
    int space_left;

1074
    space_left = DARSHAN_EXE_LEN + 1;
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
    trailing_data = malloc(space_left);
    if(!trailing_data)
    {
        return(NULL);
    }
    memset(trailing_data, 0, space_left);

    if(my_rank == 0)
    {
        darshan_get_exe_and_mounts_root(core, trailing_data, space_left);
    }

    /* broadcast trailing data to all nodes */
    DARSHAN_MPI_CALL(PMPI_Bcast)(trailing_data, space_left, MPI_CHAR, 0,
        MPI_COMM_WORLD);
    /* broadcast mount count to all nodes */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&mnt_data_count, 1, MPI_INT, 0,
        MPI_COMM_WORLD);
    /* broadcast mount data to all nodes */
    DARSHAN_MPI_CALL(PMPI_Bcast)(mnt_data_array,
        mnt_data_count*sizeof(mnt_data_array[0]), MPI_BYTE, 0, MPI_COMM_WORLD);

    return(trailing_data);
}

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
static void darshan_block_size_from_path(const char *path, int *block_size)
{
    int i;
    *block_size = -1;

    for(i=0; i<mnt_data_count; i++)
    {
        if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
        {
            *block_size = mnt_data_array[i].block_size;
            return;
        }
    }

    return;
}

1117
static void darshan_get_shared_records(struct darshan_core_runtime *core,
1118
    darshan_record_id **shared_recs, int *shared_rec_cnt)
1119
{
1120 1121
    int i, j;
    int tmp_cnt = core->rec_count;
1122
    struct darshan_core_record_ref *tmp, *ref;
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
    darshan_record_id *id_array;
    uint64_t *mod_flags;
    uint64_t *global_mod_flags;

    /* broadcast root's number of records to all other processes */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&tmp_cnt, 1, MPI_INT, 0, MPI_COMM_WORLD);

    /* use root record count to allocate data structures */
    id_array = malloc(tmp_cnt * sizeof(darshan_record_id));
    mod_flags = malloc(tmp_cnt * sizeof(uint64_t));
    global_mod_flags = malloc(tmp_cnt * sizeof(uint64_t));
    *shared_recs = malloc(tmp_cnt * sizeof(darshan_record_id));
    assert(id_array && mod_flags && global_mod_flags && *shared_recs);

    memset(mod_flags, 0, tmp_cnt * sizeof(uint64_t));
    memset(global_mod_flags, 0, tmp_cnt * sizeof(uint64_t));
    memset(*shared_recs, 0, tmp_cnt * sizeof(darshan_record_id));
1140 1141 1142 1143

    /* first, determine list of records root process has opened */
    if(my_rank == 0)
    {
1144
        i = 0;
1145
        HASH_ITER(hlink, core->rec_hash, ref, tmp)
1146
        {
1147
            id_array[i++] = ref->rec.id;           
1148 1149 1150 1151
        }
    }

    /* broadcast root's list of records to all other processes */
1152
    DARSHAN_MPI_CALL(PMPI_Bcast)(id_array, (tmp_cnt * sizeof(darshan_record_id)),
1153 1154 1155
        MPI_BYTE, 0, MPI_COMM_WORLD);

    /* everyone looks to see if they opened the same records as root */
1156
    for(i=0; i<tmp_cnt; i++)
1157
    {
1158 1159
        HASH_FIND(hlink, core->rec_hash, &id_array[i], sizeof(darshan_record_id), ref);
        if(ref)
1160
        {
1161 1162
            /* we opened that record too, save the mod_flags */
            mod_flags[i] = ref->mod_flags;
1163 1164 1165
        }
    }

1166 1167 1168
    /* now allreduce so everyone agrees which files are shared and
     * which modules accessed them collectively
     */
1169 1170
    DARSHAN_MPI_CALL(PMPI_Allreduce)(mod_flags, global_mod_flags, tmp_cnt,
        MPI_UINT64_T, MPI_BAND, MPI_COMM_WORLD);
1171

1172 1173
    j = 0;
    for(i=0; i<tmp_cnt; i++)
1174
    {
1175
        if(global_mod_flags[i] != 0)
1176
        {
1177
            (*shared_recs)[j++] = id_array[i];
1178 1179 1180 1181 1182 1183 1184 1185

            /* set global_mod_flags so we know which modules collectively
             * accessed this module. we need this info to support shared
             * file reductions
             */
            HASH_FIND(hlink, core->rec_hash, &id_array[i], sizeof(darshan_record_id), ref);
            assert(ref);
            ref->global_mod_flags = global_mod_flags[i];
1186 1187
        }
    }
1188
    *shared_rec_cnt = j;
1189