darshan-core.c 53.5 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _XOPEN_SOURCE 500
8
#define _GNU_SOURCE
9

10 11 12 13 14 15
#include "darshan-runtime-config.h"

#include <stdio.h>
#ifdef HAVE_MNTENT_H
#include <mntent.h>
#endif
16 17 18 19 20
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <pthread.h>
21 22 23
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/vfs.h>
24
#include <zlib.h>
25
#include <mpi.h>
26
#include <assert.h>
27

28
#include "uthash.h"
Shane Snyder's avatar
Shane Snyder committed
29
#include "darshan.h"
30
#include "darshan-core.h"
Shane Snyder's avatar
Shane Snyder committed
31
#include "darshan-dynamic.h"
32

33
extern char* __progname;
34
extern char* __progname_full;
35

36
/* internal variable delcarations */
37
static struct darshan_core_runtime *darshan_core = NULL;
38
static pthread_mutex_t darshan_core_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
39
static int my_rank = -1;
40
static int nprocs = -1;
41
static int darshan_mem_alignment = 1;
42

43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
/* paths prefixed with the following directories are not traced by darshan */
char* darshan_path_exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
#ifdef DARSHAN_BGQ
extern void bgq_runtime_initialize();
#endif

/* array of init functions for modules which need to be statically
 * initialized by darshan at startup time
 */
void (*mod_static_init_fns[])(void) =
{
#ifdef DARSHAN_BGQ
    &bgq_runtime_initialize,
#endif
    NULL
};

Shane Snyder's avatar
Shane Snyder committed
73 74 75
#define DARSHAN_CORE_LOCK() pthread_mutex_lock(&darshan_core_mutex)
#define DARSHAN_CORE_UNLOCK() pthread_mutex_unlock(&darshan_core_mutex)

76 77 78 79 80 81
/* FS mount information */
#define DARSHAN_MAX_MNTS 64
#define DARSHAN_MAX_MNT_PATH 256
#define DARSHAN_MAX_MNT_TYPE 32
struct mnt_data
{
82
    int block_size;
83 84 85 86 87 88
    char path[DARSHAN_MAX_MNT_PATH];
    char type[DARSHAN_MAX_MNT_TYPE];
};
static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;

89 90 91 92
/* prototypes for internal helper functions */
static void darshan_get_logfile_name(
    char* logfile_name, int jobid, struct tm* start_tm);
static void darshan_log_record_hints_and_ver(
93 94 95 96 97 98
    struct darshan_core_runtime* core);
static void darshan_get_exe_and_mounts_root(
    struct darshan_core_runtime *core, char* trailing_data,
    int space_left);
static char* darshan_get_exe_and_mounts(
    struct darshan_core_runtime *core);
99 100
static void darshan_block_size_from_path(
    const char *path, int *block_size);
101
static void darshan_get_shared_records(
102 103
    struct darshan_core_runtime *core, darshan_record_id **shared_recs,
    int *shared_rec_cnt);
104
static int darshan_log_open_all(
105
    char *logfile_name, MPI_File *log_fh);
106
static int darshan_deflate_buffer(
Shane Snyder's avatar
Shane Snyder committed
107 108
    void **pointers, int *lengths, int count, char *comp_buf,
    int *comp_buf_length);
109
static int darshan_log_write_record_hash(
110
    MPI_File log_fh, struct darshan_core_runtime *core,
111 112 113
    uint64_t *inout_off);
static int darshan_log_append_all(
    MPI_File log_fh, struct darshan_core_runtime *core, void *buf,
Shane Snyder's avatar
Shane Snyder committed
114
    int count, uint64_t *inout_off);
Shane Snyder's avatar
Shane Snyder committed
115 116
static void darshan_core_cleanup(
    struct darshan_core_runtime* core);
117

118 119
/* *********************************** */

Shane Snyder's avatar
Shane Snyder committed
120
void darshan_core_initialize(int argc, char **argv)
121 122 123 124
{
    int i;
    int internal_timing_flag = 0;
    double init_start, init_time, init_max;
125
    char *envstr;
126 127 128
    char* truncate_string = "<TRUNCATED>";
    int truncate_offset;
    int chars_left = 0;
129 130
    int ret;
    int tmpval;
131 132

    DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
133
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
134 135 136 137

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

138
    if(internal_timing_flag)
139 140 141
        init_start = DARSHAN_MPI_CALL(PMPI_Wtime)();

    /* setup darshan runtime if darshan is enabled and hasn't been initialized already */
142
    if(!getenv("DARSHAN_DISABLE") && !darshan_core)
143
    {
144
        #if (__DARSHAN_MEM_ALIGNMENT < 1)
145 146
            #error Darshan must be configured with a positive value for --with-mem-align
        #endif
147
        envstr = getenv(DARSHAN_MEM_ALIGNMENT_OVERRIDE);
148 149 150 151 152 153 154 155 156 157 158
        if(envstr)
        {
            ret = sscanf(envstr, "%d", &tmpval);
            /* silently ignore if the env variable is set poorly */
            if(ret == 1 && tmpval > 0)
            {
                darshan_mem_alignment = tmpval;
            }
        }
        else
        {
159
            darshan_mem_alignment = __DARSHAN_MEM_ALIGNMENT;
160 161 162 163 164 165 166
        }

        /* avoid floating point errors on faulty input */
        if (darshan_mem_alignment < 1)
        {
            darshan_mem_alignment = 1;
        }
167 168 169 170

        /* allocate structure to track darshan_core_runtime information */
        darshan_core = malloc(sizeof(*darshan_core));
        if(darshan_core)
171
        {
172
            memset(darshan_core, 0, sizeof(*darshan_core));
173

174 175 176 177
            darshan_core->log_job.uid = getuid();
            darshan_core->log_job.start_time = time(NULL);
            darshan_core->log_job.nprocs = nprocs;
            darshan_core->wtime_offset = DARSHAN_MPI_CALL(PMPI_Wtime)();
178 179

            /* record exe and arguments */
180
            for(i=0; i<argc; i++)
181
            {
182
                chars_left = DARSHAN_EXE_LEN-strlen(darshan_core->exe);
183 184
                strncat(darshan_core->exe, argv[i], chars_left);
                if(i < (argc-1))
185
                {
186
                    chars_left = DARSHAN_EXE_LEN-strlen(darshan_core->exe);
187
                    strncat(darshan_core->exe, " ", chars_left);
188 189 190 191 192 193 194 195
                }
            }

            /* if we don't see any arguments, then use glibc symbol to get
             * program name at least (this happens in fortran)
             */
            if(argc == 0)
            {
196
                chars_left = DARSHAN_EXE_LEN-strlen(darshan_core->exe);
197
                strncat(darshan_core->exe, __progname_full, chars_left);
198
                chars_left = DARSHAN_EXE_LEN-strlen(darshan_core->exe);
199
                strncat(darshan_core->exe, " <unknown args>", chars_left);
200 201 202 203 204
            }

            if(chars_left == 0)
            {
                /* we ran out of room; mark that string was truncated */
205
                truncate_offset = DARSHAN_EXE_LEN - strlen(truncate_string);
206
                sprintf(&darshan_core->exe[truncate_offset], "%s",
207 208
                    truncate_string);
            }
209 210 211

            /* collect information about command line and mounted file systems */
            darshan_core->trailing_data = darshan_get_exe_and_mounts(darshan_core);
212

Shane Snyder's avatar
Shane Snyder committed
213 214 215 216 217 218 219
            /* bootstrap any modules with static initialization routines */
            i = 0;
            while(mod_static_init_fns[i])
            {
                (*mod_static_init_fns[i])();
                i++;
            }
220
        }
221 222
    }

223 224 225 226 227
    if(internal_timing_flag)
    {
        init_time = DARSHAN_MPI_CALL(PMPI_Wtime)() - init_start;
        DARSHAN_MPI_CALL(PMPI_Reduce)(&init_time, &init_max, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
228
        if(my_rank == 0)
229
        {
230 231
            fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
            fprintf(stderr, "darshan:init\t%d\t%f\n", nprocs, init_max);
232 233 234 235 236 237
        }
    }

    return;
}

Shane Snyder's avatar
Shane Snyder committed
238
void darshan_core_shutdown()
239
{
240
    int i;
241
    char *logfile_name;
242
    struct darshan_core_runtime *final_core;
243
    int internal_timing_flag = 0;
244 245
    char *envjobid;
    char *jobid_str;
246
    int jobid;
247
    struct tm *start_tm;
248
    time_t start_time_tmp;
249 250
    int ret = 0;
    int all_ret = 0;
251 252
    int64_t first_start_time;
    int64_t last_end_time;
253 254
    int local_mod_use[DARSHAN_MAX_MODS] = {0};
    int global_mod_use_count[DARSHAN_MAX_MODS] = {0};
255 256 257
    darshan_record_id *shared_recs;
    darshan_record_id *mod_shared_recs;
    int shared_rec_cnt = 0;
258
    double start_log_time;
259 260 261 262 263 264 265
    double open1, open2;
    double job1, job2;
    double rec1, rec2;
    double mod1[DARSHAN_MAX_MODS] = {0};
    double mod2[DARSHAN_MAX_MODS] = {0};
    double header1, header2;
    double tm_end;
266
    uint64_t gz_fp = 0;
267 268
    MPI_File log_fh;
    MPI_Status status;
269 270 271 272

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

273 274
    start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();

Shane Snyder's avatar
Shane Snyder committed
275
    /* disable darhan-core while we shutdown */
276
    DARSHAN_CORE_LOCK();
277
    if(!darshan_core)
278
    {
279
        DARSHAN_CORE_UNLOCK();
280 281
        return;
    }
282 283
    final_core = darshan_core;
    darshan_core = NULL;
Shane Snyder's avatar
Shane Snyder committed
284

285 286 287 288 289 290 291
    final_core->comp_buf = malloc(DARSHAN_COMP_BUF_SIZE);
    if(!(final_core->comp_buf))
    {
        darshan_core_cleanup(final_core);
        return;
    }

292
    /* we also need to set which modules were registered on this process and
293 294
     * call into those modules and give them a chance to perform any necessary
     * pre-shutdown steps.
Shane Snyder's avatar
Shane Snyder committed
295 296 297 298 299 300
     */
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
    {
        if(final_core->mod_array[i])
        {
            local_mod_use[i] = 1;
301
            final_core->mod_array[i]->mod_funcs.begin_shutdown();
Shane Snyder's avatar
Shane Snyder committed
302 303
        }
    }
304
    DARSHAN_CORE_UNLOCK();
305 306 307 308

    logfile_name = malloc(PATH_MAX);
    if(!logfile_name)
    {
309
        darshan_core_cleanup(final_core);
310 311 312
        return;
    }

313
    /* set darshan job id/metadata and constuct log file name on rank 0 */
314
    if(my_rank == 0)
315
    {
316
        /* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
317
        envjobid = getenv(DARSHAN_JOBID_OVERRIDE);
318
        if(!envjobid)
319
        {
320
            envjobid = __DARSHAN_JOBID;
321 322
        }

323
        /* find a job id */
324 325 326 327 328 329 330 331 332 333 334 335
        jobid_str = getenv(envjobid);
        if(jobid_str)
        {
            /* in cobalt we can find it in env var */
            ret = sscanf(jobid_str, "%d", &jobid);
        }
        if(!jobid_str || ret != 1)
        {
            /* use pid as fall back */
            jobid = getpid();
        }

336
        final_core->log_job.jobid = (int64_t)jobid;
337

338
        /* if we are using any hints to write the log file, then record those
339
         * hints with the darshan job information
340
         */
341
        darshan_log_record_hints_and_ver(final_core);
342

343
        /* use human readable start time format in log filename */
344
        start_time_tmp = final_core->log_job.start_time;
345
        start_tm = localtime(&start_time_tmp);
346

347 348
        /* construct log file name */
        darshan_get_logfile_name(logfile_name, jobid, start_tm);
349 350 351 352 353 354 355 356 357
    }

    /* broadcast log file name */
    DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
        MPI_COMM_WORLD);

    if(strlen(logfile_name) == 0)
    {
        /* failed to generate log file name */
358
        free(logfile_name);
359
        darshan_core_cleanup(final_core);
360 361 362
        return;
    }

363
    final_core->log_job.end_time = time(NULL);
364

365 366 367
    /* reduce to report first start time and last end time across all ranks
     * at rank 0
     */
368 369
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.start_time, &first_start_time, 1, MPI_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.end_time, &last_end_time, 1, MPI_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
370 371
    if(my_rank == 0)
    {
372 373
        final_core->log_job.start_time = first_start_time;
        final_core->log_job.end_time = last_end_time;
374
    }
375

376 377 378
    /* reduce the number of times a module was opened globally and bcast to everyone */   
    DARSHAN_MPI_CALL(PMPI_Allreduce)(local_mod_use, global_mod_use_count, DARSHAN_MAX_MODS, MPI_INT, MPI_SUM, MPI_COMM_WORLD);

379
    /* get a list of records which are shared across all processes */
380
    darshan_get_shared_records(final_core, &shared_recs, &shared_rec_cnt);
381

382 383
    if(internal_timing_flag)
        open1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
384
    /* collectively open the darshan log file */
385
    ret = darshan_log_open_all(logfile_name, &log_fh);
386 387
    if(internal_timing_flag)
        open2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
388 389 390 391 392 393 394 395

    /* error out if unable to open log file */
    DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
        MPI_LOR, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        if(my_rank == 0)
        {
396 397
            fprintf(stderr, "darshan library warning: unable to open log file %s\n",
                logfile_name);
398 399
        }
        free(logfile_name);
400
        darshan_core_cleanup(final_core);
401 402 403
        return;
    }

404 405
    if(internal_timing_flag)
        job1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
406
    /* rank 0 is responsible for writing the compressed darshan job information */
Shane Snyder's avatar
Shane Snyder committed
407
    if(my_rank == 0)
408
    {
409
        void *pointers[2] = {&final_core->log_job, final_core->trailing_data};
410
        int lengths[2] = {sizeof(struct darshan_job), strlen(final_core->trailing_data)};
411
        int comp_buf_sz = 0;
412

413
        /* compress the job info and the trailing mount/exe data */
Shane Snyder's avatar
Shane Snyder committed
414
        all_ret = darshan_deflate_buffer(pointers, lengths, 2,
415 416
            final_core->comp_buf, &comp_buf_sz);
        if(all_ret)
417
        {
418
            fprintf(stderr, "darshan library warning: unable to compress job data\n");
419
            unlink(logfile_name);
420
        }
421 422 423
        else
        {
            /* write the job information, preallocing space for the log header */
Shane Snyder's avatar
Shane Snyder committed
424
            gz_fp += sizeof(struct darshan_header);
425 426
            all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, gz_fp,
                final_core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
427 428 429 430 431
            if(all_ret != MPI_SUCCESS)
            {
                fprintf(stderr, "darshan library warning: unable to write job data to log file %s\n",
                        logfile_name);
                unlink(logfile_name);
Shane Snyder's avatar
Shane Snyder committed
432
                
433
            }
434
            gz_fp += comp_buf_sz;
435
        }
436 437
    }

438 439 440 441 442
    /* error out if unable to write job information */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        free(logfile_name);
443
        darshan_core_cleanup(final_core);
444 445
        return;
    }
446 447
    if(internal_timing_flag)
        job2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
448

449 450
    if(internal_timing_flag)
        rec1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
451
    /* write the record name->id hash to the log file */
Shane Snyder's avatar
Shane Snyder committed
452
    final_core->log_header.rec_map.off = gz_fp;
453
    ret = darshan_log_write_record_hash(log_fh, final_core, &gz_fp);
Shane Snyder's avatar
Shane Snyder committed
454
    final_core->log_header.rec_map.len = gz_fp - final_core->log_header.rec_map.off;
455

456
    /* error out if unable to write record hash */
457 458 459 460 461 462
    DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
        MPI_LOR, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        if(my_rank == 0)
        {
463
            fprintf(stderr, "darshan library warning: unable to write record hash to log file %s\n",
464
                logfile_name);
465
            unlink(logfile_name);
466 467
        }
        free(logfile_name);
468
        darshan_core_cleanup(final_core);
469 470
        return;
    }
Shane Snyder's avatar
Shane Snyder committed
471 472
    if(internal_timing_flag)
        rec2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
473

474 475 476
    mod_shared_recs = malloc(shared_rec_cnt * sizeof(darshan_record_id));
    assert(mod_shared_recs);

477
    /* loop over globally used darshan modules and:
478
     *      - perform shared file reductions, if possible
479
     *      - get final output buffer
480
     *      - compress (zlib) provided output buffer
Shane Snyder's avatar
Shane Snyder committed
481
     *      - append compressed buffer to log file
482 483
     *      - add module index info (file offset/length) to log header
     *      - shutdown the module
484
     */
485
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
486
    {
487
        struct darshan_core_module* this_mod = final_core->mod_array[i];
488
        struct darshan_core_record_ref *ref = NULL;
489
        int mod_shared_rec_cnt = 0;
490
        void* mod_buf = NULL;
491
        int mod_buf_sz = 0;
492
        int j;
493

494
        if(global_mod_use_count[i] == 0)
495 496
        {
            if(my_rank == 0)
497 498 499 500
            {
                final_core->log_header.mod_map[i].off = 0;
                final_core->log_header.mod_map[i].len = 0;
            }
501
            continue;
502
        }
503 504
 
        if(internal_timing_flag)
505
            mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
506

507
        /* set the shared file list for this module */
508 509
        memset(mod_shared_recs, 0, shared_rec_cnt * sizeof(darshan_record_id));
        for(j = 0; j < shared_rec_cnt; j++)
510 511 512 513
        {
            HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
                sizeof(darshan_record_id), ref);
            assert(ref);
514
            if(DARSHAN_MOD_FLAG_ISSET(ref->global_mod_flags, i))
515
            {
516
                mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
517
            }
518
        }
519

520 521 522 523 524
        /* if module is registered locally, get the corresponding output buffer
         * 
         * NOTE: this function can be used to run collective operations across
         * modules, if there are file records shared globally.
         */
525
        if(this_mod)
526
        {
527 528
            this_mod->mod_funcs.get_output_data(MPI_COMM_WORLD, mod_shared_recs,
                mod_shared_rec_cnt, &mod_buf, &mod_buf_sz);
529 530
        }

531
        /* append this module's data to the darshan log */
Shane Snyder's avatar
Shane Snyder committed
532 533 534 535
        final_core->log_header.mod_map[i].off = gz_fp;
        ret = darshan_log_append_all(log_fh, final_core, mod_buf, mod_buf_sz, &gz_fp);
        final_core->log_header.mod_map[i].len =
            gz_fp - final_core->log_header.mod_map[i].off;
536

537
        /* error out if the log append failed */
538 539 540
        DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
            MPI_LOR, MPI_COMM_WORLD);
        if(all_ret != 0)
541
        {
542 543 544 545 546 547 548 549
            if(my_rank == 0)
            {
                fprintf(stderr,
                    "darshan library warning: unable to write %s module data to log file %s\n",
                    darshan_module_names[i], logfile_name);
                unlink(logfile_name);
            }
            free(logfile_name);
550
            darshan_core_cleanup(final_core);
551
            return;
552 553 554
        }

        /* shutdown module if registered locally */
555
        if(this_mod)
556 557 558
        {
            this_mod->mod_funcs.shutdown();
        }
559 560
        if(internal_timing_flag)
            mod2[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
561 562
    }

563 564
    if(internal_timing_flag)
        header1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
565 566 567 568 569 570
    /* write out log header, after running 2 reduction on header variables:
     *  1) reduce 'partial_flag' variable to determine which modules ran out
     *     of memory for storing I/O data
     *  2) reduce 'mod_ver' array to determine which log format version each
     *     module used for this output log
     */
571 572
    if(my_rank == 0)
    {
573 574 575 576 577 578 579 580
        DARSHAN_MPI_CALL(PMPI_Reduce)(MPI_IN_PLACE,
            &(final_core->log_header.partial_flag), 1, MPI_UINT32_T,
            MPI_BOR, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(MPI_IN_PLACE,
            final_core->log_header.mod_ver, DARSHAN_MAX_MODS, MPI_UINT32_T,
            MPI_MAX, 0, MPI_COMM_WORLD);

        /* rank 0 is responsible for writing the log header */
581 582 583
        /* initialize the remaining header fields */
        strcpy(final_core->log_header.version_string, DARSHAN_LOG_VERSION);
        final_core->log_header.magic_nr = DARSHAN_MAGIC_NR;
584
        final_core->log_header.comp_type = DARSHAN_ZLIB_COMP;
585

Shane Snyder's avatar
Shane Snyder committed
586 587 588
        all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, 0, &(final_core->log_header),
            sizeof(struct darshan_header), MPI_BYTE, &status);
        if(all_ret != MPI_SUCCESS)
589
        {
Shane Snyder's avatar
Shane Snyder committed
590 591
            fprintf(stderr, "darshan library warning: unable to write header to log file %s\n",
                    logfile_name);
592
            unlink(logfile_name);
593
        }
594
    }
595 596 597 598 599 600 601 602 603
    else
    {
        DARSHAN_MPI_CALL(PMPI_Reduce)(&(final_core->log_header.partial_flag),
            &(final_core->log_header.partial_flag), 1, MPI_UINT32_T,
            MPI_BOR, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(final_core->log_header.mod_ver,
            final_core->log_header.mod_ver, DARSHAN_MAX_MODS, MPI_UINT32_T,
            MPI_MAX, 0, MPI_COMM_WORLD);
    }
604

605 606 607 608 609
    /* error out if unable to write log header */
    DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
    if(all_ret != 0)
    {
        free(logfile_name);
610
        darshan_core_cleanup(final_core);
611 612
        return;
    }
613 614
    if(internal_timing_flag)
        header2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
615

616 617 618
    DARSHAN_MPI_CALL(PMPI_File_close)(&log_fh);

    /* if we got this far, there are no errors, so rename from *.darshan_partial
619
     * to *-<logwritetime>.darshan, which indicates that this log file is
620 621
     * complete and ready for analysis
     */
622 623
    if(my_rank == 0)
    {
Shane Snyder's avatar
Shane Snyder committed
624
        if(getenv("DARSHAN_LOGFILE"))
625
        {
626
#ifdef __DARSHAN_GROUP_READABLE_LOGS
Shane Snyder's avatar
Shane Snyder committed
627
            chmod(logfile_name, (S_IRUSR|S_IRGRP));
628
#else
Shane Snyder's avatar
Shane Snyder committed
629
            chmod(logfile_name, (S_IRUSR));
630
#endif
Shane Snyder's avatar
Shane Snyder committed
631 632 633 634 635 636 637 638 639 640 641 642 643 644
        }
        else
        {
            char* tmp_index;
            double end_log_time;
            char* new_logfile_name;

            new_logfile_name = malloc(PATH_MAX);
            if(new_logfile_name)
            {
                new_logfile_name[0] = '\0';
                end_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
                strcat(new_logfile_name, logfile_name);
                tmp_index = strstr(new_logfile_name, ".darshan_partial");
645
                sprintf(tmp_index, "_%d.darshan", (int)(end_log_time-start_log_time+1));
Shane Snyder's avatar
Shane Snyder committed
646 647
                rename(logfile_name, new_logfile_name);
                /* set permissions on log file */
648
#ifdef __DARSHAN_GROUP_READABLE_LOGS
Shane Snyder's avatar
Shane Snyder committed
649 650 651 652 653 654
                chmod(new_logfile_name, (S_IRUSR|S_IRGRP));
#else
                chmod(new_logfile_name, (S_IRUSR));
#endif
                free(new_logfile_name);
            }
655
        }
656
    }
657

658
    free(logfile_name);
659
    darshan_core_cleanup(final_core);
660

661
    if(internal_timing_flag)
662
    {
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
        double open_tm, open_slowest;
        double header_tm, header_slowest;
        double job_tm, job_slowest;
        double rec_tm, rec_slowest;
        double mod_tm[DARSHAN_MAX_MODS], mod_slowest[DARSHAN_MAX_MODS];
        double all_tm, all_slowest;

        tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)();

        open_tm = open2 - open1;
        header_tm = header2 - header1;
        job_tm = job2 - job1;
        rec_tm = rec2 - rec1;
        all_tm = tm_end - start_log_time;
        for(i = 0;i < DARSHAN_MAX_MODS; i++)
        {
            mod_tm[i] = mod2[i] - mod1[i];
        }

        DARSHAN_MPI_CALL(PMPI_Reduce)(&open_tm, &open_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&header_tm, &header_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&job_tm, &job_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&rec_tm, &rec_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&all_tm, &all_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(mod_tm, mod_slowest, DARSHAN_MAX_MODS,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);

        if(my_rank == 0)
        {
697 698 699 700 701
            fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
            fprintf(stderr, "darshan:log_open\t%d\t%f\n", nprocs, open_slowest);
            fprintf(stderr, "darshan:job_write\t%d\t%f\n", nprocs, job_slowest);
            fprintf(stderr, "darshan:hash_write\t%d\t%f\n", nprocs, rec_slowest);
            fprintf(stderr, "darshan:header_write\t%d\t%f\n", nprocs, header_slowest);
702 703 704
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                if(global_mod_use_count[i])
705
                    fprintf(stderr, "darshan:%s_shutdown\t%d\t%f\n", darshan_module_names[i],
Shane Snyder's avatar
Shane Snyder committed
706
                        nprocs, mod_slowest[i]);
707
            }
708
            fprintf(stderr, "darshan:core_shutdown\t%d\t%f\n", nprocs, all_slowest);
709
        }
710 711 712 713
    }
    
    return;
}
714

Shane Snyder's avatar
Shane Snyder committed
715
/* *********************************** */
716

717
/* construct the darshan log file name */
718
static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* start_tm)
719
{
Shane Snyder's avatar
Shane Snyder committed
720
    char* user_logfile_name;
721 722 723
    char* logpath;
    char* logname_string;
    char* logpath_override = NULL;
724
#ifdef __DARSHAN_LOG_ENV
725 726 727 728 729 730 731 732 733
    char env_check[256];
    char* env_tok;
#endif
    uint64_t hlevel;
    char hname[HOST_NAME_MAX];
    uint64_t logmod;
    char cuser[L_cuserid] = {0};
    int ret;

Shane Snyder's avatar
Shane Snyder committed
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
    /* first, check if user specifies a complete logpath to use */
    user_logfile_name = getenv("DARSHAN_LOGFILE");
    if(user_logfile_name)
    {
        if(strlen(user_logfile_name) >= (PATH_MAX-1))
        {
            fprintf(stderr, "darshan library warning: user log file name too long.\n");
            logfile_name[0] = '\0';
        }
        else
        {
            strcpy(logfile_name, user_logfile_name);
        }
    }
    else
749
    {
Shane Snyder's avatar
Shane Snyder committed
750 751
        /* otherwise, generate the log path automatically */

752 753
        /* Use DARSHAN_LOG_PATH_OVERRIDE for the value or __DARSHAN_LOG_PATH */
        logpath = getenv(DARSHAN_LOG_PATH_OVERRIDE);
Shane Snyder's avatar
Shane Snyder committed
754 755
        if(!logpath)
        {
756 757
#ifdef __DARSHAN_LOG_PATH
            logpath = __DARSHAN_LOG_PATH;
758
#endif
Shane Snyder's avatar
Shane Snyder committed
759
        }
760

Shane Snyder's avatar
Shane Snyder committed
761 762 763 764 765 766 767 768 769 770
        /* get the username for this job.  In order we will try each of the
         * following until one of them succeeds:
         *
         * - cuserid()
         * - getenv("LOGNAME")
         * - snprintf(..., geteuid());
         *
         * Note that we do not use getpwuid() because it generally will not
         * work in statically compiled binaries.
         */
771 772

#ifndef DARSHAN_DISABLE_CUSERID
Shane Snyder's avatar
Shane Snyder committed
773
        cuserid(cuser);
774 775
#endif

Shane Snyder's avatar
Shane Snyder committed
776 777
        /* if cuserid() didn't work, then check the environment */
        if(strcmp(cuser, "") == 0)
778
        {
Shane Snyder's avatar
Shane Snyder committed
779 780 781 782 783
            logname_string = getenv("LOGNAME");
            if(logname_string)
            {
                strncpy(cuser, logname_string, (L_cuserid-1));
            }
784 785
        }

Shane Snyder's avatar
Shane Snyder committed
786 787 788 789 790 791
        /* if cuserid() and environment both fail, then fall back to uid */
        if(strcmp(cuser, "") == 0)
        {
            uid_t uid = geteuid();
            snprintf(cuser, sizeof(cuser), "%u", uid);
        }
792

Shane Snyder's avatar
Shane Snyder committed
793 794 795 796
        /* generate a random number to help differentiate the log */
        hlevel=DARSHAN_MPI_CALL(PMPI_Wtime)() * 1000000;
        (void)gethostname(hname, sizeof(hname));
        logmod = darshan_hash((void*)hname,strlen(hname),hlevel);
797

Shane Snyder's avatar
Shane Snyder committed
798 799 800 801
        /* see if darshan was configured using the --with-logpath-by-env
         * argument, which allows the user to specify an absolute path to
         * place logs via an env variable.
         */
802
#ifdef __DARSHAN_LOG_ENV
Shane Snyder's avatar
Shane Snyder committed
803
        /* just silently skip if the environment variable list is too big */
804
        if(strlen(__DARSHAN_LOG_ENV) < 256)
805
        {
Shane Snyder's avatar
Shane Snyder committed
806
            /* copy env variable list to a temporary buffer */
807
            strcpy(env_check, __DARSHAN_LOG_ENV);
Shane Snyder's avatar
Shane Snyder committed
808 809 810
            /* tokenize the comma-separated list */
            env_tok = strtok(env_check, ",");
            if(env_tok)
811
            {
Shane Snyder's avatar
Shane Snyder committed
812
                do
813
                {
Shane Snyder's avatar
Shane Snyder committed
814 815 816 817 818 819 820 821 822
                    /* check each env variable in order */
                    logpath_override = getenv(env_tok);
                    if(logpath_override)
                    {
                        /* stop as soon as we find a match */
                        break;
                    }
                }while((env_tok = strtok(NULL, ",")));
            }
823 824 825
        }
#endif

Shane Snyder's avatar
Shane Snyder committed
826
        if(logpath_override)
827
        {
Shane Snyder's avatar
Shane Snyder committed
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
            ret = snprintf(logfile_name, PATH_MAX,
                "%s/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                logpath_override,
                cuser, __progname, jobid,
                (start_tm->tm_mon+1),
                start_tm->tm_mday,
                (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
                logmod);
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath_override, jobid);
            }
843
        }
Shane Snyder's avatar
Shane Snyder committed
844
        else if(logpath)
845
        {
Shane Snyder's avatar
Shane Snyder committed
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
            ret = snprintf(logfile_name, PATH_MAX,
                "%s/%d/%d/%d/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                logpath, (start_tm->tm_year+1900),
                (start_tm->tm_mon+1), start_tm->tm_mday,
                cuser, __progname, jobid,
                (start_tm->tm_mon+1),
                start_tm->tm_mday,
                (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
                logmod);
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath, jobid);
            }
        }
        else
        {
            logfile_name[0] = '\0';
866 867 868 869
        }
    }

    return;
870 871
}

872
/* record any hints used to write the darshan log in the log header */
873
static void darshan_log_record_hints_and_ver(struct darshan_core_runtime* core)
874 875 876 877 878 879 880 881 882
{
    char* hints;
    char* header_hints;
    int meta_remain = 0;
    char* m;

    /* check environment variable to see if the default MPI file hints have
     * been overridden
     */
883
    hints = getenv(DARSHAN_LOG_HINTS_OVERRIDE);
884 885
    if(!hints)
    {
886
        hints = __DARSHAN_LOG_HINTS;