darshan-core.c

/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"

#include <stdio.h>
#ifdef HAVE_MNTENT_H
#include <mntent.h>
#endif
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <pthread.h>
#include <fcntl.h>
#include <stdarg.h>
#include <dirent.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/time.h>
#include <sys/vfs.h>
#include <zlib.h>
#include <assert.h>

#ifdef HAVE_MPI
#include <mpi.h>
#endif

#include "uthash.h"
#include "darshan.h"
#include "darshan-core.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"

#ifdef DARSHAN_LUSTRE
#include <lustre/lustre_user.h>
#endif

extern char* __progname;
extern char* __progname_full;

/* internal variable delcarations */
static struct darshan_core_runtime *darshan_core = NULL;
static pthread_mutex_t darshan_core_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int using_mpi = 0;
static int my_rank = 0;
static int nprocs = 1;
static int darshan_mem_alignment = 1;
static long darshan_mod_mem_quota = DARSHAN_MOD_MEM_MAX;

static struct darshan_core_mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;

/* paths prefixed with the following directories are not tracked by darshan */
char* darshan_path_exclusions[] = {
    "/etc/",
    "/dev/",
    "/usr/",
    "/bin/",
    "/boot/",
    "/lib/",
    "/opt/",
    "/sbin/",
    "/sys/",
    "/proc/",
    "/var/",
    NULL
};
/* paths prefixed with the following directories are tracked by darshan even if
 * they share a root with a path listed in darshan_path_exclusions
 */
char* darshan_path_inclusions[] = {
    "/var/opt/cray/dws/mounts/",
    NULL
};

/* allow users to override the path exclusions */
char** user_darshan_path_exclusions = NULL;

#ifdef DARSHAN_BGQ
extern void bgq_runtime_initialize();
#endif

/* array of init functions for modules which need to be statically
 * initialized by darshan at startup time
 */
void (*mod_static_init_fns[])(void) =
{
#ifdef DARSHAN_BGQ
    &bgq_runtime_initialize,
#endif
    NULL
};

#ifdef DARSHAN_LUSTRE
/* XXX need to use extern to get Lustre module's instrumentation function
 * since modules have no way of providing this to darshan-core
 */
extern void darshan_instrument_lustre_file(const char *filepath, int fd);
#endif

/* prototypes for internal helper functions */
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
static void *darshan_init_mmap_log(
    struct darshan_core_runtime* core, int jobid);
#endif
static void darshan_log_record_hints_and_ver(
    struct darshan_core_runtime* core);
static void darshan_get_exe_and_mounts(
    struct darshan_core_runtime *core, int argc, char **argv);
static void darshan_fs_info_from_path(
    const char *path, struct darshan_fs_info *fs_info);
static int darshan_add_name_record_ref(
    struct darshan_core_runtime *core, darshan_record_id rec_id,
    const char *name, darshan_module_id mod_id);
static void darshan_get_user_name(
    char *user);
#ifdef HAVE_MPI
static void darshan_get_shared_records(
    struct darshan_core_runtime *core, darshan_record_id **shared_recs,
    int *shared_rec_cnt);
#endif
static void darshan_get_logfile_name(
    char* logfile_name, struct darshan_core_runtime* core);
static int darshan_log_open(
    char *logfile_name, struct darshan_core_runtime *core,
    darshan_core_log_fh *log_fh);
static int darshan_log_write_job_record(
    darshan_core_log_fh log_fh, struct darshan_core_runtime *core,
    uint64_t *inout_off);
static int darshan_log_write_name_record_hash(
    darshan_core_log_fh log_fh, struct darshan_core_runtime *core,
    uint64_t *inout_off);
static int darshan_log_write_header(
    darshan_core_log_fh log_fh, struct darshan_core_runtime *core);
static int darshan_log_append(
    darshan_core_log_fh log_fh, struct darshan_core_runtime *core,
    void *buf, int count, uint64_t *inout_off);
void darshan_log_close(
    darshan_core_log_fh log_fh);
void darshan_log_finalize(
    char *logfile_name, double start_log_time);
static int darshan_deflate_buffer(
    void **pointers, int *lengths, int count, char *comp_buf,
    int *comp_buf_length);
static void darshan_core_cleanup(
    struct darshan_core_runtime* core);
static double darshan_core_wtime_absolute(void);

#define DARSHAN_CORE_LOCK() pthread_mutex_lock(&darshan_core_mutex)
#define DARSHAN_CORE_UNLOCK() pthread_mutex_unlock(&darshan_core_mutex)

#define DARSHAN_WARN(__err_str, ...) do { \
    darshan_core_fprintf(stderr, "darshan_library_warning: " \
        __err_str ".\n", ## __VA_ARGS__); \
} while(0)

#ifdef HAVE_MPI

/* MPI variant of darshan logging helpers */
#define DARSHAN_CHECK_ERR(__ret, __err_str, ...) do { \
    if(using_mpi) \
        PMPI_Allreduce(MPI_IN_PLACE, &__ret, 1, MPI_INT, MPI_LOR, final_core->mpi_comm); \
    if(__ret != 0) { \
        if(my_rank == 0) { \
            DARSHAN_WARN(__err_str, ## __VA_ARGS__); \
            if(log_created) \
                unlink(logfile_name); \
        } \
        goto exit; \
    } \
} while(0)

#else

/* Non-MPI variant of darshan logging helpers */
#define DARSHAN_CHECK_ERR(__ret, __err_str, ...) do { \
    if(__ret != 0) { \
        DARSHAN_WARN(__err_str, ## __VA_ARGS__); \
        if(log_created) \
            unlink(logfile_name); \
        goto exit; \
    } \
} while(0)

#endif

/* *********************************** */

void darshan_core_initialize(int argc, char **argv)
{
    struct darshan_core_runtime *init_core = NULL;
    int internal_timing_flag = 0;
    double init_start, init_time;
    char *envstr;
    char *jobid_str;
    int jobid;
    int ret;
    int i;
    int tmpval;
    double tmpfloat;

    /* setup darshan runtime if darshan is enabled and hasn't been initialized already */
    if (darshan_core != NULL || getenv("DARSHAN_DISABLE"))
        return;

    if(getenv("DARSHAN_INTERNAL_TIMING"))
    {
        internal_timing_flag = 1;
        init_start = darshan_core_wtime();
    }

    #if (__DARSHAN_MEM_ALIGNMENT < 1)
        #error Darshan must be configured with a positive value for --with-mem-align
    #endif
    envstr = getenv(DARSHAN_MEM_ALIGNMENT_OVERRIDE);
    if(envstr)
    {
        ret = sscanf(envstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            darshan_mem_alignment = tmpval;
        }
    }
    else
    {
        darshan_mem_alignment = __DARSHAN_MEM_ALIGNMENT;
    }

    /* avoid floating point errors on faulty input */
    if(darshan_mem_alignment < 1)
    {
        darshan_mem_alignment = 1;
    }

    /* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
    envstr = getenv(DARSHAN_JOBID_OVERRIDE);
    if(!envstr)
    {
        envstr = __DARSHAN_JOBID;
    }

    /* find a job id */
    jobid_str = getenv(envstr);
    if(jobid_str)
    {
        /* in cobalt we can find it in env var */
        ret = sscanf(jobid_str, "%d", &jobid);
    }
    if(!jobid_str || ret != 1)
    {
        /* use pid as fall back */
        jobid = getpid();
    }

    /* set the memory quota for darshan modules' records */
    envstr = getenv(DARSHAN_MOD_MEM_OVERRIDE);
    if(envstr)
    {
        ret = sscanf(envstr, "%lf", &tmpfloat);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpfloat > 0)
        {
            darshan_mod_mem_quota = tmpfloat * 1024 * 1024; /* convert from MiB */
        }
    }

    /* allocate structure to track darshan core runtime information */
    init_core = malloc(sizeof(*init_core));
    if(init_core)
    {
        memset(init_core, 0, sizeof(*init_core));

#ifdef HAVE_MPI
        PMPI_Initialized(&using_mpi);
        if(using_mpi)
        {
            PMPI_Comm_dup(MPI_COMM_WORLD, &init_core->mpi_comm);
            PMPI_Comm_size(init_core->mpi_comm, &nprocs);
            PMPI_Comm_rank(init_core->mpi_comm, &my_rank);
        }
#endif

        /* record absolute start time at startup so that we can later
         * generate relative times with this as a reference point.
         */
        init_core->wtime_offset = darshan_core_wtime_absolute();

    /* TODO: do we alloc new memory as we go or just do everything up front? */

#ifndef __DARSHAN_ENABLE_MMAP_LOGS
        /* just allocate memory for each log file region */
        init_core->log_hdr_p = malloc(sizeof(struct darshan_header));
        init_core->log_job_p = malloc(sizeof(struct darshan_job));
        init_core->log_exemnt_p = malloc(DARSHAN_EXE_LEN+1);
        init_core->log_name_p = malloc(DARSHAN_NAME_RECORD_BUF_SIZE);
        init_core->log_mod_p = malloc(darshan_mod_mem_quota);

        if(!(init_core->log_hdr_p) || !(init_core->log_job_p) ||
           !(init_core->log_exemnt_p) || !(init_core->log_name_p) ||
           !(init_core->log_mod_p))
        {
            free(init_core);
            return;
        }
        /* if allocation succeeds, zero fill memory regions */
        memset(init_core->log_hdr_p, 0, sizeof(struct darshan_header));
        memset(init_core->log_job_p, 0, sizeof(struct darshan_job));
        memset(init_core->log_exemnt_p, 0, DARSHAN_EXE_LEN+1);
        memset(init_core->log_name_p, 0, DARSHAN_NAME_RECORD_BUF_SIZE);
        memset(init_core->log_mod_p, 0, darshan_mod_mem_quota);
#else
        /* if mmap logs are enabled, we need to initialize the mmap region
         * before setting the corresponding log file region pointers
         */
        void *mmap_p = darshan_init_mmap_log(init_core, jobid);
        if(!mmap_p)
        {
            free(init_core);
            return;
        }

        /* set the memory pointers for each log file region */
        init_core->log_hdr_p = (struct darshan_header *)mmap_p;
        init_core->log_job_p = (struct darshan_job *)
            ((char *)init_core->log_hdr_p + sizeof(struct darshan_header));
        init_core->log_exemnt_p = (char *)
            ((char *)init_core->log_job_p + sizeof(struct darshan_job));
        init_core->log_name_p = (void *)
            ((char *)init_core->log_exemnt_p + DARSHAN_EXE_LEN + 1);
        init_core->log_mod_p = (void *)
            ((char *)init_core->log_name_p + DARSHAN_NAME_RECORD_BUF_SIZE);

        /* set header fields needed for the mmap log mechanism */
        init_core->log_hdr_p->comp_type = DARSHAN_NO_COMP;
        init_core->log_hdr_p->name_map.off =
            ((char *)init_core->log_name_p - (char *)init_core->log_hdr_p);
#endif

        /* set known header fields for the log file */
        strcpy(init_core->log_hdr_p->version_string, DARSHAN_LOG_VERSION);
        init_core->log_hdr_p->magic_nr = DARSHAN_MAGIC_NR;

        /* set known job-level metadata fields for the log file */
        init_core->log_job_p->uid = getuid();
        init_core->log_job_p->start_time = time(NULL);
        init_core->log_job_p->nprocs = nprocs;
        init_core->log_job_p->jobid = (int64_t)jobid;

        /* if we are using any hints to write the log file, then record those
         * hints with the darshan job information
         */
        darshan_log_record_hints_and_ver(init_core);

        /* collect information about command line and mounted file systems */
        darshan_get_exe_and_mounts(init_core, argc, argv);

        /* determine if/when DXT should be enabled by looking for triggers */
        char *trigger_conf = getenv("DXT_TRIGGER_CONF_PATH");
        if(trigger_conf)
        {
            dxt_load_trigger_conf(trigger_conf);
        }

        /* if darshan was successfully initialized, set the global pointer
         * and bootstrap any modules with static initialization routines
         */
        DARSHAN_CORE_LOCK();
        darshan_core = init_core;
        DARSHAN_CORE_UNLOCK();

        i = 0;
        while(mod_static_init_fns[i])
        {
            (*mod_static_init_fns[i])();
            i++;
        }
    }

    if(internal_timing_flag)
    {
        init_time = darshan_core_wtime() - init_start;
#ifdef HAVE_MPI
        if(using_mpi)
        {
            if(my_rank == 0)
            {
                PMPI_Reduce(MPI_IN_PLACE, &init_time, 1,
                    MPI_DOUBLE, MPI_MAX, 0, darshan_core->mpi_comm);
            }
            else
            {
                PMPI_Reduce(&init_time, &init_time, 1,
                    MPI_DOUBLE, MPI_MAX, 0, darshan_core->mpi_comm);
                return; /* return early so every rank doesn't print */
            }
        }
#endif

        darshan_core_fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
        darshan_core_fprintf(stderr, "darshan:init\t%d\t%f\n", nprocs, init_time);
    }

    return;
}

void darshan_core_shutdown()
{
    struct darshan_core_runtime *final_core;
    double start_log_time;
    int internal_timing_flag = 0;
    double open1 = 0, open2 = 0;
    double job1 = 0, job2 = 0;
    double rec1 = 0, rec2 = 0;
    double mod1[DARSHAN_MAX_MODS] = {0};
    double mod2[DARSHAN_MAX_MODS] = {0};
    double header1 = 0, header2 = 0;
    double tm_end;
    int active_mods[DARSHAN_MAX_MODS] = {0};
    uint64_t gz_fp = 0;
    char *logfile_name = NULL;
    darshan_core_log_fh log_fh;
    int log_created = 0;
    int i;
    int ret;

    /* disable darhan-core while we shutdown */
    DARSHAN_CORE_LOCK();
    if(!darshan_core)
    {
        DARSHAN_CORE_UNLOCK();
        return;
    }
    final_core = darshan_core;
    darshan_core = NULL;
    DARSHAN_CORE_UNLOCK();

    /* grab some initial timing information */
#ifdef HAVE_MPI
    /* if using mpi, sync across procs first */
    if(using_mpi)
        PMPI_Barrier(final_core->mpi_comm);
#endif
    start_log_time = darshan_core_wtime();
    final_core->log_job_p->end_time = time(NULL);

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

#ifdef __DARSHAN_ENABLE_MMAP_LOGS
    /* remove the temporary mmap log files */
    /* NOTE: this unlink is not immediate as it must wait for the mapping
     * to no longer be referenced, which in our case happens when the
     * executable exits. If the application terminates mid-shutdown, then
     * there will be no mmap files and no final log file.
     */
    unlink(final_core->mmap_log_name);
#endif

    final_core->comp_buf = malloc(darshan_mod_mem_quota);
    logfile_name = malloc(PATH_MAX);
    if(!final_core->comp_buf || !logfile_name)
        goto exit;

    /* set which modules were used locally */
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
    {
        if(final_core->mod_array[i])
            active_mods[i] = 1;
    }

#ifdef HAVE_MPI
    darshan_record_id *shared_recs = NULL;
    darshan_record_id *mod_shared_recs = NULL;
    int shared_rec_cnt = 0;

    if(using_mpi)
    {
        /* allreduce locally active mods to determine globally active mods */
        PMPI_Allreduce(MPI_IN_PLACE, active_mods, DARSHAN_MAX_MODS, MPI_INT,
            MPI_SUM, final_core->mpi_comm);

        /* reduce to report first start and last end time across all ranks at rank 0 */
        if(my_rank == 0)
        {
            PMPI_Reduce(MPI_IN_PLACE, &final_core->log_job_p->start_time,
                1, MPI_INT64_T, MPI_MIN, 0, final_core->mpi_comm);
            PMPI_Reduce(MPI_IN_PLACE, &final_core->log_job_p->end_time,
                1, MPI_INT64_T, MPI_MAX, 0, final_core->mpi_comm);
        }
        else
        {
            PMPI_Reduce(&final_core->log_job_p->start_time,
                &final_core->log_job_p->start_time,
                1, MPI_INT64_T, MPI_MIN, 0, final_core->mpi_comm);
            PMPI_Reduce(&final_core->log_job_p->end_time,
                &final_core->log_job_p->end_time,
                1, MPI_INT64_T, MPI_MAX, 0, final_core->mpi_comm);
        }

        /* get a list of records which are shared across all processes */
        darshan_get_shared_records(final_core, &shared_recs, &shared_rec_cnt);

        mod_shared_recs = malloc(shared_rec_cnt * sizeof(darshan_record_id));
        assert(mod_shared_recs);
    }
#endif

    /* get the log file name */
    darshan_get_logfile_name(logfile_name, final_core);
    if(strlen(logfile_name) == 0)
    {
        /* failed to generate log file name */
        goto exit;
    }

    if(internal_timing_flag)
        open1 = darshan_core_wtime();
    /* open the darshan log file */
    ret = darshan_log_open(logfile_name, final_core, &log_fh);
    if(internal_timing_flag)
        open2 = darshan_core_wtime();
    /* error out if unable to open log file */
    DARSHAN_CHECK_ERR(ret, "unable to create log file %s", logfile_name);
    log_created = 1;

    if(internal_timing_flag)
        job1 = darshan_core_wtime();
    /* write the the compressed darshan job information */
    ret = darshan_log_write_job_record(log_fh, final_core, &gz_fp);
    if(internal_timing_flag)
        job2 = darshan_core_wtime();
    /* error out if unable to write job information */
    DARSHAN_CHECK_ERR(ret, "unable to write job record to file %s", logfile_name);

    if(internal_timing_flag)
        rec1 = darshan_core_wtime();
    /* write the record name->id hash to the log file */
    final_core->log_hdr_p->name_map.off = gz_fp;
    ret = darshan_log_write_name_record_hash(log_fh, final_core, &gz_fp);
    if(internal_timing_flag)
        rec2 = darshan_core_wtime();
    final_core->log_hdr_p->name_map.len = gz_fp - final_core->log_hdr_p->name_map.off;
    /* error out if unable to write name records */
    DARSHAN_CHECK_ERR(ret, "unable to write name records to log file %s", logfile_name);

    /* loop over globally used darshan modules and:
     *      - get final output buffer
     *      - compress (zlib) provided output buffer
     *      - append compressed buffer to log file
     *      - add module map info (file offset/length) to log header
     *      - shutdown the module
     */
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
    {
        struct darshan_core_module* this_mod = final_core->mod_array[i];
        void* mod_buf = NULL;
        int mod_buf_sz = 0;

        if(!active_mods[i])
        {
            final_core->log_hdr_p->mod_map[i].off = 0;
            final_core->log_hdr_p->mod_map[i].len = 0;
            continue;
        }

        if(internal_timing_flag)
            mod1[i] = darshan_core_wtime();

        /* if module is registered locally, perform module shutdown operations */
        if(this_mod)
        {
            mod_buf = final_core->mod_array[i]->rec_buf_start;
            mod_buf_sz = final_core->mod_array[i]->rec_buf_p - mod_buf;

#ifdef HAVE_MPI
            if(using_mpi)
            {
                struct darshan_core_name_record_ref *ref = NULL;
                int mod_shared_rec_cnt = 0;
                int j;

                /* set the shared record list for this module */
                for(j = 0; j < shared_rec_cnt; j++)
                {
                    HASH_FIND(hlink, final_core->name_hash, &shared_recs[j],
                        sizeof(darshan_record_id), ref);
                    assert(ref);
                    if(DARSHAN_MOD_FLAG_ISSET(ref->global_mod_flags, i))
                    {
                        mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
                    }
                }

                /* allow the module an opportunity to reduce shared files */
                if(this_mod->mod_funcs.mod_redux_func && (mod_shared_recs > 0) &&
                   (!getenv("DARSHAN_DISABLE_SHARED_REDUCTION")))
                    this_mod->mod_funcs.mod_redux_func(mod_buf, final_core->mpi_comm,
                        mod_shared_recs, mod_shared_rec_cnt);
            }
#endif

            /* get the final output buffer */
            this_mod->mod_funcs.mod_shutdown_func(&mod_buf, &mod_buf_sz);
        }

        /* append this module's data to the darshan log */
        final_core->log_hdr_p->mod_map[i].off = gz_fp;
        ret = darshan_log_append(log_fh, final_core, mod_buf, mod_buf_sz, &gz_fp);
        final_core->log_hdr_p->mod_map[i].len =
            gz_fp - final_core->log_hdr_p->mod_map[i].off;

        /* XXX: DXT manages its own module memory buffers, so we need to
         * explicitly free them
         */
        if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
            free(mod_buf);

        if(internal_timing_flag)
            mod2[i] = darshan_core_wtime();

        /* error out if unable to write module data */
        DARSHAN_CHECK_ERR(ret, "unable to write %s module data to log file %s",
            darshan_module_names[i], logfile_name);
    }

    if(internal_timing_flag)
        header1 = darshan_core_wtime();
    ret = darshan_log_write_header(log_fh, final_core);
    if(internal_timing_flag)
        header2 = darshan_core_wtime();
    DARSHAN_CHECK_ERR(ret, "unable to write header to file %s", logfile_name);

    /* done writing data, close the log file */
    darshan_log_close(log_fh);

    /* finalize log file name and permissions */
    darshan_log_finalize(logfile_name, start_log_time);

    if(internal_timing_flag)
    {
        double open_tm;
        double header_tm;
        double job_tm;
        double rec_tm;
        double mod_tm[DARSHAN_MAX_MODS];
        double all_tm;

        tm_end = darshan_core_wtime();

        open_tm = open2 - open1;
        header_tm = header2 - header1;
        job_tm = job2 - job1;
        rec_tm = rec2 - rec1;
        all_tm = tm_end - start_log_time;
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            mod_tm[i] = mod2[i] - mod1[i];
        }

#ifdef HAVE_MPI
        if(using_mpi)
        {
            if(my_rank == 0)
            {
                PMPI_Reduce(MPI_IN_PLACE, &open_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(MPI_IN_PLACE, &header_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(MPI_IN_PLACE, &job_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(MPI_IN_PLACE, &rec_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(MPI_IN_PLACE, &all_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(MPI_IN_PLACE, mod_tm, DARSHAN_MAX_MODS,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
            }
            else
            {
                PMPI_Reduce(&open_tm, &open_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(&header_tm, &header_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(&job_tm, &job_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(&rec_tm, &rec_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(&all_tm, &all_tm, 1,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);
                PMPI_Reduce(mod_tm, mod_tm, DARSHAN_MAX_MODS,
                    MPI_DOUBLE, MPI_MAX, 0, final_core->mpi_comm);

                /* let rank 0 report the timing info */
                goto exit;
            }
        }
#endif

        darshan_core_fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
        darshan_core_fprintf(stderr, "darshan:log_open\t%d\t%f\n", nprocs, open_tm);
        darshan_core_fprintf(stderr, "darshan:job_write\t%d\t%f\n", nprocs, job_tm);
        darshan_core_fprintf(stderr, "darshan:hash_write\t%d\t%f\n", nprocs, rec_tm);
        darshan_core_fprintf(stderr, "darshan:header_write\t%d\t%f\n", nprocs, header_tm);
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(active_mods[i])
                darshan_core_fprintf(stderr, "darshan:%s_shutdown\t%d\t%f\n",
                    darshan_module_names[i], nprocs, mod_tm[i]);
        }
        darshan_core_fprintf(stderr, "darshan:core_shutdown\t%d\t%f\n", nprocs, all_tm);
    }

exit:
#ifdef HAVE_MPI
    if(using_mpi)
    {
        free(shared_recs);
        free(mod_shared_recs);
    }
#endif
    free(logfile_name);
    darshan_core_cleanup(final_core);

    return;
}

/* *********************************** */

#ifdef __DARSHAN_ENABLE_MMAP_LOGS
static void *darshan_init_mmap_log(struct darshan_core_runtime* core, int jobid)
{
    int ret;
    int mmap_fd;
    int mmap_size;
    int sys_page_size;
    char cuser[L_cuserid] = {0};
    uint64_t hlevel;
    char hname[HOST_NAME_MAX];
    uint64_t logmod;
    char *envstr;
    char *mmap_log_path;
    void *mmap_p;

    sys_page_size = sysconf(_SC_PAGESIZE);
    assert(sys_page_size > 0);

    mmap_size = sizeof(struct darshan_header) + DARSHAN_JOB_RECORD_SIZE +
        + DARSHAN_NAME_RECORD_BUF_SIZE + darshan_mod_mem_quota;
    if(mmap_size % sys_page_size)
        mmap_size = ((mmap_size / sys_page_size) + 1) * sys_page_size;

    envstr = getenv(DARSHAN_MMAP_LOG_PATH_OVERRIDE);
    if(envstr)
        mmap_log_path = envstr;
    else
        mmap_log_path = DARSHAN_DEF_MMAP_LOG_PATH;

    darshan_get_user_name(cuser);

    /* generate a random number to help differentiate the temporary log */
    /* NOTE: job id is not sufficient for constructing a unique log file name,
     * since a job could be composed of multiple application runs, so we also
     * add a random number component to the log name
     */
    if(my_rank == 0)
    {
        hlevel = darshan_core_wtime_absolute() * 1000000;
        (void)gethostname(hname, sizeof(hname));
        logmod = darshan_hash((void*)hname,strlen(hname),hlevel);
    }
    PMPI_Bcast(&logmod, 1, MPI_UINT64_T, 0, core->mpi_comm);

    /* construct a unique temporary log file name for this process
     * to write mmap log data to
     */
    snprintf(core->mmap_log_name, PATH_MAX,
        "/%s/%s_%s_id%d_mmap-log-%" PRIu64 "-%d.darshan",
        mmap_log_path, cuser, __progname, jobid, logmod, my_rank);

    /* create the temporary mmapped darshan log */
    mmap_fd = open(core->mmap_log_name, O_CREAT|O_RDWR|O_EXCL , 0644);
    if(mmap_fd < 0)
    {
        darshan_core_fprintf(stderr, "darshan library warning: "
            "unable to create darshan log file %s\n", core->mmap_log_name);
        return(NULL);
    }

    /* TODO: ftruncate or just zero fill? */
    /* allocate the necessary space in the log file */
    ret = ftruncate(mmap_fd, mmap_size);
    if(ret < 0)
    {
        darshan_core_fprintf(stderr, "darshan library warning: "
            "unable to allocate darshan log file %s\n", core->mmap_log_name);
        close(mmap_fd);
        unlink(core->mmap_log_name);
        return(NULL);
    }

    /* create the memory map for darshan's data structures so they are
     * persisted to file as the application executes
     */
    mmap_p = mmap(NULL, mmap_size, PROT_WRITE, MAP_SHARED, mmap_fd, 0);
    if(mmap_p == MAP_FAILED)
    {
        darshan_core_fprintf(stderr, "darshan library warning: "
            "unable to mmap darshan log file %s\n", core->mmap_log_name);
        close(mmap_fd);
        unlink(core->mmap_log_name);
        return(NULL);
    }

    /* close darshan log file (this does *not* unmap the log file) */
    close(mmap_fd);

    return(mmap_p);
}
#endif

/* record any hints used to write the darshan log in the job data */
static void darshan_log_record_hints_and_ver(struct darshan_core_runtime* core)
{
    char* hints;
    char* job_hints;
    int meta_remain = 0;
    char* m;

    /* check environment variable to see if the default MPI file hints have
     * been overridden
     */
    hints = getenv(DARSHAN_LOG_HINTS_OVERRIDE);
    if(!hints)
    {
        hints = __DARSHAN_LOG_HINTS;
    }

    if(!hints || strlen(hints) < 1)
        return;

    job_hints = strdup(hints);
    if(!job_hints)
        return;

    meta_remain = DARSHAN_JOB_METADATA_LEN -
        strlen(core->log_job_p->metadata) - 1;
    if(meta_remain >= (strlen(PACKAGE_VERSION) + 9))
    {
        sprintf(core->log_job_p->metadata, "lib_ver=%s\n", PACKAGE_VERSION);
        meta_remain -= (strlen(PACKAGE_VERSION) + 9);
    }
    if(meta_remain >= (3 + strlen(job_hints)))
    {
        m = core->log_job_p->metadata + strlen(core->log_job_p->metadata);
        /* We have room to store the hints in the metadata portion of
         * the job structure.  We just prepend an h= to the hints list.  The
         * metadata parser will ignore = characters that appear in the value
         * portion of the metadata key/value pair.
         */
        sprintf(m, "h=%s\n", job_hints);
    }
    free(job_hints);

    return;
}

static int mnt_data_cmp(const void* a, const void* b)
{
    const struct darshan_core_mnt_data *d_a = (const struct darshan_core_mnt_data*)a;
    const struct darshan_core_mnt_data *d_b = (const struct darshan_core_mnt_data*)b;

    if(strlen(d_a->path) > strlen(d_b->path))
        return(-1);
    else if(strlen(d_a->path) < strlen(d_b->path))
        return(1);
    else
        return(0);
}

/* adds an entry to table of mounted file systems */
static void add_entry(char* buf, int* space_left, struct mntent* entry)
{
    int i;
    int ret;
    char tmp_mnt[256];
    struct statfs statfsbuf;

    /* avoid adding the same mount points multiple times -- to limit
     * storage space and potential statfs, ioctl, etc calls
     */
    for(i = 0; i < mnt_data_count; i++)
    {
        if((strncmp(mnt_data_array[i].path, entry->mnt_dir, DARSHAN_MAX_MNT_PATH) == 0) &&
           (strncmp(mnt_data_array[i].type, entry->mnt_type, DARSHAN_MAX_MNT_PATH) == 0))
            return;
    }

    strncpy(mnt_data_array[mnt_data_count].path, entry->mnt_dir,
        DARSHAN_MAX_MNT_PATH-1);
    strncpy(mnt_data_array[mnt_data_count].type, entry->mnt_type,
        DARSHAN_MAX_MNT_TYPE-1);
    /* NOTE: we now try to detect the preferred block size for each file
     * system using fstatfs().  On Lustre we assume a size of 1 MiB
     * because fstatfs() reports 4 KiB.
     */
#ifndef LL_SUPER_MAGIC
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
    ret = statfs(entry->mnt_dir, &statfsbuf);
    mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
    if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
        mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
    else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
        mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
    else
        mnt_data_array[mnt_data_count].fs_info.block_size = 4096;

#ifdef DARSHAN_LUSTRE
    /* attempt to retrieve OST and MDS counts from Lustre */
    mnt_data_array[mnt_data_count].fs_info.ost_count = -1;
    mnt_data_array[mnt_data_count].fs_info.mdt_count = -1;
    if ( statfsbuf.f_type == LL_SUPER_MAGIC )
    {
        int n_ost, n_mdt;
        int ret_ost, ret_mdt;
        DIR *mount_dir;

        mount_dir = opendir( entry->mnt_dir );
        if ( mount_dir  )
        {
            /* n_ost and n_mdt are used for both input and output to ioctl */
            n_ost = 0;
            n_mdt = 1;

            ret_ost = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_ost );
            ret_mdt = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_mdt );

            if ( !(ret_ost < 0 || ret_mdt < 0) )
            {
                mnt_data_array[mnt_data_count].fs_info.ost_count = n_ost;
                mnt_data_array[mnt_data_count].fs_info.mdt_count = n_mdt;
            }
            closedir( mount_dir );
        }
    }
#endif

    /* store mount information with the job-level metadata in darshan log */
    ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
        entry->mnt_type, entry->mnt_dir);
    if(ret < 256 && strlen(tmp_mnt) <= (*space_left))
    {
        strcat(buf, tmp_mnt);
        (*space_left) -= strlen(tmp_mnt);
    }

    mnt_data_count++;
    return;
}

/* darshan_get_exe_and_mounts()
 *
 * collects command line and list of mounted file systems into a string that
 * will be stored with the job-level metadata
 */
static void darshan_get_exe_and_mounts(struct darshan_core_runtime *core,
    int argc, char **argv)
{
    FILE* tab;
    struct mntent *entry;
    char* exclude;
    char* truncate_string = "<TRUNCATED>";
    int truncate_offset;
    int space_left = DARSHAN_EXE_LEN;
    FILE *fh;
    int i, ii;
    char cmdl[DARSHAN_EXE_LEN];
    int tmp_index = 0;
    int skip = 0;
    char* env_exclusions;
    char* string;
    char* token;

    /* skip these fs types */
    static char* fs_exclusions[] = {
        "tmpfs",
        "proc",
        "sysfs",
        "devpts",
        "binfmt_misc",