merge from dev-modular

darshan-hdf5-log-format.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_HDF5_LOG_FORMAT_H
+#define __DARSHAN_HDF5_LOG_FORMAT_H
+
+#include "darshan-log-format.h"
+
+#define HDF5_COUNTERS \
+    /* count of HDF5 opens */\
+    X(HDF5_OPENS) \
+    /* end of counters */\
+    X(HDF5_NUM_INDICES)
+
+#define HDF5_F_COUNTERS \
+    /* timestamp of first open */\
+    X(HDF5_F_OPEN_TIMESTAMP) \
+    /* timestamp of last close */\
+    X(HDF5_F_CLOSE_TIMESTAMP) \
+    /* end of counters*/\
+    X(HDF5_F_NUM_INDICES)
+
+#define X(a) a,
+/* integer statistics for HDF5 file records */
+enum darshan_hdf5_indices
+{
+    HDF5_COUNTERS
+};
+
+/* floating point statistics for HDF5 file records */
+enum darshan_hdf5_f_indices
+{
+    HDF5_F_COUNTERS
+};
+#undef X
+
+/* file record structure for HDF5 files. a record is created and stored for
+ * every HDF5 file opened by the original application. For the HDF5 module,
+ * the record includes:
+ *      - a corresponding record identifier (created by hashing the file path)
+ *      - the rank of the process which opened the file (-1 for shared files)
+ *      - integer file I/O statistics (open, read/write counts, etc)
+ *      - floating point I/O statistics (timestamps, cumulative timers, etc.)
+ */
+struct darshan_hdf5_file
+{
+    darshan_record_id f_id;
+    int64_t rank;
+    int64_t counters[HDF5_NUM_INDICES];
+    double fcounters[HDF5_F_NUM_INDICES];
+};
+
+#endif /* __DARSHAN_HDF5_LOG_FORMAT_H */

darshan-log-format.h

@@ -31,32 +31,53 @@
 /* max length of exe string within job record (not counting '\0') */
 #define DARSHAN_EXE_LEN (DARSHAN_JOB_RECORD_SIZE - sizeof(struct darshan_job) - 1)
 
-typedef uint64_t darshan_record_id;
+#define DARSHAN_MAX_MODS 16
+
+/* TODO: do we want the logutil defs here ? */
+/* X-macro for keeping module ordering consistent */
+/* NOTE: first val used to define module enum values, 
+ * second val used to define module name strings, and
+ * third val is used to provide the name of a 
+ * corresponding logutils structure for parsing module
+ * data out of the log file (only used in darshan-util,
+ * just pass NULL (no quotes) if no log parsing
+ * functions are required).
+ */
+#define DARSHAN_MODULE_IDS \
+    X(DARSHAN_NULL_MOD, "NULL", NULL) \
+    X(DARSHAN_POSIX_MOD, "POSIX", posix_logutils) \
+    X(DARSHAN_MPIIO_MOD, "MPI-IO", mpiio_logutils) \
+    X(DARSHAN_HDF5_MOD, "HDF5", hdf5_logutils) \
+    X(DARSHAN_PNETCDF_MOD, "PNETCDF", pnetcdf_logutils) \
+    X(DARSHAN_BGQ_MODE, "BG/Q", bgq_logutils)
 
 /* unique identifiers to distinguish between available darshan modules */
 /* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
  *        - order of ids control module shutdown order (and consequently, order in log file)
  */
-#define DARSHAN_MAX_MODS 16
+#define X(a, b, c) a,
 typedef enum
 {
-    DARSHAN_NULL_MOD = 0,
-    DARSHAN_POSIX_MOD,
-    DARSHAN_MPIIO_MOD,
-    DARSHAN_HDF5_MOD,
-    DARSHAN_PNETCDF_MOD,
-    DARSHAN_BGQ_MOD,
+    DARSHAN_MODULE_IDS
 } darshan_module_id;
+#undef X
 
+/* module name strings */
+#define X(a, b, c) b,
 static char * const darshan_module_names[] =
 {
-    "NULL",
-    "POSIX",
-    "MPI-IO",
-    "HDF5",
-    "PNETCDF",
-    "BG/Q",
+    DARSHAN_MODULE_IDS
 };
+#undef X
+
+/* compression method used on darshan log file */
+enum darshan_comp_type
+{
+    DARSHAN_ZLIB_COMP,
+    DARSHAN_BZIP2_COMP,
+};
+
+typedef uint64_t darshan_record_id;
 
 /* the darshan_log_map structure is used to indicate the location of
  * specific module data in a Darshan log. Note that 'off' and 'len' are
@@ -78,6 +99,7 @@ struct darshan_header
 {
     char version_string[8];
     int64_t magic_nr;
+    unsigned char comp_type;
     struct darshan_log_map rec_map;
     struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
 };

darshan-mpiio-log-format.h

@@ -9,29 +9,147 @@
 
 #include "darshan-log-format.h"
 
+/* TODO: maybe use a counter to track cases in which a derived datatype is used? */
+
+#define MPIIO_COUNTERS \
+    /* count of MPI independent opens */\
+    X(MPIIO_INDEP_OPENS) \
+    /* count of MPI collective opens */\
+    X(MPIIO_COLL_OPENS) \
+    /* count of MPI independent reads */\
+    X(MPIIO_INDEP_READS) \
+    /* count of MPI independent writes */\
+    X(MPIIO_INDEP_WRITES) \
+    /* count of MPI collective reads */\
+    X(MPIIO_COLL_READS) \
+    /* count of MPI collective writes */\
+    X(MPIIO_COLL_WRITES) \
+    /* count of MPI split collective reads */\
+    X(MPIIO_SPLIT_READS) \
+    /* count of MPI split collective writes */\
+    X(MPIIO_SPLIT_WRITES) \
+    /* count of MPI nonblocking reads */\
+    X(MPIIO_NB_READS) \
+    /* count of MPI nonblocking writes */\
+    X(MPIIO_NB_WRITES) \
+    /* count of MPI file syncs */\
+    X(MPIIO_SYNCS) \
+    /* count of MPI hints used */\
+    X(MPIIO_HINTS) \
+    /* count of MPI set view calls */\
+    X(MPIIO_VIEWS) \
+    /* MPI-IO access mode of the file */\
+    X(MPIIO_MODE) \
+    /* total bytes read at MPI-IO layer */\
+    X(MPIIO_BYTES_READ) \
+    /* total bytes written at MPI-IO layer */\
+    X(MPIIO_BYTES_WRITTEN) \
+    /* number of times switching between MPI read and write */\
+    X(MPIIO_RW_SWITCHES) \
+    /* sizes of the maximum read/write operations */\
+    X(MPIIO_MAX_READ_TIME_SIZE) \
+    X(MPIIO_MAX_WRITE_TIME_SIZE) \
+    /* buckets for MPI read size ranges */\
+    X(MPIIO_SIZE_READ_AGG_0_100) \
+    X(MPIIO_SIZE_READ_AGG_100_1K) \
+    X(MPIIO_SIZE_READ_AGG_1K_10K) \
+    X(MPIIO_SIZE_READ_AGG_10K_100K) \
+    X(MPIIO_SIZE_READ_AGG_100K_1M) \
+    X(MPIIO_SIZE_READ_AGG_1M_4M) \
+    X(MPIIO_SIZE_READ_AGG_4M_10M) \
+    X(MPIIO_SIZE_READ_AGG_10M_100M) \
+    X(MPIIO_SIZE_READ_AGG_100M_1G) \
+    X(MPIIO_SIZE_READ_AGG_1G_PLUS) \
+    /* buckets for MPI write size ranges */\
+    X(MPIIO_SIZE_WRITE_AGG_0_100) \
+    X(MPIIO_SIZE_WRITE_AGG_100_1K) \
+    X(MPIIO_SIZE_WRITE_AGG_1K_10K) \
+    X(MPIIO_SIZE_WRITE_AGG_10K_100K) \
+    X(MPIIO_SIZE_WRITE_AGG_100K_1M) \
+    X(MPIIO_SIZE_WRITE_AGG_1M_4M) \
+    X(MPIIO_SIZE_WRITE_AGG_4M_10M) \
+    X(MPIIO_SIZE_WRITE_AGG_10M_100M) \
+    X(MPIIO_SIZE_WRITE_AGG_100M_1G) \
+    X(MPIIO_SIZE_WRITE_AGG_1G_PLUS) \
+    /* the four most frequently appearing MPI access sizes */\
+    X(MPIIO_ACCESS1_ACCESS) \
+    X(MPIIO_ACCESS2_ACCESS) \
+    X(MPIIO_ACCESS3_ACCESS) \
+    X(MPIIO_ACCESS4_ACCESS) \
+    /* count of each of the most frequent MPI access sizes */\
+    X(MPIIO_ACCESS1_COUNT) \
+    X(MPIIO_ACCESS2_COUNT) \
+    X(MPIIO_ACCESS3_COUNT) \
+    X(MPIIO_ACCESS4_COUNT) \
+    /* rank and number of bytes moved for fastest/slowest ranks */\
+    X(MPIIO_FASTEST_RANK) \
+    X(MPIIO_FASTEST_RANK_BYTES) \
+    X(MPIIO_SLOWEST_RANK) \
+    X(MPIIO_SLOWEST_RANK_BYTES) \
+    /* end of counters */\
+    X(MPIIO_NUM_INDICES)
+
+#define MPIIO_F_COUNTERS \
+    /* timestamp of first open */\
+    X(MPIIO_F_OPEN_TIMESTAMP) \
+    /* timestamp of first read */\
+    X(MPIIO_F_READ_START_TIMESTAMP) \
+    /* timestamp of first write */\
+    X(MPIIO_F_WRITE_START_TIMESTAMP) \
+    /* timestamp of last read */\
+    X(MPIIO_F_READ_END_TIMESTAMP) \
+    /* timestamp of last write */\
+    X(MPIIO_F_WRITE_END_TIMESTAMP) \
+    /* timestamp of last close */\
+    X(MPIIO_F_CLOSE_TIMESTAMP) \
+    /* cumulative MPI-IO read time */\
+    X(MPIIO_F_READ_TIME) \
+    /* cumulative MPI-IO write time */\
+    X(MPIIO_F_WRITE_TIME) \
+    /* cumulative MPI-IO meta time */\
+    X(MPIIO_F_META_TIME) \
+    /* maximum MPI-IO read duration */\
+    X(MPIIO_F_MAX_READ_TIME) \
+    /* maximum MPI-IO write duration */\
+    X(MPIIO_F_MAX_WRITE_TIME) \
+    /* total i/o and meta time for fastest/slowest ranks */\
+    X(MPIIO_F_FASTEST_RANK_TIME) \
+    X(MPIIO_F_SLOWEST_RANK_TIME) \
+    /* variance of total i/o time and bytes moved across all ranks */\
+    /* NOTE: for shared records only */\
+    X(MPIIO_F_VARIANCE_RANK_TIME) \
+    X(MPIIO_F_VARIANCE_RANK_BYTES) \
+    /* end of counters*/\
+    X(MPIIO_F_NUM_INDICES)
+
+#define X(a) a,
+/* integer statistics for MPI-IO file records */
 enum darshan_mpiio_indices
 {
-    DARSHAN_MPIIO_INDEP_OPENS,   /* independent opens */
-    DARSHAN_MPIIO_COLL_OPENS,    /* collective opens */
-    DARSHAN_MPIIO_HINTS,    /* how many times hints were used */
-
-    DARSHAN_MPIIO_NUM_INDICES,
+    MPIIO_COUNTERS
 };
 
+/* floating point statistics for MPI-IO file records */
 enum darshan_mpiio_f_indices
 {
-    DARSHAN_MPIIO_F_META_TIME,      /* cumulative metadata time */
-    DARSHAN_MPIIO_F_OPEN_TIMESTAMP, /* first open timestamp */
-
-    DARSHAN_MPIIO_F_NUM_INDICES,
+    MPIIO_F_COUNTERS
 };
+#undef X
 
+/* file record structure for MPI-IO files. a record is created and stored for
+ * every MPI-IO file opened by the original application. For the MPI-IO module,
+ * the record includes:
+ *      - a corresponding record identifier (created by hashing the file path)
+ *      - the rank of the process which opened the file (-1 for shared files)
+ *      - integer file I/O statistics (open, read/write counts, etc)
+ *      - floating point I/O statistics (timestamps, cumulative timers, etc.)
+ */
 struct darshan_mpiio_file
 {
     darshan_record_id f_id;
     int64_t rank;
-    int64_t counters[DARSHAN_MPIIO_NUM_INDICES];
-    double fcounters[DARSHAN_MPIIO_F_NUM_INDICES];
+    int64_t counters[MPIIO_NUM_INDICES];
+    double fcounters[MPIIO_F_NUM_INDICES];
 };
 
 #endif /* __DARSHAN_MPIIO_LOG_FORMAT_H */

darshan-pnetcdf-log-format.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_PNETCDF_LOG_FORMAT_H
+#define __DARSHAN_PNETCDF_LOG_FORMAT_H
+
+#include "darshan-log-format.h"
+
+#define PNETCDF_COUNTERS \
+    /* count of PNETCDF independent opens */\
+    X(PNETCDF_INDEP_OPENS) \
+    /* count of PNETCDF collective opens */\
+    X(PNETCDF_COLL_OPENS) \
+    /* end of counters */\
+    X(PNETCDF_NUM_INDICES)
+
+#define PNETCDF_F_COUNTERS \
+    /* timestamp of first open */\
+    X(PNETCDF_F_OPEN_TIMESTAMP) \
+    /* timestamp of last close */\
+    X(PNETCDF_F_CLOSE_TIMESTAMP) \
+    /* end of counters*/\
+    X(PNETCDF_F_NUM_INDICES)
+
+#define X(a) a,
+/* integer statistics for PNETCDF file records */
+enum darshan_pnetcdf_indices
+{
+    PNETCDF_COUNTERS
+};
+
+/* floating point statistics for PNETCDF file records */
+enum darshan_pnetcdf_f_indices
+{
+    PNETCDF_F_COUNTERS
+};
+#undef X
+
+/* file record structure for PNETCDF files. a record is created and stored for
+ * every PNETCDF file opened by the original application. For the PNETCDF module,
+ * the record includes:
+ *      - a corresponding record identifier (created by hashing the file path)
+ *      - the rank of the process which opened the file (-1 for shared files)
+ *      - integer file I/O statistics (open, read/write counts, etc)
+ *      - floating point I/O statistics (timestamps, cumulative timers, etc.)
+ */
+struct darshan_pnetcdf_file
+{
+    darshan_record_id f_id;
+    int64_t rank;
+    int64_t counters[PNETCDF_NUM_INDICES];
+    double fcounters[PNETCDF_F_NUM_INDICES];
+};
+
+#endif /* __DARSHAN_PNETCDF_LOG_FORMAT_H */

darshan-posix-log-format.h

@@ -8,108 +8,157 @@
 
 #include "darshan-log-format.h"
 
+#define POSIX_COUNTERS \
+    /* count of posix opens */\
+    X(POSIX_OPENS) \
+    /* count of posix reads */\
+    X(POSIX_READS) \
+    /* count of posix writes */\
+    X(POSIX_WRITES) \
+    /* count of posix seeks */\
+    X(POSIX_SEEKS) \
+    /* count of posix stat/lstat/fstats */\
+    X(POSIX_STATS) \
+    /* count of posix mmaps */\
+    X(POSIX_MMAPS) \
+    /* count of posix fopens */\
+    X(POSIX_FOPENS) \
+    /* count of posix freads */\
+    X(POSIX_FREADS) \
+    /* count of posix fwrites */\
+    X(POSIX_FWRITES) \
+    /* count of posix fseeks */\
+    X(POSIX_FSEEKS) \
+    /* count of posix fsyncs */\
+    X(POSIX_FSYNCS) \
+    /* count of posix fdatasyncs */\
+    X(POSIX_FDSYNCS) \
+    /* mode of file */\
+    X(POSIX_MODE) \
+    /* total bytes read */\
+    X(POSIX_BYTES_READ) \
+    /* total bytes written */\
+    X(POSIX_BYTES_WRITTEN) \
+    /* highest offset byte read */\
+    X(POSIX_MAX_BYTE_READ) \
+    /* highest offset byte written */\
+    X(POSIX_MAX_BYTE_WRITTEN) \
+    /* count of consecutive reads */\
+    X(POSIX_CONSEC_READS) \
+    /* count of consecutive writes */\
+    X(POSIX_CONSEC_WRITES) \
+    /* count of sequential reads */\
+    X(POSIX_SEQ_READS) \
+    /* count of sequential writes */\
+    X(POSIX_SEQ_WRITES) \
+    /* number of times switched between read and write */\
+    X(POSIX_RW_SWITCHES) \
+    /* count of accesses not mem aligned */\
+    X(POSIX_MEM_NOT_ALIGNED) \
+    /* mem alignment in bytes */\
+    X(POSIX_MEM_ALIGNMENT) \
+    /* count of accesses not file aligned */\
+    X(POSIX_FILE_NOT_ALIGNED) \
+    /* file alignment in bytes */\
+    X(POSIX_FILE_ALIGNMENT) \
+    X(POSIX_MAX_READ_TIME_SIZE) \
+    X(POSIX_MAX_WRITE_TIME_SIZE) \
+    /* buckets for POSIX read size ranges */\
+    X(POSIX_SIZE_READ_0_100) \
+    X(POSIX_SIZE_READ_100_1K) \
+    X(POSIX_SIZE_READ_1K_10K) \
+    X(POSIX_SIZE_READ_10K_100K) \
+    X(POSIX_SIZE_READ_100K_1M) \
+    X(POSIX_SIZE_READ_1M_4M) \
+    X(POSIX_SIZE_READ_4M_10M) \
+    X(POSIX_SIZE_READ_10M_100M) \
+    X(POSIX_SIZE_READ_100M_1G) \
+    X(POSIX_SIZE_READ_1G_PLUS) \
+    /* buckets for POSIX write size ranges */\
+    X(POSIX_SIZE_WRITE_0_100) \
+    X(POSIX_SIZE_WRITE_100_1K) \
+    X(POSIX_SIZE_WRITE_1K_10K) \
+    X(POSIX_SIZE_WRITE_10K_100K) \
+    X(POSIX_SIZE_WRITE_100K_1M) \
+    X(POSIX_SIZE_WRITE_1M_4M) \
+    X(POSIX_SIZE_WRITE_4M_10M) \
+    X(POSIX_SIZE_WRITE_10M_100M) \
+    X(POSIX_SIZE_WRITE_100M_1G) \
+    X(POSIX_SIZE_WRITE_1G_PLUS) \
+    /* the four most frequently appearing strides */\
+    X(POSIX_STRIDE1_STRIDE) \
+    X(POSIX_STRIDE2_STRIDE) \
+    X(POSIX_STRIDE3_STRIDE) \
+    X(POSIX_STRIDE4_STRIDE) \
+    /* count of each of the most frequent strides */\
+    X(POSIX_STRIDE1_COUNT) \
+    X(POSIX_STRIDE2_COUNT) \
+    X(POSIX_STRIDE3_COUNT) \
+    X(POSIX_STRIDE4_COUNT) \
+    /* the four most frequently appearing access sizes */\
+    X(POSIX_ACCESS1_ACCESS) \
+    X(POSIX_ACCESS2_ACCESS) \
+    X(POSIX_ACCESS3_ACCESS) \
+    X(POSIX_ACCESS4_ACCESS) \
+    /* count of each of the most frequent access sizes */\
+    X(POSIX_ACCESS1_COUNT) \
+    X(POSIX_ACCESS2_COUNT) \
+    X(POSIX_ACCESS3_COUNT) \
+    X(POSIX_ACCESS4_COUNT) \
+    /* rank and number of bytes moved for fastest/slowest ranks */\
+    X(POSIX_FASTEST_RANK) \
+    X(POSIX_FASTEST_RANK_BYTES) \
+    X(POSIX_SLOWEST_RANK) \
+    X(POSIX_SLOWEST_RANK_BYTES) \
+    /* end of counters */\
+    X(POSIX_NUM_INDICES)
+
+#define POSIX_F_COUNTERS \
+    /* timestamp of first open */\
+    X(POSIX_F_OPEN_TIMESTAMP) \
+    /* timestamp of first read */\
+    X(POSIX_F_READ_START_TIMESTAMP) \
+    /* timestamp of first write */\
+    X(POSIX_F_WRITE_START_TIMESTAMP) \
+    /* timestamp of last read */\
+    X(POSIX_F_READ_END_TIMESTAMP) \
+    /* timestamp of last write */\
+    X(POSIX_F_WRITE_END_TIMESTAMP) \
+    /* timestamp of last close */\
+    X(POSIX_F_CLOSE_TIMESTAMP) \
+    /* cumulative posix read time */\
+    X(POSIX_F_READ_TIME) \
+    /* cumulative posix write time */\
+    X(POSIX_F_WRITE_TIME) \
+    /* cumulative posix meta time */\
+    X(POSIX_F_META_TIME) \
+    /* maximum posix read duration */\
+    X(POSIX_F_MAX_READ_TIME) \
+    /* maximum posix write duration */\
+    X(POSIX_F_MAX_WRITE_TIME) \
+    /* total i/o and meta time consumed for fastest/slowest ranks */\
+    X(POSIX_F_FASTEST_RANK_TIME) \
+    X(POSIX_F_SLOWEST_RANK_TIME) \
+    /* variance of total i/o time and bytes moved across all ranks */\
+    /* NOTE: for shared records only */\
+    X(POSIX_F_VARIANCE_RANK_TIME) \
+    X(POSIX_F_VARIANCE_RANK_BYTES) \
+    /* end of counters */\
+    X(POSIX_F_NUM_INDICES)
+
+#define X(a) a,
 /* integer statistics for POSIX file records */
 enum darshan_posix_indices
 {
-    POSIX_OPENS,              /* count of posix opens */
-    POSIX_READS,              /* count of posix reads */
-    POSIX_WRITES,             /* count of posix writes */
-    POSIX_SEEKS,              /* count of posix seeks */
-    POSIX_STATS,              /* count of posix stat/lstat/fstats */
-    POSIX_MMAPS,              /* count of posix mmaps */
-    POSIX_FOPENS,             /* count of posix fopens */
-    POSIX_FREADS,             /* count of posix freads */
-    POSIX_FWRITES,            /* count of posix fwrites */
-    POSIX_FSEEKS,             /* count of posix fseeks */
-    POSIX_FSYNCS,             /* count of posix fsyncs */
-    POSIX_FDSYNCS,            /* count of posix fdatasyncs */
-    POSIX_MODE,               /* mode of file */
-    POSIX_BYTES_READ,         /* total bytes read */
-    POSIX_BYTES_WRITTEN,      /* total bytes written */
-    POSIX_MAX_BYTE_READ,      /* highest offset byte read */
-    POSIX_MAX_BYTE_WRITTEN,   /* highest offset byte written */
-    POSIX_CONSEC_READS,       /* count of consecutive reads */
-    POSIX_CONSEC_WRITES,      /* count of consecutive writes */ 
-    POSIX_SEQ_READS,          /* count of sequential reads */
-    POSIX_SEQ_WRITES,         /* count of sequential writes */
-    POSIX_RW_SWITCHES,        /* number of times switched between read and write */
-    POSIX_MEM_NOT_ALIGNED,    /* count of accesses not mem aligned */
-    POSIX_MEM_ALIGNMENT,      /* mem alignment in bytes */
-    POSIX_FILE_NOT_ALIGNED,   /* count of accesses not file aligned */
-    POSIX_FILE_ALIGNMENT,     /* file alignment in bytes */
-    POSIX_MAX_READ_TIME_SIZE,
-    POSIX_MAX_WRITE_TIME_SIZE,
-    /* buckets */
-    POSIX_SIZE_READ_0_100,    /* count of posix read size ranges */
-    POSIX_SIZE_READ_100_1K,
-    POSIX_SIZE_READ_1K_10K,
-    POSIX_SIZE_READ_10K_100K,
-    POSIX_SIZE_READ_100K_1M,
-    POSIX_SIZE_READ_1M_4M,
-    POSIX_SIZE_READ_4M_10M,
-    POSIX_SIZE_READ_10M_100M,
-    POSIX_SIZE_READ_100M_1G,
-    POSIX_SIZE_READ_1G_PLUS,
-    /* buckets */
-    POSIX_SIZE_WRITE_0_100,   /* count of posix write size ranges */
-    POSIX_SIZE_WRITE_100_1K,
-    POSIX_SIZE_WRITE_1K_10K,
-    POSIX_SIZE_WRITE_10K_100K,
-    POSIX_SIZE_WRITE_100K_1M,
-    POSIX_SIZE_WRITE_1M_4M,
-    POSIX_SIZE_WRITE_4M_10M,
-    POSIX_SIZE_WRITE_10M_100M,
-    POSIX_SIZE_WRITE_100M_1G,
-    POSIX_SIZE_WRITE_1G_PLUS,
-    /* stride/access counters */
-    POSIX_STRIDE1_STRIDE,     /* the four most frequently appearing strides */
-    POSIX_STRIDE2_STRIDE,
-    POSIX_STRIDE3_STRIDE,
-    POSIX_STRIDE4_STRIDE,
-    POSIX_STRIDE1_COUNT,      /* count of each of the most frequent strides */
-    POSIX_STRIDE2_COUNT,
-    POSIX_STRIDE3_COUNT,
-    POSIX_STRIDE4_COUNT,
-    POSIX_ACCESS1_ACCESS,     /* the four most frequently appearing access sizes */
-    POSIX_ACCESS2_ACCESS,
-    POSIX_ACCESS3_ACCESS,
-    POSIX_ACCESS4_ACCESS,
-    POSIX_ACCESS1_COUNT,      /* count of each of the most frequent access sizes */
-    POSIX_ACCESS2_COUNT,
-    POSIX_ACCESS3_COUNT,
-    POSIX_ACCESS4_COUNT,
-    POSIX_FASTEST_RANK,
-    POSIX_FASTEST_RANK_BYTES,
-    POSIX_SLOWEST_RANK,
-    POSIX_SLOWEST_RANK_BYTES,
-
-    POSIX_NUM_INDICES,
+    POSIX_COUNTERS
 };
 
 /* floating point statistics for POSIX file records */
 enum darshan_posix_f_indices
 {
-    POSIX_F_OPEN_TIMESTAMP = 0,    /* timestamp of first open */
-    POSIX_F_READ_START_TIMESTAMP,  /* timestamp of first read */
-    POSIX_F_WRITE_START_TIMESTAMP, /* timestamp of first write */
-    POSIX_F_READ_END_TIMESTAMP,    /* timestamp of last read */
-    POSIX_F_WRITE_END_TIMESTAMP,   /* timestamp of last write */
-    POSIX_F_CLOSE_TIMESTAMP,       /* timestamp of last close */
-    POSIX_F_READ_TIME,             /* cumulative posix read time */
-    POSIX_F_WRITE_TIME,            /* cumulative posix write time */
-    POSIX_F_META_TIME,             /* cumulative posix meta time */
-    POSIX_F_MAX_READ_TIME,
-    POSIX_F_MAX_WRITE_TIME,
-    /* Total I/O and meta time consumed by fastest and slowest ranks */ 
-    POSIX_F_FASTEST_RANK_TIME,
-    POSIX_F_SLOWEST_RANK_TIME,
-#if 0
-    F_VARIANCE_RANK_TIME,
-    F_VARIANCE_RANK_BYTES,
-#endif
-
-    POSIX_F_NUM_INDICES,
+    POSIX_F_COUNTERS
 };
+#undef X
 
 /* file record structure for POSIX files. a record is created and stored for
  * every POSIX file opened by the original application. For the POSIX module,
@@ -127,4 +176,11 @@ struct darshan_posix_file
     double fcounters[POSIX_F_NUM_INDICES];
 };
 
+/* This macro can be used to identify files that have been opened using
+ * pnetcdf, hdf5, or mpi-io, but were never opened at the posix level.  As a
+ * result the record will not necessarily have all of the expected fields
+ * populated.
+ */
+#define POSIX_FILE_PARTIAL(__file)((((__file)->counters[POSIX_OPENS] || (__file)->counters[POSIX_FOPENS] || (__file)->counters[POSIX_STATS]) ? 0 : 1))
+
 #endif /* __DARSHAN_POSIX_LOG_FORMAT_H */

darshan-runtime/Makefile.in

-all: lib/libdarshan.a lib/darshan-null.o
+all: lib/libdarshan.a lib/libdarshan-stubs.a lib/darshan-null.o
 
 DESTDIR =
 srcdir = @srcdir@
@@ -25,7 +25,7 @@ endif
 VPATH = $(srcdir)
 
 CFLAGS = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I ../ -I $(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE 
-#
+
 CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE -shared -fpic -DPIC -DDARSHAN_PRELOAD
 
 LIBS = -lz @LIBBZ2@
@@ -78,6 +78,23 @@ lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/.
 lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
+lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+