header file cleanups & resolve some small todos

darshan-mpiio-log-format.h

@@ -10,7 +10,8 @@
 #include "darshan-log-format.h"
 
 /* TODO: slowest/fastest rank (f)counters */
-/* TODO: access size and extent common counters? */
+/* TODO: access size common counters */
+/* TODO: maybe use a counter to track cases in which a derived datatype is used? */
 
 #define MPIIO_COUNTERS \
     /* count of MPI independent opens */\
@@ -39,6 +40,8 @@
     X(MPIIO_HINTS) \
     /* count of MPI set view calls */\
     X(MPIIO_VIEWS) \
+    /* MPI-IO access mode of the file */\
+    X(MPIIO_MODE) \
     /* total bytes read at MPI-IO layer */\
     X(MPIIO_BYTES_READ) \
     /* total bytes written at MPI-IO layer */\
@@ -70,6 +73,16 @@
     X(MPIIO_SIZE_WRITE_AGG_10M_100M) \
     X(MPIIO_SIZE_WRITE_AGG_100M_1G) \
     X(MPIIO_SIZE_WRITE_AGG_1G_PLUS) \
+    /* the four most frequently appearing MPI access sizes */\
+    X(MPIIO_ACCESS1_ACCESS) \
+    X(MPIIO_ACCESS2_ACCESS) \
+    X(MPIIO_ACCESS3_ACCESS) \
+    X(MPIIO_ACCESS4_ACCESS) \
+    /* count of each of the most frequent MPI access sizes */\
+    X(MPIIO_ACCESS1_COUNT) \
+    X(MPIIO_ACCESS2_COUNT) \
+    X(MPIIO_ACCESS3_COUNT) \
+    X(MPIIO_ACCESS4_COUNT) \
     /* rank and number of bytes moved for fastest/slowest ranks */\
     X(MPIIO_FASTEST_RANK) \
     X(MPIIO_FASTEST_RANK_BYTES) \

darshan-posix-log-format.h

@@ -110,6 +110,7 @@
     X(POSIX_ACCESS2_COUNT) \
     X(POSIX_ACCESS3_COUNT) \
     X(POSIX_ACCESS4_COUNT) \
+    /* rank and number of bytes moved for fastest/slowest ranks */\
     X(POSIX_FASTEST_RANK) \
     X(POSIX_FASTEST_RANK_BYTES) \
     X(POSIX_SLOWEST_RANK) \
@@ -136,7 +137,9 @@
     X(POSIX_F_WRITE_TIME) \
     /* cumulative posix meta time */\
     X(POSIX_F_META_TIME) \
+    /* maximum posix read duration */\
     X(POSIX_F_MAX_READ_TIME) \
+    /* maximum posix write duration */\
     X(POSIX_F_MAX_WRITE_TIME) \
     /* total i/o and meta time consumed for fastest/slowest ranks */\
     X(POSIX_F_FASTEST_RANK_TIME) \

darshan-runtime/darshan-common.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_COMMON_H
+#define __DARSHAN_COMMON_H
+
+/* simple macros for manipulating a module's counters
+ * 
+ * NOTE: These macros assume a module's record stores integer
+ * and floating point counters in arrays, named counters and
+ * fcounters, respectively. __rec_p is the a pointer to the
+ * data record, __counter is the counter in question, and
+ * __value is the corresponding data value.
+ */
+#define DARSHAN_COUNTER_SET(__rec_p, __counter, __value) do{ \
+    (__rec_p)->counters[__counter] = __value; \
+} while(0)
+
+#define DARSHAN_COUNTER_F_SET(__rec_p, __counter, __value) do{ \
+    (__rec_p)->fcounters[__counter] = __value; \
+} while(0)
+
+#define DARSHAN_COUNTER_INC(__rec_p, __counter, __value) do{ \
+    (__rec_p)->counters[__counter] += __value; \
+} while(0)
+
+#define DARSHAN_COUNTER_F_INC(__rec_p, __counter, __value) do{ \
+    (__rec_p)->fcounters[__counter] += __value; \
+} while(0)
+
+#define DARSHAN_COUNTER_VALUE(__rec_p, __counter) \
+    ((__rec_p)->counters[__counter])
+
+#define DARSHAN_COUNTER_F_VALUE(__rec_p, __counter) \
+    ((__rec_p)->fcounters[__counter])
+
+/* set __counter equal to the max of __counter or the passed in __value */
+#define DARSHAN_COUNTER_MAX(__rec_p, __counter, __value) do{ \
+    if((__rec_p)->counters[__counter] < __value) \
+        (__rec_p)->counters[__counter] = __value; \
+} while(0)
+
+/* increment a timer counter, making sure not to account for overlap
+ * with previous operations
+ *
+ * NOTE: __tm1 is the start timestamp of the operation, __tm2 is the end
+ * timestamp of the operation, and __last is the timestamp of the end of
+ * the previous I/O operation (which we don't want to overlap with).
+ */
+#define DARSHAN_COUNTER_F_INC_NO_OVERLAP(__rec_p, __tm1, __tm2, __last, __counter) do{ \
+    if(__tm1 > __last) \
+        DARSHAN_COUNTER_F_INC(__rec_p, __counter, (__tm2 - __tm1)); \
+    else \
+        DARSHAN_COUNTER_F_INC(__rec_p, __counter, (__tm2 - __last)); \
+    if(__tm2 > __last) \
+        __last = __tm2; \
+} while(0)
+
+/* increment histogram bucket depending on the given __value
+ *
+ * NOTE: This macro can be used to build a histogram of access
+ * sizes, offsets, etc. It assumes a 10-bucket histogram, with
+ * __counter_base representing the first counter in the sequence
+ * of buckets (i.e., the smallest bucket). The size ranges of each
+ * bucket are:
+ *      * 0 - 100 bytes
+ *      * 100 - 1 KiB
+ *      * 1 KiB - 10 KiB
+ *      * 10 KiB - 100 KiB
+ *      * 100 KiB - 1 MiB
+ *      * 1 MiB - 4 MiB
+ *      * 4 MiB - 10 MiB
+ *      * 10 MiB - 100 MiB
+ *      * 100 MiB - 1 GiB
+ *      * 1 GiB+
+ */
+#define DARSHAN_BUCKET_INC(__rec_p, __counter_base, __value) do {\
+    if(__value < 101) \
+        (__rec_p)->counters[__counter_base] += 1; \
+    else if(__value < 1025) \
+        (__rec_p)->counters[__counter_base+1] += 1; \
+    else if(__value < 10241) \
+        (__rec_p)->counters[__counter_base+2] += 1; \
+    else if(__value < 102401) \
+        (__rec_p)->counters[__counter_base+3] += 1; \
+    else if(__value < 1048577) \
+        (__rec_p)->counters[__counter_base+4] += 1; \
+    else if(__value < 4194305) \
+        (__rec_p)->counters[__counter_base+5] += 1; \
+    else if(__value < 10485761) \
+        (__rec_p)->counters[__counter_base+6] += 1; \
+    else if(__value < 104857601) \
+        (__rec_p)->counters[__counter_base+7] += 1; \
+    else if(__value < 1073741825) \
+        (__rec_p)->counters[__counter_base+8] += 1; \
+    else \
+        (__rec_p)->counters[__counter_base+9] += 1; \
+} while(0)
+
+/* i/o type (read or write) */
+enum darshan_io_type
+{
+    DARSHAN_IO_READ = 1,
+    DARSHAN_IO_WRITE = 2,
+};
+
+/***********************************************
+* darshan-common functions for darshan modules *
+***********************************************/
+
+/* darshan_clean_file_path()
+ *
+ * Allocate a new string that contains a new cleaned-up version of
+ * the file path given in 'path' argument. Converts relative paths
+ * to absolute paths and filters out some potential noise in the
+ * path string.
+ */
+char* darshan_clean_file_path(const char* path);
+
+#endif /* __DARSHAN_COMMON_H */

darshan-runtime/darshan-core.h

@@ -14,6 +14,19 @@
 #include "uthash.h"
 #include "darshan-log-format.h"
 
+/* TODO: drop CP_ and use DARSHAN_ */
+/* Environment variable to override CP_JOBID */
+#define CP_JOBID_OVERRIDE "DARSHAN_JOBID"
+
+/* Environment variable to override __CP_LOG_PATH */
+#define CP_LOG_PATH_OVERRIDE "DARSHAN_LOGPATH"
+
+/* Environment variable to override __CP_LOG_PATH */
+#define CP_LOG_HINTS_OVERRIDE "DARSHAN_LOGHINTS"
+
+/* Environment variable to override __CP_MEM_ALIGNMENT */
+#define CP_MEM_ALIGNMENT_OVERRIDE "DARSHAN_MEMALIGN"
+
 #define DARSHAN_CORE_MAX_RECORDS 1024
 
 /* TODO: revisit this default size if we change memory per module */

darshan-runtime/darshan.h

@@ -13,18 +13,7 @@
 #include <mpi.h>
 
 #include "darshan-log-format.h"
-
-/* Environment variable to override CP_JOBID */
-#define CP_JOBID_OVERRIDE "DARSHAN_JOBID"
-
-/* Environment variable to override __CP_LOG_PATH */
-#define CP_LOG_PATH_OVERRIDE "DARSHAN_LOGPATH"
-
-/* Environment variable to override __CP_LOG_PATH */
-#define CP_LOG_HINTS_OVERRIDE "DARSHAN_LOGHINTS"
-
-/* Environment variable to override __CP_MEM_ALIGNMENT */
-#define CP_MEM_ALIGNMENT_OVERRIDE "DARSHAN_MEMALIGN"
+#include "darshan-common.h"
 
 /* macros for declaring wrapper functions and calling MPI routines
  * consistently regardless of whether static or dynamic linking is used
@@ -64,77 +53,6 @@
 
 #endif
 
-/* macros for manipulating a module's counter variables */
-/* NOTE: These macros assume a module's record stores integer
- * and floating point counters in arrays, named counters and
- * fcounters, respectively. __rec_p is the a pointer to the
- * data record, __counter is the counter in question, and
- * __value is the corresponding data value.
- */
-#define DARSHAN_COUNTER_SET(__rec_p, __counter, __value) do{ \
-    (__rec_p)->counters[__counter] = __value; \
-} while(0)
-
-#define DARSHAN_COUNTER_F_SET(__rec_p, __counter, __value) do{ \
-    (__rec_p)->fcounters[__counter] = __value; \
-} while(0)
-
-#define DARSHAN_COUNTER_INC(__rec_p, __counter, __value) do{ \
-    (__rec_p)->counters[__counter] += __value; \
-} while(0)
-
-#define DARSHAN_COUNTER_F_INC(__rec_p, __counter, __value) do{ \
-    (__rec_p)->fcounters[__counter] += __value; \
-} while(0)
-
-#define DARSHAN_COUNTER_F_INC_NO_OVERLAP(__rec_p, __tm1, __tm2, __last, __counter) do{ \
-    if(__tm1 > __last) \
-        DARSHAN_COUNTER_F_INC(__rec_p, __counter, (__tm2 - __tm1)); \
-    else \
-        DARSHAN_COUNTER_F_INC(__rec_p, __counter, (__tm2 - __last)); \
-    if(__tm2 > __last) \
-        __last = __tm2; \
-} while(0)
-
-#define DARSHAN_COUNTER_VALUE(__rec_p, __counter) \
-    ((__rec_p)->counters[__counter])
-
-#define DARSHAN_COUNTER_F_VALUE(__rec_p, __counter) \
-    ((__rec_p)->fcounters[__counter])
-
-#define DARSHAN_COUNTER_MAX(__rec_p, __counter, __value) do{ \
-    if((__rec_p)->counters[__counter] < __value) \
-        (__rec_p)->counters[__counter] = __value; \
-} while(0)
-
-/* NOTE: This macro is for storing histogram counters with 10
- * distinct buckets. For an example of how it is used, consult
- * the POSIX module implementation, which stores histograms of
- * access sizes for POSIX I/O functions.
- */
-#define DARSHAN_BUCKET_INC(__rec_p, __counter_base, __value) do {\
-    if(__value < 101) \
-        (__rec_p)->counters[__counter_base] += 1; \
-    else if(__value < 1025) \
-        (__rec_p)->counters[__counter_base+1] += 1; \
-    else if(__value < 10241) \
-        (__rec_p)->counters[__counter_base+2] += 1; \
-    else if(__value < 102401) \
-        (__rec_p)->counters[__counter_base+3] += 1; \
-    else if(__value < 1048577) \
-        (__rec_p)->counters[__counter_base+4] += 1; \
-    else if(__value < 4194305) \
-        (__rec_p)->counters[__counter_base+5] += 1; \
-    else if(__value < 10485761) \
-        (__rec_p)->counters[__counter_base+6] += 1; \
-    else if(__value < 104857601) \
-        (__rec_p)->counters[__counter_base+7] += 1; \
-    else if(__value < 1073741825) \
-        (__rec_p)->counters[__counter_base+8] += 1; \
-    else \
-        (__rec_p)->counters[__counter_base+9] += 1; \
-} while(0)
-
 /* module developers provide the following functions to darshan-core */
 struct darshan_module_funcs
 {
@@ -172,12 +90,6 @@ struct darshan_module_funcs
     );
 };
 
-enum darshan_io_type
-{
-    DARSHAN_IO_READ = 1,
-    DARSHAN_IO_WRITE = 2,
-};
-
 /* paths that darshan will not trace */
 extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
 
@@ -185,15 +97,44 @@ extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
 * darshan-core functions exported to darshan modules *
 *****************************************************/
 
+/* darshan_core_register_module()
+ *
+ * Register module identifier 'mod_id' with the darshan-core runtime
+ * environment, allowing the module to store I/O characterization data.
+ * 'funcs' is a pointer to a structure containing each of the function
+ * pointers required by darshan-core to shut down the module. The function
+ * returns the following integers passed in as pointers: 'my_rank' is the
+ * MPI rank of the calling process, 'mod_mem_limit' is the maximum amount
+ * of memory the module may use, and 'sys_mem_alignment' is the configured
+ * memory alignment value Darshan was configured with.
+ */
 void darshan_core_register_module(
     darshan_module_id mod_id,
     struct darshan_module_funcs *funcs,
+    int *my_rank,
     int *mod_mem_limit,
     int *sys_mem_alignment);
 
+/* darshan_core_unregister_module()
+ * 
+ * Unregisters module identifier 'mod_id' with the darshan-core runtime,
+ * removing the given module from the resulting I/O characterization log.
+ */
 void darshan_core_unregister_module(
     darshan_module_id mod_id);
 
+/* darshan_core_register_record()
+ *
+ * Register the Darshan record given by 'name' with the darshan-core
+ * runtime, allowing it to be properly tracked and (potentially)
+ * correlated with records from other modules. 'len' is the size of
+ * the name pointer (string length for string names), 'printable_flag'
+ * indicates whether the name is a string, and 'mod_id' is the identifier
+ * of the calling module. 'rec_id' is an output pointer storing the
+ * correspoing Darshan record identifier and 'file_alignment' is an
+ * output pointer storing the file system alignment value for the given
+ * record.
+ */
 void darshan_core_register_record(
     void *name,
     int len,
@@ -202,16 +143,22 @@ void darshan_core_register_record(
     darshan_record_id *rec_id,
     int *file_alignment);
 
+/* darshan_core_unregister_record()
+ *
+ * Unregister record identifier 'rec_id' in the darshan-core runtime.
+ * This unregister is only in the context of module identifier 'mod_id',
+ * meaning that if the file record has other module's associated with
+ * it, then the record won't be completely removed.
+ */
 void darshan_core_unregister_record(
     darshan_record_id rec_id,
     darshan_module_id mod_id);
 
+/* darshan_core_wtime()
+ *
+ * Returns the elapsed time relative to (roughly) the start of
+ * the application.
+ */
 double darshan_core_wtime(void);
 
-/***********************************************
-* darshan-common functions for darshan modules *
-***********************************************/
-
-char* darshan_clean_file_path(const char* path);
-
 #endif /* __DARSHAN_H */

darshan-runtime/lib/darshan-common.c

@@ -13,10 +13,6 @@
 
 #include "darshan.h"
 
-/* Allocate a new string that contains a cleaned-up version of the path
- * passed in as an argument.  Converts relative paths to absolute paths and
- * filters out some potential noise in the path string.
- */
 char* darshan_clean_file_path(const char* path)
 {
     char* newpath = NULL;

darshan-runtime/lib/darshan-core.c

@@ -1513,6 +1513,7 @@ static void darshan_core_cleanup(struct darshan_core_runtime* core)
 void darshan_core_register_module(
     darshan_module_id mod_id,
     struct darshan_module_funcs *funcs,
+    int *my_rank,
     int *mod_mem_limit,
     int *sys_mem_alignment)
 {
@@ -1549,6 +1550,9 @@ void darshan_core_register_module(
     /* register module with darshan */
     darshan_core->mod_array[mod_id] = mod;
 
+    /* get the calling process's rank */
+    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, my_rank);
+
     /* TODO: something smarter than just 2 MiB per module */
     *mod_mem_limit = 2 * 1024 * 1024;
 

darshan-runtime/lib/darshan-mpiio.c

@@ -129,8 +129,6 @@ static void mpiio_record_reduction_op(void* infile_v, void* inoutfile_v,
 static void mpiio_get_output_data(void **buffer, int *size);
 static void mpiio_shutdown(void);
 
-/* TODO: maybe use a counter to track cases in which a derived datatype is used? */
-
 #define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
 #define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
 
@@ -224,6 +222,7 @@ int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_F
         if(file)
         {
             file->file_record->rank = my_rank;
+            DARSHAN_COUNTER_SET(file->file_record, MPIIO_MODE, amode);
             DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
             if(comm_size == 1)
             {
@@ -839,6 +838,7 @@ static void mpiio_runtime_initialize()
     darshan_core_register_module(
         DARSHAN_MPIIO_MOD,
         &mpiio_mod_fns,
+        &my_rank,
         &mem_limit,
         NULL);
 
@@ -871,9 +871,6 @@ static void mpiio_runtime_initialize()
     memset(mpiio_runtime->file_record_array, 0, mpiio_runtime->file_array_size *
            sizeof(struct darshan_mpiio_file));
 
-    /* TODO: can we move this out of here? perhaps register_module returns rank? */
-    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
-
     return;
 }
 

darshan-runtime/lib/darshan-null.c

@@ -28,18 +28,12 @@
  * that may be reused and expanded on by developers adding new instrumentation modules.
  */
 
-/* TODO: this probably shouldn't be here -- LD_PRELOADing POSIX wrappers will cause MPI linker dependency */
-#ifdef DARSHAN_PRELOAD
-extern double (*__real_PMPI_Comm_rank)(MPI_Comm comm, int *rank);
-#endif
-
 /* The DARSHAN_FORWARD_DECL macro (defined in darshan.h) is used to provide forward
  * declarations for wrapped funcions, regardless if Darshan is used with statically
  * or dynamically linked executables.
  */
 DARSHAN_FORWARD_DECL(foo, int, (const char *name, int arg1, int arg2));
 
-
 /* The null_record_runtime structure maintains necessary runtime metadata
  * for a "NULL" module data record (darshan_null_record structure, defined
  * in darshan-null-log-format.h). This metadata assists with the instrumenting
@@ -224,6 +218,7 @@ static void null_runtime_initialize()
     darshan_core_register_module(
         DARSHAN_NULL_MOD,   /* Darshan module identifier, defined in darshan-log-format.h */
         &null_mod_fns,
+        &my_rank,
         &mem_limit,
         NULL);
 
@@ -262,9 +257,6 @@ static void null_runtime_initialize()
     memset(null_runtime->record_array, 0, null_runtime->rec_array_size *
            sizeof(struct darshan_null_record));
 
-    /* TODO: we should move this out of here.. perhaps register_module returns rank? */
-    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
-
     return;
 }
 

darshan-runtime/lib/darshan-posix.c

@@ -39,11 +39,6 @@ typedef int64_t off64_t;
 #define aiocb64 aiocb
 #endif
 
-/* TODO: this probably shouldn't be here long term -- MPI symbols mess up LD_PRELOAD */
-#ifdef DARSHAN_PRELOAD
-extern double (*__real_PMPI_Comm_rank)(MPI_Comm comm, int *rank);
-#endif
-
 /* TODO: more libc, fgetc, etc etc etc. */
 
 DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
@@ -1517,6 +1512,7 @@ static void posix_runtime_initialize()
     darshan_core_register_module(
         DARSHAN_POSIX_MOD,
         &posix_mod_fns,
+        &my_rank,
         &mem_limit,
         &darshan_mem_alignment);
 
@@ -1550,9 +1546,6 @@ static void posix_runtime_initialize()
     memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
            sizeof(struct darshan_posix_file));
 
-    /* TODO: can we move this out of here? perhaps register_module returns rank? */
-    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
-
     return;
 }