add LD_PRELOAD functionality

darshan-runtime/Makefile.in

@@ -33,10 +33,16 @@ LIBS = -lz @LIBBZ2@
 lib::
 	@mkdir -p $@
 
-lib/darshan-core.o: lib/darshan-core.c darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core-init-finalize.o: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-core.po: lib/darshan-core.c darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+lib/darshan-core.o: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
 lib/darshan-common.o: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
@@ -72,10 +78,10 @@ lib/lookup8.po: lib/lookup8.c
 #%.i: %.c
 #	$(CC) -E $(CFLAGS) -c $< -o $@
 
-lib/libdarshan.a: lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-core.o lib/darshan-common.o lib/lookup3.o lib/lookup8.o
+lib/libdarshan.a: lib/darshan-core-init-finalize.o lib/darshan-core.o lib/darshan-common.o lib/darshan-posix.o lib/darshan-mpiio.o lib/lookup3.o lib/lookup8.o
 	ar rcs $@ $^
 
-lib/libdarshan.so: lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/darshan-mpiio.po lib/lookup3.po lib/lookup8.po
+lib/libdarshan.so: lib/darshan-core-init-finalize.po lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/darshan-mpiio.po lib/lookup3.po lib/lookup8.po
 	$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ -lpthread -lrt -lz -ldl
 
 install:: all

darshan-runtime/configure

@@ -687,7 +687,6 @@ with_zlib
 enable_cuserid
 enable_ld_preload
 enable_group_readable_logs
-enable_stat_at_open
 with_mem_align
 with_log_path_by_env
 with_log_hints
@@ -1315,7 +1314,6 @@ Optional Features:
   --disable-cuserid       Disables attempted use of cuserid() at run time
   --disable-ld-preload    Disables support for LD_PRELOAD library
   --enable-group-readable-logs Set log files to be group readable
-  --enable-stat-at-open   Perform stat() call at open time to gather extra statistics
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -3567,17 +3565,6 @@ fi
 fi
 
 
-# Check whether --enable-stat-at-open was given.
-if test "${enable_stat_at_open+set}" = set; then :
-  enableval=$enable_stat_at_open; if test "x$enableval" = "xyes" ; then
-
-$as_echo "#define __CP_STAT_AT_OPEN 1" >>confdefs.h
-
-fi
-
-fi
-
-
 
 # Check whether --with-mem-align was given.
 if test "${with_mem_align+set}" = set; then :

darshan-runtime/darshan-core.h

@@ -11,7 +11,8 @@
 #include <sys/types.h>
 #include <stdint.h>
 
-#include "darshan.h"
+#include "uthash.h"
+#include "darshan-log-format.h"
 
 #define DARSHAN_CORE_MAX_RECORDS 1024
 
@@ -50,6 +51,9 @@ struct darshan_core_record_ref
     UT_hash_handle hlink;
 };
 
+void darshan_core_initialize(int argc, char **argv);
+void darshan_core_shutdown(void);
+
 uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
 uint64_t darshan_hash(const register unsigned char *k, register uint64_t length, register uint64_t level);
 

darshan-runtime/darshan-dynamic.h

@@ -9,16 +9,10 @@
 
 #ifdef DARSHAN_PRELOAD
 
-#include <dlfcn.h>
-#include <stdlib.h>
-
-#define DARSHAN_MPI_CALL(func) __real_ ## func
-
 #define DARSHAN_EXTERN_DECL(name,ret,args) \
   extern ret (*__real_ ## name)args;
 
 DARSHAN_EXTERN_DECL(PMPI_File_close, int, (MPI_File *fh));
-DARSHAN_EXTERN_DECL(PMPI_File_set_size, int, (MPI_File fh, MPI_Offset size));
 DARSHAN_EXTERN_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
 DARSHAN_EXTERN_DECL(PMPI_File_iread, int, (MPI_File fh, void  *buf, int  count, MPI_Datatype  datatype, __D_MPI_REQUEST  *request));
 DARSHAN_EXTERN_DECL(PMPI_File_iread_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
@@ -105,7 +99,6 @@ DARSHAN_EXTERN_DECL(PMPI_File_write_shared, int, (MPI_File fh, void *buf, int co
 DARSHAN_EXTERN_DECL(PMPI_Finalize, int, ());
 DARSHAN_EXTERN_DECL(PMPI_Init, int, (int *argc, char ***argv));
 DARSHAN_EXTERN_DECL(PMPI_Init_thread, int, (int *argc, char ***argv, int required, int *provided));
-
 DARSHAN_EXTERN_DECL(PMPI_Wtime, double, ());
 DARSHAN_EXTERN_DECL(PMPI_Allreduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm));
 DARSHAN_EXTERN_DECL(PMPI_Bcast, int, (void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm));
@@ -117,16 +110,14 @@ DARSHAN_EXTERN_DECL(PMPI_Type_contiguous, int, (int count, MPI_Datatype oldtype,
 DARSHAN_EXTERN_DECL(PMPI_Type_extent, int, (MPI_Datatype datatype, MPI_Aint *extent));
 DARSHAN_EXTERN_DECL(PMPI_Type_free, int, (MPI_Datatype *datatype));
 DARSHAN_EXTERN_DECL(PMPI_Type_hindexed, int, (int count, int *array_of_blocklengths, MPI_Aint *array_of_displacements, MPI_Datatype oldtype, MPI_Datatype *newtype));
+DARSHAN_EXTERN_DECL(PMPI_Type_get_envelope, int, (MPI_Datatype datatype, int *num_integers, int *num_addresses, int *num_datatypes, int *combiner));
+DARSHAN_EXTERN_DECL(PMPI_Type_size, int, (MPI_Datatype datatype, int *size));
 DARSHAN_EXTERN_DECL(PMPI_Op_create, int, (MPI_User_function *function, int commute, MPI_Op *op));
 DARSHAN_EXTERN_DECL(PMPI_Op_free, int, (MPI_Op *op));
 DARSHAN_EXTERN_DECL(PMPI_Reduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm));
-DARSHAN_EXTERN_DECL(PMPI_Type_get_envelope, int, (MPI_Datatype datatype, int *num_integers, int *num_addresses, int *num_datatypes, int *combiner));
-DARSHAN_EXTERN_DECL(PMPI_Type_size, int, (MPI_Datatype datatype, int *size));
-
-#else
-
-#define DARSHAN_MPI_CALL(func) func
+DARSHAN_EXTERN_DECL(PMPI_Send, int, (void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm));
+DARSHAN_EXTERN_DECL(PMPI_Recv, int, (void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status));
 
 #endif
 
-#endif
+#endif /* __DARSHAN_DYNAMIC_H */

darshan-runtime/darshan-runtime-config.h.in

@@ -105,9 +105,5 @@
 /* Memory alignment in bytes */
 #undef __CP_MEM_ALIGNMENT
 
-/* Define if Darshan should stat files at open time to collect extra
-   statistics */
-#undef __CP_STAT_AT_OPEN
-
 /* Generalized request type for MPI-IO */
 #undef __D_MPI_REQUEST

darshan-runtime/darshan.h

@@ -30,12 +30,12 @@
  * consistently regardless of whether static or dynamic linking is used
  */
 #ifdef DARSHAN_PRELOAD
-#define __USE_GNU
+
 #include <dlfcn.h>
 #include <stdlib.h>
 
 #define DARSHAN_FORWARD_DECL(name,ret,args) \
-  ret (*__real_ ## name)args = NULL;
+  ret (*__real_ ## name)args = NULL
 
 #define DARSHAN_DECL(__name) __name
 

darshan-runtime/lib/darshan-core.c

@@ -26,7 +26,9 @@
 #include <assert.h>
 
 #include "uthash.h"
+#include "darshan.h"
 #include "darshan-core.h"
+#include "darshan-dynamic.h"
 
 extern char* __progname;
 extern char* __progname_full;
@@ -70,12 +72,6 @@ static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
 static int mnt_data_count = 0;
 
 /* prototypes for internal helper functions */
-static void darshan_core_initialize(
-    int argc, char **argv);
-static void darshan_core_shutdown(
-    void);
-static void darshan_core_cleanup(
-    struct darshan_core_runtime* core);
 static void darshan_get_logfile_name(
     char* logfile_name, int jobid, struct tm* start_tm);
 static void darshan_log_record_hints_and_ver(
@@ -100,67 +96,12 @@ static int darshan_log_write_record_hash(
 static int darshan_log_append_all(
     MPI_File log_fh, struct darshan_core_runtime *core, void *buf,
     int count, uint64_t *inout_off, uint64_t *agg_uncomp_sz);
-
-/* intercept MPI initialize and finalize to manage darshan core runtime */
-int MPI_Init(int *argc, char ***argv)
-{
-    int ret;
-
-    ret = DARSHAN_MPI_CALL(PMPI_Init)(argc, argv);
-    if(ret != MPI_SUCCESS)
-    {
-        return(ret);
-    }
-
-    if(argc && argv)
-    {
-        darshan_core_initialize(*argc, *argv);
-    }
-    else
-    {
-        /* we don't see argc and argv here in fortran */
-        darshan_core_initialize(0, NULL);
-    }
-
-    return(ret);
-}
-
-int MPI_Init_thread(int *argc, char ***argv, int required, int *provided)
-{
-    int ret;
-
-    ret = DARSHAN_MPI_CALL(PMPI_Init_thread)(argc, argv, required, provided);
-    if(ret != MPI_SUCCESS)
-    {
-        return(ret);
-    }
-
-    if(argc && argv)
-    {
-        darshan_core_initialize(*argc, *argv);
-    }
-    else
-    {
-        /* we don't see argc and argv here in fortran */
-        darshan_core_initialize(0, NULL);
-    }
-
-    return(ret);
-}
-
-int MPI_Finalize(void)
-{
-    int ret;
-
-    darshan_core_shutdown();
-
-    ret = DARSHAN_MPI_CALL(PMPI_Finalize)();
-    return(ret);
-}
+static void darshan_core_cleanup(
+    struct darshan_core_runtime* core);
 
 /* *********************************** */
 
-static void darshan_core_initialize(int argc, char **argv)
+void darshan_core_initialize(int argc, char **argv)
 {
     int i;
     int internal_timing_flag = 0;
@@ -270,7 +211,7 @@ static void darshan_core_initialize(int argc, char **argv)
     return;
 }
 
-static void darshan_core_shutdown()
+void darshan_core_shutdown()
 {
     int i;
     char *logfile_name;
@@ -769,33 +710,7 @@ static void darshan_core_shutdown()
     return;
 }
 
-/* free darshan core data structures to shutdown */
-static void darshan_core_cleanup(struct darshan_core_runtime* core)
-{
-    struct darshan_core_record_ref *tmp, *ref;
-    int i;
-
-    HASH_ITER(hlink, core->rec_hash, ref, tmp)
-    {
-        HASH_DELETE(hlink, core->rec_hash, ref);
-        free(ref->rec.name);
-        free(ref);
-    }
-
-    for(i = 0; i < DARSHAN_MAX_MODS; i++)
-    {
-        if(core->mod_array[i])
-        {        
-            free(core->mod_array[i]);
-            core->mod_array[i] = NULL;
-        }
-    }
-
-    free(core->trailing_data);
-    free(core);
-
-    return;
-}
+/* *********************************** */
 
 /* construct the darshan log file name */
 static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* start_tm)
@@ -1564,6 +1479,34 @@ static int darshan_log_append_all(MPI_File log_fh, struct darshan_core_runtime *
     return(0);
 }
 
+/* free darshan core data structures to shutdown */
+static void darshan_core_cleanup(struct darshan_core_runtime* core)
+{
+    struct darshan_core_record_ref *tmp, *ref;
+    int i;
+
+    HASH_ITER(hlink, core->rec_hash, ref, tmp)
+    {
+        HASH_DELETE(hlink, core->rec_hash, ref);
+        free(ref->rec.name);
+        free(ref);
+    }
+
+    for(i = 0; i < DARSHAN_MAX_MODS; i++)
+    {
+        if(core->mod_array[i])
+        {        
+            free(core->mod_array[i]);
+            core->mod_array[i] = NULL;
+        }
+    }
+
+    free(core->trailing_data);
+    free(core);
+
+    return;
+}
+
 /* ********************************************************* */
 
 void darshan_core_register_module(

darshan-runtime/lib/darshan-mpiio.c

@@ -27,6 +27,7 @@
 #include "uthash.h"
 #include "darshan.h"
 #include "darshan-mpiio-log-format.h"
+#include "darshan-dynamic.h"
 
 /* The mpiio_file_runtime structure maintains necessary runtime metadata
  * for the MPIIO file record (darshan_mpiio_file structure, defined in

darshan-runtime/lib/darshan-posix.c

@@ -35,6 +35,10 @@ typedef int64_t off64_t;
 #define aiocb64 aiocb
 #endif
 
+#ifdef DARSHAN_PRELOAD
+extern double (*__real_PMPI_Comm_rank)(MPI_Comm comm, int *rank);
+#endif
+
 /* TODO: more libc, fgetc, etc etc etc. */
 
 DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
@@ -215,7 +219,7 @@ static int darshan_mem_alignment = 1;
 } while(0)
 
 #define POSIX_RECORD_READ(__ret, __fd, __pread_flag, __pread_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
-    size_t stride; \
+    /* size_t stride; */\
     int64_t this_offset; \
     struct posix_file_runtime* file; \
     int64_t file_alignment; \
@@ -231,11 +235,11 @@ static int darshan_mem_alignment = 1;
         DARSHAN_COUNTER_INC(file->file_record, POSIX_SEQ_READS, 1); \
     if(this_offset == (file->last_byte_read + 1)) \
         DARSHAN_COUNTER_INC(file->file_record, POSIX_CONSEC_READS, 1); \
-    if(this_offset > 0 && this_offset > file->last_byte_read \
+    /* if(this_offset > 0 && this_offset > file->last_byte_read \
         && file->last_byte_read != 0) \
         stride = this_offset - file->last_byte_read - 1; \
     else \
-        stride = 0; \
+        stride = 0; */\
     file->last_byte_read = this_offset + __ret - 1; \
     file->offset = this_offset + __ret; \
     DARSHAN_COUNTER_MAX(file->file_record, POSIX_MAX_BYTE_READ, (this_offset + __ret - 1)); \
@@ -265,7 +269,7 @@ static int darshan_mem_alignment = 1;
 } while(0)
 
 #define POSIX_RECORD_WRITE(__ret, __fd, __pwrite_flag, __pwrite_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
-    size_t stride; \
+    /* size_t stride; */\
     int64_t this_offset; \
     struct posix_file_runtime* file; \
     int64_t file_alignment; \
@@ -281,11 +285,11 @@ static int darshan_mem_alignment = 1;
         DARSHAN_COUNTER_INC(file->file_record, POSIX_SEQ_WRITES, 1); \
     if(this_offset == (file->last_byte_written + 1)) \
         DARSHAN_COUNTER_INC(file->file_record, POSIX_CONSEC_WRITES, 1); \
-    if(this_offset > 0 && this_offset > file->last_byte_written \
+    /* if(this_offset > 0 && this_offset > file->last_byte_written \
         && file->last_byte_written != 0) \
         stride = this_offset - file->last_byte_written - 1; \
     else \
-        stride = 0; \
+        stride = 0; */\
     file->last_byte_written = this_offset + __ret - 1; \
     file->offset = this_offset + __ret; \
     DARSHAN_COUNTER_MAX(file->file_record, POSIX_MAX_BYTE_WRITTEN, (this_offset + __ret - 1)); \
@@ -579,7 +583,7 @@ ssize_t DARSHAN_DECL(read)(int fd, void *buf, size_t count)
     int aligned_flag = 0;
     double tm1, tm2;
 
-    MAP_OR_FAIL(write);
+    MAP_OR_FAIL(read);
 
     if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;