From ed3099c9e12732eafa48b21caa203278e92650e1 Mon Sep 17 00:00:00 2001 From: harms Date: Mon, 20 Dec 2010 19:54:49 +0000 Subject: [PATCH] Rework how the ldpreload code is initialized. Each function checks to see if the function is resolved and if not, resolves it at that time. The other method failed when constructors were called for libraries that were not linked in so the dlsym lookup failed. git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@369 3b7491f3-a168-0410-bf4b-c445ed680a29 --- Makefile.in | 2 +- lib/darshan-hdf5.c | 43 +++--- lib/darshan-mpi-io.c | 352 ++++++++++++++++++++++++++++++------------ lib/darshan-pnetcdf.c | 44 +++--- lib/darshan-posix.c | 149 +++++++++++------- 5 files changed, 398 insertions(+), 192 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8713930..20506af 100644 --- a/Makefile.in +++ b/Makefile.in @@ -111,7 +111,7 @@ lib/libdarshan-posix.a: lib/darshan-posix.o lib/lookup3.o lib/lookup8.o ar rcs $@ $^ lib/libdarshan.so: lib/darshan-mpi-io.po lib/darshan-pnetcdf.po lib/darshan-hdf5.po lib/darshan-posix.po lib/lookup3.po lib/lookup8.po - $(CC) $(CFLAGS_MPI_SHARED) -ldl -o $@ $^ + gcc $(CFLAGS_MPI_SHARED) -ldl -o $@ $^ -lpthread -lrt install:: all install -d $(libdir) diff --git a/lib/darshan-hdf5.c b/lib/darshan-hdf5.c index f0469b7..6de2bcc 100644 --- a/lib/darshan-hdf5.c +++ b/lib/darshan-hdf5.c @@ -16,11 +16,25 @@ typedef int herr_t; #ifdef DARSHAN_PRELOAD +#define __USE_GNU +#include +#include + #define DARSHAN_FORWARD_DECL(name,ret,args) \ ret (*__real_ ## name)args = NULL; #define DARSHAN_DECL(__name) __name +#define MAP_OR_FAIL(func) \ + if (!(__real_ ## func)) \ + { \ + __real_ ## func = dlsym(RTLD_NEXT, #func); \ + if(!(__real_ ## func)) { \ + fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \ + exit(1); \ + } \ + } + #else #define DARSHAN_FORWARD_DECL(name,ret,args) \ @@ -28,33 +42,14 @@ typedef int herr_t; #define DARSHAN_DECL(__name) __wrap_ ## __name +#define MAP_OR_FAIL(func) + #endif DARSHAN_FORWARD_DECL(H5Fcreate, hid_t, (const char *filename, unsigned flags, hid_t create_plist, hid_t access_plist)); DARSHAN_FORWARD_DECL(H5Fopen, hid_t, (const char *filename, unsigned flags, hid_t access_plist)); DARSHAN_FORWARD_DECL(H5Fclose, herr_t, (hid_t file_id)); -#ifdef DARSHAN_PRELOAD -#define __USE_GNU -#include -#include -static void __attribute__ ((constructor)) darshan_ldpreload_init(void) -{ -#define MAP_OR_FAIL(func) \ - __real_ ## func = dlsym(RTLD_NEXT, #func); \ - if(!(__real_ ## func)) { \ - fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \ - exit(1); \ - } - - MAP_OR_FAIL(H5Fcreate); - MAP_OR_FAIL(H5Fopen); - MAP_OR_FAIL(H5Fclose); - - return; -} -#endif - static struct darshan_file_runtime* darshan_file_by_hid(int hid); hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags, @@ -65,6 +60,8 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags, char* tmp; int hash_index; + MAP_OR_FAIL(H5Fcreate); + ret = __real_H5Fcreate(filename, flags, create_plist, access_plist); if(ret >= 0) { @@ -106,6 +103,8 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags, char* tmp; int hash_index; + MAP_OR_FAIL(H5Fopen); + ret = __real_H5Fopen(filename, flags, access_plist); if(ret >= 0) { @@ -148,6 +147,8 @@ herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id) int tmp_hid = file_id; int ret; + MAP_OR_FAIL(H5Fclose); + ret = __real_H5Fclose(file_id); CP_LOCK(); diff --git a/lib/darshan-mpi-io.c b/lib/darshan-mpi-io.c index ca140fa..a1e09d7 100644 --- a/lib/darshan-mpi-io.c +++ b/lib/darshan-mpi-io.c @@ -27,6 +27,137 @@ #include "mpi.h" #include "darshan.h" +#ifdef DARSHAN_PRELOAD +#include +#include + +#define DARSHAN_MPI_CALL(func) __real_ ## func + +#define DARSHAN_FORWARD_DECL(name,ret,args) \ + ret (*__real_ ## name)args = NULL; + +#define MAP_OR_FAIL(func) \ + __real_ ## func = dlsym(RTLD_NEXT, #func); \ + if (!(__real_ ## func)) { \ + fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \ + } + +DARSHAN_FORWARD_DECL(PMPI_File_close, int, (MPI_File *fh)); +DARSHAN_FORWARD_DECL(PMPI_File_set_size, int, (MPI_File fh, MPI_Offset size)); +DARSHAN_FORWARD_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_iread, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_iread_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_iwrite_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_iwrite, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_iwrite_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)); +DARSHAN_FORWARD_DECL(PMPI_File_open, int, (MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_File *fh)); +DARSHAN_FORWARD_DECL(PMPI_File_read_all_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_read_all, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_read_at_all, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_read_at_all_begin, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_read_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_read, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_read_ordered_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_read_ordered, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_read_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_set_view, int, (MPI_File fh, MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, char *datarep, MPI_Info info)); +DARSHAN_FORWARD_DECL(PMPI_File_sync, int, (MPI_File fh)); +DARSHAN_FORWARD_DECL(PMPI_File_write_all_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_write_all, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_write_at_all_begin, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_write_at_all, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_write_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_write, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_write_ordered_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype)); +DARSHAN_FORWARD_DECL(PMPI_File_write_ordered, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_File_write_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)); +DARSHAN_FORWARD_DECL(PMPI_Finalize, int, ()); +DARSHAN_FORWARD_DECL(PMPI_Init, int, (int *argc, char ***argv)); +DARSHAN_FORWARD_DECL(PMPI_Init_thread, int, (int *argc, char ***argv, int required, int *provided)); + +DARSHAN_FORWARD_DECL(PMPI_Wtime, double, ()); +DARSHAN_FORWARD_DECL(PMPI_Allreduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)); +DARSHAN_FORWARD_DECL(PMPI_Bcast, int, (void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)); +DARSHAN_FORWARD_DECL(PMPI_Comm_rank, int, (MPI_Comm comm, int *rank)); +DARSHAN_FORWARD_DECL(PMPI_Comm_size, int, (MPI_Comm comm, int *size)); +DARSHAN_FORWARD_DECL(PMPI_Scan, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)); +DARSHAN_FORWARD_DECL(PMPI_Type_commit, int, (MPI_Datatype *datatype)); +DARSHAN_FORWARD_DECL(PMPI_Type_contiguous, int, (int count, MPI_Datatype oldtype, MPI_Datatype *newtype)); +DARSHAN_FORWARD_DECL(PMPI_Type_extent, int, (MPI_Datatype datatype, MPI_Aint *extent)); +DARSHAN_FORWARD_DECL(PMPI_Type_free, int, (MPI_Datatype *datatype)); +DARSHAN_FORWARD_DECL(PMPI_Type_hindexed, int, (int count, int *array_of_blocklengths, MPI_Aint *array_of_displacements, MPI_Datatype oldtype, MPI_Datatype *newtype)); +DARSHAN_FORWARD_DECL(PMPI_Op_create, int, (MPI_User_function *function, int commute, MPI_Op *op)); +DARSHAN_FORWARD_DECL(PMPI_Reduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)); +DARSHAN_FORWARD_DECL(PMPI_Type_get_envelope, int, (MPI_Datatype datatype, int *num_integers, int *num_addresses, int *num_datatypes, int *combiner)); +DARSHAN_FORWARD_DECL(PMPI_Type_size, int, (MPI_Datatype datatype, int *size)); + +void resolve_mpi_symbols (void) +{ + /* + * Overloaded functions + */ + MAP_OR_FAIL(PMPI_File_close); + MAP_OR_FAIL(PMPI_File_set_size); + MAP_OR_FAIL(PMPI_File_iread_at); + MAP_OR_FAIL(PMPI_File_iread); + MAP_OR_FAIL(PMPI_File_iread_shared); + MAP_OR_FAIL(PMPI_File_iwrite_at); + MAP_OR_FAIL(PMPI_File_iwrite); + MAP_OR_FAIL(PMPI_File_iwrite_shared); + MAP_OR_FAIL(PMPI_File_open); + MAP_OR_FAIL(PMPI_File_read_all_begin); + MAP_OR_FAIL(PMPI_File_read_all); + MAP_OR_FAIL(PMPI_File_read_at_all_begin); + MAP_OR_FAIL(PMPI_File_read_at_all); + MAP_OR_FAIL(PMPI_File_read_at); + MAP_OR_FAIL(PMPI_File_read); + MAP_OR_FAIL(PMPI_File_read_ordered_begin); + MAP_OR_FAIL(PMPI_File_read_ordered); + MAP_OR_FAIL(PMPI_File_read_shared); + MAP_OR_FAIL(PMPI_File_set_view); + MAP_OR_FAIL(PMPI_File_sync); + MAP_OR_FAIL(PMPI_File_write_all_begin); + MAP_OR_FAIL(PMPI_File_write_all); + MAP_OR_FAIL(PMPI_File_write_at_all_begin); + MAP_OR_FAIL(PMPI_File_write_at_all); + MAP_OR_FAIL(PMPI_File_write_at); + MAP_OR_FAIL(PMPI_File_write); + MAP_OR_FAIL(PMPI_File_write_ordered_begin); + MAP_OR_FAIL(PMPI_File_write_ordered); + MAP_OR_FAIL(PMPI_File_write_shared); + MAP_OR_FAIL(PMPI_Finalize); + MAP_OR_FAIL(PMPI_Init); + MAP_OR_FAIL(PMPI_Init_thread); + + /* + * These function are not intercepted but are used + * by darshan itself. + */ + MAP_OR_FAIL(PMPI_Wtime); + MAP_OR_FAIL(PMPI_Allreduce); + MAP_OR_FAIL(PMPI_Bcast); + MAP_OR_FAIL(PMPI_Comm_rank); + MAP_OR_FAIL(PMPI_Comm_size); + MAP_OR_FAIL(PMPI_Scan); + MAP_OR_FAIL(PMPI_Type_commit); + MAP_OR_FAIL(PMPI_Type_contiguous); + MAP_OR_FAIL(PMPI_Type_extent); + MAP_OR_FAIL(PMPI_Type_free); + MAP_OR_FAIL(PMPI_Type_size); + MAP_OR_FAIL(PMPI_Type_hindexed); + MAP_OR_FAIL(PMPI_Op_create); + MAP_OR_FAIL(PMPI_Reduce); + MAP_OR_FAIL(PMPI_Type_get_envelope); + + return; +} + +#else + +#define DARSHAN_MPI_CALL(func) func + +#endif + extern char* __progname; /* maximum number of memory segments each process will write to the log */ @@ -34,8 +165,8 @@ extern char* __progname; #define CP_DATATYPE_INC(__file, __datatype) do {\ int num_integers, num_addresses, num_datatypes, combiner, ret; \ - ret = MPI_Type_get_envelope(__datatype, &num_integers, &num_addresses, \ - &num_datatypes, &combiner); \ + ret = DARSHAN_MPI_CALL(PMPI_Type_get_envelope)(__datatype, &num_integers, \ + &num_addresses, &num_datatypes, &combiner); \ if(ret == MPI_SUCCESS) { \ switch(combiner) { \ case MPI_COMBINER_NAMED:\ @@ -85,9 +216,9 @@ extern char* __progname; if(__ret != MPI_SUCCESS) break; \ file = darshan_file_by_fh(__fh); \ if(!file) break; \ - MPI_Type_size(__datatype, &size); \ + DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \ size = size * __count; \ - MPI_Type_extent(__datatype, &extent); \ + DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \ CP_BUCKET_INC(file, CP_SIZE_WRITE_AGG_0_100, size); \ CP_BUCKET_INC(file, CP_EXTENT_WRITE_0_100, extent); \ CP_INC(file, __counter, 1); \ @@ -105,9 +236,9 @@ extern char* __progname; if(__ret != MPI_SUCCESS) break; \ file = darshan_file_by_fh(__fh); \ if(!file) break; \ - MPI_Type_size(__datatype, &size); \ + DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \ size = size * __count; \ - MPI_Type_extent(__datatype, &extent); \ + DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \ CP_BUCKET_INC(file, CP_SIZE_READ_AGG_0_100, size); \ CP_BUCKET_INC(file, CP_EXTENT_READ_0_100, extent); \ CP_INC(file, __counter, 1); \ @@ -164,7 +295,11 @@ int MPI_Init(int *argc, char ***argv) { int ret; - ret = PMPI_Init(argc, argv); +#ifdef DARSHAN_PRELOAD + resolve_mpi_symbols(); +#endif + + ret = DARSHAN_MPI_CALL(PMPI_Init)(argc, argv); if(ret != MPI_SUCCESS) { return(ret); @@ -179,7 +314,7 @@ int MPI_Init_thread (int *argc, char ***argv, int required, int *provided) { int ret; - ret = PMPI_Init_thread(argc, argv, required, provided); + ret = DARSHAN_MPI_CALL(PMPI_Init_thread)(argc, argv, required, provided); if (ret != MPI_SUCCESS) { return(ret); @@ -195,8 +330,8 @@ static void darshan_mpi_initialize(int *argc, char ***argv) int nprocs; int rank; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs); + DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank); CP_LOCK(); if(argc && argv) @@ -256,7 +391,7 @@ void darshan_shutdown(int timing_flag) flags = final_job->flags; CP_UNLOCK(); - start_log_time = MPI_Wtime(); + start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)(); /* figure out which access sizes to log */ darshan_walk_file_accesses(final_job); @@ -303,7 +438,7 @@ void darshan_shutdown(int timing_flag) return; } - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank); /* collect information about command line and * mounted file systems @@ -319,12 +454,12 @@ void darshan_shutdown(int timing_flag) CP_MAX_MNTS*sizeof(int64_t)); } - bcst1=MPI_Wtime(); - MPI_Bcast(mnt_id_array_root, CP_MAX_MNTS*sizeof(int64_t), MPI_BYTE, 0, - MPI_COMM_WORLD); - MPI_Bcast(mnt_hash_array_root, CP_MAX_MNTS*sizeof(uint64_t), MPI_BYTE, 0, - MPI_COMM_WORLD); - bcst2=MPI_Wtime(); + bcst1=DARSHAN_MPI_CALL(PMPI_Wtime)(); + DARSHAN_MPI_CALL(PMPI_Bcast)(mnt_id_array_root, + CP_MAX_MNTS*sizeof(int64_t), MPI_BYTE, 0, MPI_COMM_WORLD); + DARSHAN_MPI_CALL(PMPI_Bcast)(mnt_hash_array_root, + CP_MAX_MNTS*sizeof(uint64_t), MPI_BYTE, 0, MPI_COMM_WORLD); + bcst2=DARSHAN_MPI_CALL(PMPI_Wtime)(); /* identify any common mount points that have different device ids on * non-root processes @@ -431,19 +566,20 @@ void darshan_shutdown(int timing_flag) } /* broadcast log file name */ - bcst3=MPI_Wtime(); - MPI_Bcast(logfile_name, PATH_MAX, MPI_CHAR, 0, MPI_COMM_WORLD); + bcst3=DARSHAN_MPI_CALL(PMPI_Wtime)(); + DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0, + MPI_COMM_WORLD); final_job->log_job.end_time = time(NULL); /* reduce records for shared files */ if(timing_flag) - red1 = MPI_Wtime(); + red1 = DARSHAN_MPI_CALL(PMPI_Wtime)(); local_ret = cp_log_reduction(final_job, rank, logfile_name, &next_offset); if(timing_flag) - red2 = MPI_Wtime(); - MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR, + red2 = DARSHAN_MPI_CALL(PMPI_Wtime)(); + DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD); if(all_ret == 0) @@ -457,26 +593,26 @@ void darshan_shutdown(int timing_flag) { /* compress data */ if(timing_flag) - gz1 = MPI_Wtime(); + gz1 = DARSHAN_MPI_CALL(PMPI_Wtime)(); local_ret = cp_log_compress(final_job, rank, &index_count, lengths, pointers); if(timing_flag) - gz2 = MPI_Wtime(); - MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR, - MPI_COMM_WORLD); + gz2 = DARSHAN_MPI_CALL(PMPI_Wtime)(); + DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1, + MPI_INT, MPI_LOR, MPI_COMM_WORLD); } if(all_ret == 0) { /* actually write out log file */ if(timing_flag) - write1 = MPI_Wtime(); + write1 = DARSHAN_MPI_CALL(PMPI_Wtime)(); local_ret = cp_log_write(final_job, rank, logfile_name, index_count, lengths, pointers, start_log_time); if(timing_flag) - write2 = MPI_Wtime(); - MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR, - MPI_COMM_WORLD); + write2 = DARSHAN_MPI_CALL(PMPI_Wtime)(); + DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1, + MPI_INT, MPI_LOR, MPI_COMM_WORLD); } /* if any process failed to write log, then delete the whole file so we @@ -500,7 +636,7 @@ void darshan_shutdown(int timing_flag) double all_tm, all_slowest; double bcst_tm, bcst_slowest; - tm_end = MPI_Wtime(); + tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)(); bcst_tm=(bcst2-bcst1)+(red1-bcst3); red_tm = red2-red1; @@ -508,20 +644,20 @@ void darshan_shutdown(int timing_flag) write_tm = write2-write1; all_tm = tm_end-start_log_time; - MPI_Allreduce(&red_tm, &red_slowest, 1, + DARSHAN_MPI_CALL(PMPI_Allreduce)(&red_tm, &red_slowest, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&gz_tm, &gz_slowest, 1, + DARSHAN_MPI_CALL(PMPI_Allreduce)(&gz_tm, &gz_slowest, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&write_tm, &write_slowest, 1, + DARSHAN_MPI_CALL(PMPI_Allreduce)(&write_tm, &write_slowest, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&all_tm, &all_slowest, 1, + DARSHAN_MPI_CALL(PMPI_Allreduce)(&all_tm, &all_slowest, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&bcst_tm, &bcst_slowest, 1, + DARSHAN_MPI_CALL(PMPI_Allreduce)(&bcst_tm, &bcst_slowest, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if(rank == 0) { - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs); printf("#\t\t