Commit ed3099c9 authored by Kevin Harms's avatar Kevin Harms

Rework how the ldpreload code is initialized. Each function checks to

see if the function is resolved and if not, resolves it at that time.
The other method failed when constructors were called for libraries that
were not linked in so the dlsym lookup failed.


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@369 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 85a1981a
......@@ -111,7 +111,7 @@ lib/libdarshan-posix.a: lib/darshan-posix.o lib/lookup3.o lib/lookup8.o
ar rcs $@ $^
lib/libdarshan.so: lib/darshan-mpi-io.po lib/darshan-pnetcdf.po lib/darshan-hdf5.po lib/darshan-posix.po lib/lookup3.po lib/lookup8.po
$(CC) $(CFLAGS_MPI_SHARED) -ldl -o $@ $^
gcc $(CFLAGS_MPI_SHARED) -ldl -o $@ $^ -lpthread -lrt
install:: all
install -d $(libdir)
......
......@@ -16,11 +16,25 @@ typedef int herr_t;
#ifdef DARSHAN_PRELOAD
#define __USE_GNU
#include <dlfcn.h>
#include <stdlib.h>
#define DARSHAN_FORWARD_DECL(name,ret,args) \
ret (*__real_ ## name)args = NULL;
#define DARSHAN_DECL(__name) __name
#define MAP_OR_FAIL(func) \
if (!(__real_ ## func)) \
{ \
__real_ ## func = dlsym(RTLD_NEXT, #func); \
if(!(__real_ ## func)) { \
fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \
exit(1); \
} \
}
#else
#define DARSHAN_FORWARD_DECL(name,ret,args) \
......@@ -28,33 +42,14 @@ typedef int herr_t;
#define DARSHAN_DECL(__name) __wrap_ ## __name
#define MAP_OR_FAIL(func)
#endif
DARSHAN_FORWARD_DECL(H5Fcreate, hid_t, (const char *filename, unsigned flags, hid_t create_plist, hid_t access_plist));
DARSHAN_FORWARD_DECL(H5Fopen, hid_t, (const char *filename, unsigned flags, hid_t access_plist));
DARSHAN_FORWARD_DECL(H5Fclose, herr_t, (hid_t file_id));
#ifdef DARSHAN_PRELOAD
#define __USE_GNU
#include <dlfcn.h>
#include <stdlib.h>
static void __attribute__ ((constructor)) darshan_ldpreload_init(void)
{
#define MAP_OR_FAIL(func) \
__real_ ## func = dlsym(RTLD_NEXT, #func); \
if(!(__real_ ## func)) { \
fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \
exit(1); \
}
MAP_OR_FAIL(H5Fcreate);
MAP_OR_FAIL(H5Fopen);
MAP_OR_FAIL(H5Fclose);
return;
}
#endif
static struct darshan_file_runtime* darshan_file_by_hid(int hid);
hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
......@@ -65,6 +60,8 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
char* tmp;
int hash_index;
MAP_OR_FAIL(H5Fcreate);
ret = __real_H5Fcreate(filename, flags, create_plist, access_plist);
if(ret >= 0)
{
......@@ -106,6 +103,8 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
char* tmp;
int hash_index;
MAP_OR_FAIL(H5Fopen);
ret = __real_H5Fopen(filename, flags, access_plist);
if(ret >= 0)
{
......@@ -148,6 +147,8 @@ herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
int tmp_hid = file_id;
int ret;
MAP_OR_FAIL(H5Fclose);
ret = __real_H5Fclose(file_id);
CP_LOCK();
......
......@@ -27,6 +27,137 @@
#include "mpi.h"
#include "darshan.h"
#ifdef DARSHAN_PRELOAD
#include <dlfcn.h>
#include <stdlib.h>
#define DARSHAN_MPI_CALL(func) __real_ ## func
#define DARSHAN_FORWARD_DECL(name,ret,args) \
ret (*__real_ ## name)args = NULL;
#define MAP_OR_FAIL(func) \
__real_ ## func = dlsym(RTLD_NEXT, #func); \
if (!(__real_ ## func)) { \
fprintf(stderr, "Darshan failed to map symbol: %s\n", #func); \
}
DARSHAN_FORWARD_DECL(PMPI_File_close, int, (MPI_File *fh));
DARSHAN_FORWARD_DECL(PMPI_File_set_size, int, (MPI_File fh, MPI_Offset size));
DARSHAN_FORWARD_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_iread, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_iread_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_iwrite_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_iwrite, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_iwrite_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request));
DARSHAN_FORWARD_DECL(PMPI_File_open, int, (MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_File *fh));
DARSHAN_FORWARD_DECL(PMPI_File_read_all_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_read_all, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_read_at_all, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_read_at_all_begin, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_read_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_read, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_read_ordered_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_read_ordered, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_read_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_set_view, int, (MPI_File fh, MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, char *datarep, MPI_Info info));
DARSHAN_FORWARD_DECL(PMPI_File_sync, int, (MPI_File fh));
DARSHAN_FORWARD_DECL(PMPI_File_write_all_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_write_all, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_write_at_all_begin, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_write_at_all, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_write_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_write, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_write_ordered_begin, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype));
DARSHAN_FORWARD_DECL(PMPI_File_write_ordered, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_File_write_shared, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status));
DARSHAN_FORWARD_DECL(PMPI_Finalize, int, ());
DARSHAN_FORWARD_DECL(PMPI_Init, int, (int *argc, char ***argv));
DARSHAN_FORWARD_DECL(PMPI_Init_thread, int, (int *argc, char ***argv, int required, int *provided));
DARSHAN_FORWARD_DECL(PMPI_Wtime, double, ());
DARSHAN_FORWARD_DECL(PMPI_Allreduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm));
DARSHAN_FORWARD_DECL(PMPI_Bcast, int, (void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm));
DARSHAN_FORWARD_DECL(PMPI_Comm_rank, int, (MPI_Comm comm, int *rank));
DARSHAN_FORWARD_DECL(PMPI_Comm_size, int, (MPI_Comm comm, int *size));
DARSHAN_FORWARD_DECL(PMPI_Scan, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm));
DARSHAN_FORWARD_DECL(PMPI_Type_commit, int, (MPI_Datatype *datatype));
DARSHAN_FORWARD_DECL(PMPI_Type_contiguous, int, (int count, MPI_Datatype oldtype, MPI_Datatype *newtype));
DARSHAN_FORWARD_DECL(PMPI_Type_extent, int, (MPI_Datatype datatype, MPI_Aint *extent));
DARSHAN_FORWARD_DECL(PMPI_Type_free, int, (MPI_Datatype *datatype));
DARSHAN_FORWARD_DECL(PMPI_Type_hindexed, int, (int count, int *array_of_blocklengths, MPI_Aint *array_of_displacements, MPI_Datatype oldtype, MPI_Datatype *newtype));
DARSHAN_FORWARD_DECL(PMPI_Op_create, int, (MPI_User_function *function, int commute, MPI_Op *op));
DARSHAN_FORWARD_DECL(PMPI_Reduce, int, (void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm));
DARSHAN_FORWARD_DECL(PMPI_Type_get_envelope, int, (MPI_Datatype datatype, int *num_integers, int *num_addresses, int *num_datatypes, int *combiner));
DARSHAN_FORWARD_DECL(PMPI_Type_size, int, (MPI_Datatype datatype, int *size));
void resolve_mpi_symbols (void)
{
/*
* Overloaded functions
*/
MAP_OR_FAIL(PMPI_File_close);
MAP_OR_FAIL(PMPI_File_set_size);
MAP_OR_FAIL(PMPI_File_iread_at);
MAP_OR_FAIL(PMPI_File_iread);
MAP_OR_FAIL(PMPI_File_iread_shared);
MAP_OR_FAIL(PMPI_File_iwrite_at);
MAP_OR_FAIL(PMPI_File_iwrite);
MAP_OR_FAIL(PMPI_File_iwrite_shared);
MAP_OR_FAIL(PMPI_File_open);
MAP_OR_FAIL(PMPI_File_read_all_begin);
MAP_OR_FAIL(PMPI_File_read_all);
MAP_OR_FAIL(PMPI_File_read_at_all_begin);
MAP_OR_FAIL(PMPI_File_read_at_all);
MAP_OR_FAIL(PMPI_File_read_at);
MAP_OR_FAIL(PMPI_File_read);
MAP_OR_FAIL(PMPI_File_read_ordered_begin);
MAP_OR_FAIL(PMPI_File_read_ordered);
MAP_OR_FAIL(PMPI_File_read_shared);
MAP_OR_FAIL(PMPI_File_set_view);
MAP_OR_FAIL(PMPI_File_sync);
MAP_OR_FAIL(PMPI_File_write_all_begin);
MAP_OR_FAIL(PMPI_File_write_all);
MAP_OR_FAIL(PMPI_File_write_at_all_begin);
MAP_OR_FAIL(PMPI_File_write_at_all);
MAP_OR_FAIL(PMPI_File_write_at);
MAP_OR_FAIL(PMPI_File_write);
MAP_OR_FAIL(PMPI_File_write_ordered_begin);
MAP_OR_FAIL(PMPI_File_write_ordered);
MAP_OR_FAIL(PMPI_File_write_shared);
MAP_OR_FAIL(PMPI_Finalize);
MAP_OR_FAIL(PMPI_Init);
MAP_OR_FAIL(PMPI_Init_thread);
/*
* These function are not intercepted but are used
* by darshan itself.
*/
MAP_OR_FAIL(PMPI_Wtime);
MAP_OR_FAIL(PMPI_Allreduce);
MAP_OR_FAIL(PMPI_Bcast);
MAP_OR_FAIL(PMPI_Comm_rank);
MAP_OR_FAIL(PMPI_Comm_size);
MAP_OR_FAIL(PMPI_Scan);
MAP_OR_FAIL(PMPI_Type_commit);
MAP_OR_FAIL(PMPI_Type_contiguous);
MAP_OR_FAIL(PMPI_Type_extent);
MAP_OR_FAIL(PMPI_Type_free);
MAP_OR_FAIL(PMPI_Type_size);
MAP_OR_FAIL(PMPI_Type_hindexed);
MAP_OR_FAIL(PMPI_Op_create);
MAP_OR_FAIL(PMPI_Reduce);
MAP_OR_FAIL(PMPI_Type_get_envelope);
return;
}
#else
#define DARSHAN_MPI_CALL(func) func
#endif
extern char* __progname;
/* maximum number of memory segments each process will write to the log */
......@@ -34,8 +165,8 @@ extern char* __progname;
#define CP_DATATYPE_INC(__file, __datatype) do {\
int num_integers, num_addresses, num_datatypes, combiner, ret; \
ret = MPI_Type_get_envelope(__datatype, &num_integers, &num_addresses, \
&num_datatypes, &combiner); \
ret = DARSHAN_MPI_CALL(PMPI_Type_get_envelope)(__datatype, &num_integers, \
&num_addresses, &num_datatypes, &combiner); \
if(ret == MPI_SUCCESS) { \
switch(combiner) { \
case MPI_COMBINER_NAMED:\
......@@ -85,9 +216,9 @@ extern char* __progname;
if(__ret != MPI_SUCCESS) break; \
file = darshan_file_by_fh(__fh); \
if(!file) break; \
MPI_Type_size(__datatype, &size); \
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
MPI_Type_extent(__datatype, &extent); \
DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \
CP_BUCKET_INC(file, CP_SIZE_WRITE_AGG_0_100, size); \
CP_BUCKET_INC(file, CP_EXTENT_WRITE_0_100, extent); \
CP_INC(file, __counter, 1); \
......@@ -105,9 +236,9 @@ extern char* __progname;
if(__ret != MPI_SUCCESS) break; \
file = darshan_file_by_fh(__fh); \
if(!file) break; \
MPI_Type_size(__datatype, &size); \
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
MPI_Type_extent(__datatype, &extent); \
DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \
CP_BUCKET_INC(file, CP_SIZE_READ_AGG_0_100, size); \
CP_BUCKET_INC(file, CP_EXTENT_READ_0_100, extent); \
CP_INC(file, __counter, 1); \
......@@ -164,7 +295,11 @@ int MPI_Init(int *argc, char ***argv)
{
int ret;
ret = PMPI_Init(argc, argv);
#ifdef DARSHAN_PRELOAD
resolve_mpi_symbols();
#endif
ret = DARSHAN_MPI_CALL(PMPI_Init)(argc, argv);
if(ret != MPI_SUCCESS)
{
return(ret);
......@@ -179,7 +314,7 @@ int MPI_Init_thread (int *argc, char ***argv, int required, int *provided)
{
int ret;
ret = PMPI_Init_thread(argc, argv, required, provided);
ret = DARSHAN_MPI_CALL(PMPI_Init_thread)(argc, argv, required, provided);
if (ret != MPI_SUCCESS)
{
return(ret);
......@@ -195,8 +330,8 @@ static void darshan_mpi_initialize(int *argc, char ***argv)
int nprocs;
int rank;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank);
CP_LOCK();
if(argc && argv)
......@@ -256,7 +391,7 @@ void darshan_shutdown(int timing_flag)
flags = final_job->flags;
CP_UNLOCK();
start_log_time = MPI_Wtime();
start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* figure out which access sizes to log */
darshan_walk_file_accesses(final_job);
......@@ -303,7 +438,7 @@ void darshan_shutdown(int timing_flag)
return;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank);
/* collect information about command line and
* mounted file systems
......@@ -319,12 +454,12 @@ void darshan_shutdown(int timing_flag)
CP_MAX_MNTS*sizeof(int64_t));
}
bcst1=MPI_Wtime();
MPI_Bcast(mnt_id_array_root, CP_MAX_MNTS*sizeof(int64_t), MPI_BYTE, 0,
MPI_COMM_WORLD);
MPI_Bcast(mnt_hash_array_root, CP_MAX_MNTS*sizeof(uint64_t), MPI_BYTE, 0,
MPI_COMM_WORLD);
bcst2=MPI_Wtime();
bcst1=DARSHAN_MPI_CALL(PMPI_Wtime)();
DARSHAN_MPI_CALL(PMPI_Bcast)(mnt_id_array_root,
CP_MAX_MNTS*sizeof(int64_t), MPI_BYTE, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Bcast)(mnt_hash_array_root,
CP_MAX_MNTS*sizeof(uint64_t), MPI_BYTE, 0, MPI_COMM_WORLD);
bcst2=DARSHAN_MPI_CALL(PMPI_Wtime)();
/* identify any common mount points that have different device ids on
* non-root processes
......@@ -431,19 +566,20 @@ void darshan_shutdown(int timing_flag)
}
/* broadcast log file name */
bcst3=MPI_Wtime();
MPI_Bcast(logfile_name, PATH_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
bcst3=DARSHAN_MPI_CALL(PMPI_Wtime)();
DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
MPI_COMM_WORLD);
final_job->log_job.end_time = time(NULL);
/* reduce records for shared files */
if(timing_flag)
red1 = MPI_Wtime();
red1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
local_ret = cp_log_reduction(final_job, rank, logfile_name,
&next_offset);
if(timing_flag)
red2 = MPI_Wtime();
MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR,
red2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR,
MPI_COMM_WORLD);
if(all_ret == 0)
......@@ -457,26 +593,26 @@ void darshan_shutdown(int timing_flag)
{
/* compress data */
if(timing_flag)
gz1 = MPI_Wtime();
gz1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
local_ret = cp_log_compress(final_job, rank, &index_count,
lengths, pointers);
if(timing_flag)
gz2 = MPI_Wtime();
MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR,
MPI_COMM_WORLD);
gz2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1,
MPI_INT, MPI_LOR, MPI_COMM_WORLD);
}
if(all_ret == 0)
{
/* actually write out log file */
if(timing_flag)
write1 = MPI_Wtime();
write1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
local_ret = cp_log_write(final_job, rank, logfile_name,
index_count, lengths, pointers, start_log_time);
if(timing_flag)
write2 = MPI_Wtime();
MPI_Allreduce(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR,
MPI_COMM_WORLD);
write2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1,
MPI_INT, MPI_LOR, MPI_COMM_WORLD);
}
/* if any process failed to write log, then delete the whole file so we
......@@ -500,7 +636,7 @@ void darshan_shutdown(int timing_flag)
double all_tm, all_slowest;
double bcst_tm, bcst_slowest;
tm_end = MPI_Wtime();
tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)();
bcst_tm=(bcst2-bcst1)+(red1-bcst3);
red_tm = red2-red1;
......@@ -508,20 +644,20 @@ void darshan_shutdown(int timing_flag)
write_tm = write2-write1;
all_tm = tm_end-start_log_time;
MPI_Allreduce(&red_tm, &red_slowest, 1,
DARSHAN_MPI_CALL(PMPI_Allreduce)(&red_tm, &red_slowest, 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&gz_tm, &gz_slowest, 1,
DARSHAN_MPI_CALL(PMPI_Allreduce)(&gz_tm, &gz_slowest, 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&write_tm, &write_slowest, 1,
DARSHAN_MPI_CALL(PMPI_Allreduce)(&write_tm, &write_slowest, 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&all_tm, &all_slowest, 1,
DARSHAN_MPI_CALL(PMPI_Allreduce)(&all_tm, &all_slowest, 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&bcst_tm, &bcst_slowest, 1,
DARSHAN_MPI_CALL(PMPI_Allreduce)(&bcst_tm, &bcst_slowest, 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if(rank == 0)
{
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
printf("#<op>\t<nprocs>\t<time>\n");
printf("bcst\t%d\t%f\n", nprocs, bcst_slowest);
printf("reduce\t%d\t%f\n", nprocs, red_slowest);
......@@ -543,7 +679,7 @@ int MPI_Finalize(void)
else
darshan_shutdown(0);
ret = PMPI_Finalize();
ret = DARSHAN_MPI_CALL(PMPI_Finalize)();
return(ret);
}
......@@ -558,7 +694,7 @@ int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_F
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_open(comm, filename, amode, info, fh);
ret = DARSHAN_MPI_CALL(PMPI_File_open)(comm, filename, amode, info, fh);
tm2 = darshan_wtime();
if(ret == MPI_SUCCESS)
......@@ -583,8 +719,9 @@ int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_F
CP_SET(file, CP_MODE, amode);
CP_F_INC(file, CP_F_MPI_META_TIME, (tm2-tm1));
if(CP_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0)
CP_F_SET(file, CP_F_OPEN_TIMESTAMP, MPI_Wtime());
MPI_Comm_size(comm, &comm_size);
CP_F_SET(file, CP_F_OPEN_TIMESTAMP,
DARSHAN_MPI_CALL(PMPI_Wtime)());
DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
if(comm_size == 1)
{
CP_INC(file, CP_INDEP_OPENS, 1);
......@@ -621,7 +758,7 @@ int MPI_File_close(MPI_File *fh)
int ret;
tm1 = darshan_wtime();
ret = PMPI_File_close(fh);
ret = DARSHAN_MPI_CALL(PMPI_File_close)(fh);
tm2 = darshan_wtime();
CP_LOCK();
......@@ -629,7 +766,7 @@ int MPI_File_close(MPI_File *fh)
if(file)
{
file->fh = MPI_FILE_NULL;
CP_F_SET(file, CP_F_CLOSE_TIMESTAMP, MPI_Wtime());
CP_F_SET(file, CP_F_CLOSE_TIMESTAMP, DARSHAN_MPI_CALL(PMPI_Wtime)());
CP_F_INC(file, CP_F_MPI_META_TIME, (tm2-tm1));
if(file->fh_prev == NULL)
{
......@@ -663,7 +800,7 @@ int MPI_File_sync(MPI_File fh)
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_sync(fh);
ret = DARSHAN_MPI_CALL(PMPI_File_sync)(fh);
tm2 = darshan_wtime();
if(ret == MPI_SUCCESS)
{
......@@ -689,7 +826,8 @@ int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_set_view(fh, disp, etype, filetype, datarep, info);
ret = DARSHAN_MPI_CALL(PMPI_File_set_view)(fh, disp, etype,
filetype, datarep, info);
tm2 = darshan_wtime();
if(ret == MPI_SUCCESS)
{
......@@ -718,7 +856,7 @@ int MPI_File_read(MPI_File fh, void *buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read)(fh, buf, count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
......@@ -733,7 +871,8 @@ int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_at(fh, offset, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read_at)(fh, offset, buf,
count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
......@@ -748,7 +887,8 @@ int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_at_all(fh, offset, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read_at_all)(fh, offset, buf,
count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
......@@ -762,7 +902,8 @@ int MPI_File_read_all(MPI_File fh, void * buf, int count, MPI_Datatype datatype,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_all(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read_all)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
......@@ -776,7 +917,8 @@ int MPI_File_read_shared(MPI_File fh, void * buf, int count, MPI_Datatype dataty
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_shared(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read_shared)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
......@@ -791,7 +933,8 @@ int MPI_File_read_ordered(MPI_File fh, void * buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_ordered(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_read_ordered)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
......@@ -806,7 +949,8 @@ int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_at_all_begin(fh, offset, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_read_at_all_begin)(fh, offset, buf,
count, datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
......@@ -820,7 +964,7 @@ int MPI_File_read_all_begin(MPI_File fh, void * buf, int count, MPI_Datatype dat
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_all_begin(fh, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_read_all_begin)(fh, buf, count, datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
......@@ -834,7 +978,8 @@ int MPI_File_read_ordered_begin(MPI_File fh, void * buf, int count, MPI_Datatype
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_read_ordered_begin(fh, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_read_ordered_begin)(fh, buf, count,
datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
......@@ -849,7 +994,8 @@ int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iread_at(fh, offset, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iread_at)(fh, offset, buf, count,
datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
......@@ -863,7 +1009,7 @@ int MPI_File_iread(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MP
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iread(fh, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iread)(fh, buf, count, datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
......@@ -878,7 +1024,8 @@ int MPI_File_iread_shared(MPI_File fh, void * buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iread_shared(fh, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iread_shared)(fh, buf, count,
datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
......@@ -894,7 +1041,7 @@ int MPI_File_write(MPI_File fh, void *buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write)(fh, buf, count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_INDEP_WRITES, tm1, tm2);
......@@ -909,7 +1056,8 @@ int MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_at(fh, offset, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(fh, offset, buf,
count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_INDEP_WRITES, tm1, tm2);
......@@ -924,7 +1072,8 @@ int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_at_all(fh, offset, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write_at_all)(fh, offset, buf,
count, datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_COLL_WRITES, tm1, tm2);
......@@ -938,7 +1087,8 @@ int MPI_File_write_all(MPI_File fh, void * buf, int count, MPI_Datatype datatype
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_all(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write_all)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_COLL_WRITES, tm1, tm2);
......@@ -952,7 +1102,8 @@ int MPI_File_write_shared(MPI_File fh, void * buf, int count, MPI_Datatype datat
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_shared(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write_shared)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_INDEP_WRITES, tm1, tm2);
......@@ -967,7 +1118,8 @@ int MPI_File_write_ordered(MPI_File fh, void * buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_ordered(fh, buf, count, datatype, status);
ret = DARSHAN_MPI_CALL(PMPI_File_write_ordered)(fh, buf, count,
datatype, status);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_COLL_WRITES, tm1, tm2);
......@@ -982,7 +1134,8 @@ int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_at_all_begin(fh, offset, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_write_at_all_begin)(fh, offset,
buf, count, datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_SPLIT_WRITES, tm1, tm2);
......@@ -996,7 +1149,7 @@ int MPI_File_write_all_begin(MPI_File fh, void * buf, int count, MPI_Datatype da
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_all_begin(fh, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_write_all_begin)(fh, buf, count, datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_SPLIT_WRITES, tm1, tm2);
......@@ -1010,7 +1163,8 @@ int MPI_File_write_ordered_begin(MPI_File fh, void * buf, int count, MPI_Datatyp
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_write_ordered_begin(fh, buf, count, datatype);
ret = DARSHAN_MPI_CALL(PMPI_File_write_ordered_begin)(fh, buf, count,
datatype);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_SPLIT_WRITES, tm1, tm2);
......@@ -1025,7 +1179,8 @@ int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, void * buf,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iwrite_at(fh, offset, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iwrite_at)(fh, offset, buf,
count, datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_NB_WRITES, tm1, tm2);
......@@ -1039,7 +1194,7 @@ int MPI_File_iwrite(MPI_File fh, void * buf, int count, MPI_Datatype datatype, M
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iwrite(fh, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iwrite)(fh, buf, count, datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_NB_WRITES, tm1, tm2);
......@@ -1054,7 +1209,8 @@ int MPI_File_iwrite_shared(MPI_File fh, void * buf, int count,
double tm1, tm2;
tm1 = darshan_wtime();
ret = PMPI_File_iwrite_shared(fh, buf, count, datatype, request);
ret = DARSHAN_MPI_CALL(PMPI_File_iwrite_shared)(fh, buf, count,
datatype, request);
tm2 = darshan_wtime();
CP_LOCK();
CP_RECORD_MPI_WRITE(ret, fh, count, datatype, CP_NB_WRITES, tm1, tm2);
......@@ -1125,7 +1281,7 @@ static int cp_log_reduction(struct darshan_job_runtime* final_job, int rank,
int shared_count = 0;
/* register a reduction operation */
ret = MPI_Op_create(darshan_file_reduce, 1, &reduce_op);
ret = DARSHAN_MPI_CALL(PMPI_Op_create)(darshan_file_reduce, 1, &reduce_op);
if(ret != 0)
{
return(-1);
......@@ -1134,8 +1290,9 @@ static int cp_log_reduction(struct darshan_job_runtime* final_job, int rank,
/* construct a datatype for a file record. This is serving no purpose
* except to make sure we can do a reduction on proper boundaries