...
 
Commits (24)
## Quick Start
$ cd darshan-serial/darshan-runtime
$ autoreconf -ivf
$ ./configure --with-log-path-by-env=PWD --with-jobid-env=RANDOM --with-mem-align=8 --disable-mpiio-mod --disable-stdio-mod --disable-pnetcdf-mod --disable-dxt-mod --disable-bgq-mod --disable-lustre-mod --without-mpi CC=gcc
$ make
This will install serial Darshan in darshan-serial/install. Then to use it,
$ LD_PRELOAD=/path/to/darshan-serial/darshan-runtime/lib/libdarshan.so DARSHAN_ENABLE_NONMPI=1 ./path/to/my.exe
Your `my.exe` will run and create a Darshan log in `$PWD`. Regular old
`darshan-parser` can be used to parse the logfile.
## Non-MPI Darshan
This repository contains a version of Darshan that supports profiling in the
absence of MPI at either runtime or compile time. It can be built in two ways:
1. **With MPI**, but capable of profiling non-MPI applications if linked against
them. This is useful if a single build should work with both MPI and non-MPI
applications.
2. **Without MPI** and capable of only profiling non-MPI applications. This is
useful for building Darshan on systems that simply don't have MPI at all.
At present, mode #2 only supports the POSIX module. All other modules should be
disabled at configure-time using:
./configure --disable-mpiio-mod \
--disable-stdio-mod \
--disable-pnetcdf-mod \
--disable-bgq-mod \
--disable-lustre-mod \
--disable-dxt-mod \
--without-mpi \
...
Then to profile an application,
LD_PRELOAD=$PWD/lib/libdarshan.so DARSHAN_ENABLE_NONMPI=1 ./path/to/my.exe
Do _not_ define `DARSHAN_ENABLE_NONMPI=1` globally, as this will create Darshan
logs for _everything_ including basic commands like `ls`.
......@@ -37,13 +37,47 @@ LIBS = -lz @LIBBZ2@
BUILD_HDF5_MODULE = @BUILD_HDF5_MODULE@
BUILD_MDHIM_MODULE = @BUILD_MDHIM_MODULE@
BUILD_POSIX_MODULE = @BUILD_POSIX_MODULE@
BUILD_MPIIO_MODULE = @BUILD_MPIIO_MODULE@
BUILD_PNETCDF_MODULE = @BUILD_PNETCDF_MODULE@
BUILD_STDIO_MODULE = @BUILD_STDIO_MODULE@
BUILD_DXT_MODULE = @BUILD_DXT_MODULE@
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-pnetcdf.o lib/darshan-stdio.o lib/darshan-dxt.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-pnetcdf.po lib/darshan-stdio.po lib/darshan-dxt.po
DARSHAN_STATIC_MOD_OBJS =
DARSHAN_DYNAMIC_MOD_OBJS =
DARSHAN_STUB_OBJS =
ifdef BUILD_POSIX_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-posix.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-posix.po
endif
ifdef BUILD_MPIIO_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-mpiio.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-mpiio.po
endif
ifdef BUILD_PNETCDF_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-pnetcdf.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-pnetcdf.po
DARSHAN_STUB_OBJS += lib/darshan-pnetcdf-stubs.o
endif
ifdef BUILD_STDIO_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-stdio.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-stdio.po
endif
ifdef BUILD_DXT_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-dxt.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-dxt.po
endif
ifdef BUILD_HDF5_MODULE
DARSHAN_STATIC_MOD_OBJS += lib/darshan-hdf5.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-hdf5.po
DARSHAN_STUB_OBJS += lib/darshan-hdf5-stubs.o
endif
ifdef DARSHAN_USE_BGQ
......@@ -67,6 +101,7 @@ CFLAGS += -DDARSHAN_MDHIM
CFLAGS_SHARED += -DDARSHAN_MDHIM
endif
lib::
@mkdir -p $@
......@@ -173,13 +208,8 @@ lib/libdarshan.a: lib/darshan-core-init-finalize.o lib/darshan-core.o lib/darsha
lib/libdarshan.so: lib/darshan-core-init-finalize.po lib/darshan-core.po lib/darshan-common.po $(DARSHAN_DYNAMIC_MOD_OBJS) lib/lookup3.po lib/lookup8.po
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ -lpthread -lrt -lz -ldl
ifdef BUILD_HDF5_MODULE
lib/libdarshan-stubs.a: lib/darshan-hdf5-stubs.o lib/darshan-pnetcdf-stubs.o
ar rcs $@ $^
else
lib/libdarshan-stubs.a: lib/darshan-pnetcdf-stubs.o
lib/libdarshan-stubs.a: $(DARSHAN_STUB_OBJS)
ar rcs $@ $^
endif
install:: all
......
This diff is collapsed.
......@@ -8,18 +8,28 @@ dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-runtime], [3.1.7])
AC_CONFIG_SRCDIR([darshan.h])
AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_MACRO_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-runtime-config.h)
AC_PROG_INSTALL
dnl Check for MPI
ENABLE_MPI=1
AC_ARG_WITH(mpi,
AS_HELP_STRING([--without-mpi], [Build without support for MPI]))
AS_IF([test "x$with_mpi" = "xno"],
ENABLE_MPI=,
[AC_DEFINE([HAVE_MPI], [], [Define if build is MPI-enabled])])
dnl Check to see if CC is an MPI compiler
dnl TODO: better error message here
AC_MSG_CHECKING(whether the mpicc compiler works)
AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
AC_MSG_RESULT(yes),
AC_MSG_RESULT(no)
AC_MSG_ERROR(CC doesn't appear to be a valid MPI compiler. See INSTALL document or try adding CC=mpicc to your configure command line.)
)
AS_IF([test "x$ENABLE_MPI" = "x1"],
[AC_MSG_CHECKING(whether the mpicc compiler works)
AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
AC_MSG_RESULT(yes),
AC_MSG_RESULT(no)
AC_MSG_ERROR(CC doesn't appear to be a valid MPI compiler. See INSTALL document or try adding CC=mpicc to your configure command line.))],
AC_MSG_NOTICE(not checking whether the mpicc compiler works))
dnl runtime libraries require zlib
CHECK_ZLIB
......@@ -46,6 +56,52 @@ AC_ARG_ENABLE(group-readable-logs,
fi]
,)
#
# Check for specific module enable/disable options
#
# POSIX module
BUILD_POSIX_MODULE=1
DARSHAN_POSIX_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-posix-ld-opts"
AC_ARG_ENABLE(posix-mod,
AS_HELP_STRING([--disable-posix-mod], [Disables compilation and use of POSIX module]))
AS_IF([test "x$enable_posix_mod" = "xno"],
BUILD_POSIX_MODULE=
DARSHAN_POSIX_LD_OPTS=)
# MPI-IO module
BUILD_MPIIO_MODULE=1
DARSHAN_MPIIO_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-mpiio-ld-opts"
AC_ARG_ENABLE(mpiio-mod,
AS_HELP_STRING([--disable-mpiio-mod], [Disables compilation and use of MPI-IO module (requires MPI)]))
AS_IF([test "x$enable_mpiio_mod" = "xno" || test "x$ENABLE_MPI" = "x"],
BUILD_MPIIO_MODULE=
DARSHAN_MPIIO_LD_OPTS=)
# STDIO module
BUILD_STDIO_MODULE=1
DARSHAN_STDIO_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-stdio-ld-opts"
AC_ARG_ENABLE(stdio-mod,
AS_HELP_STRING([--disable-stdio-mod], [Disables compilation and use of STDIO module]))
AS_IF([test "x$enable_stdio_mod" = "xno"],
BUILD_STDIO_MODULE=
DARSHAN_STDIO_LD_OPTS=)
# PNETCDF module
BUILD_PNETCDF_MODULE=1
DARSHAN_PNETCDF_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-pnetcdf-ld-opts"
AC_ARG_ENABLE(pnetcdf-mod,
AS_HELP_STRING([--disable-pnetcdf-mod], [Disables compilation and use of PnetCDF module (requires MPI)]))
AS_IF([test "x$enable_pnetcdf_mod" = "xno"],
BUILD_PNETCDF_MODULE=
DARSHAN_PNETCDF_LD_OPTS=)
# DXT module
BUILD_DXT_MODULE=1
AC_ARG_ENABLE(dxt-mod,
AS_HELP_STRING([--disable-dxt-mod], [Disables compilation and use of DXT module]))
AS_IF([test "x$enable_dxt_mod" = "xno"],
BUILD_DXT_MODULE=)
AC_ARG_ENABLE(
[bgq-mod],
[ --disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)],
......@@ -279,6 +335,7 @@ AC_ARG_ENABLE(HDF5-pre-1.10,
if test "x$BUILD_HDF5_MODULE" = "x1"; then
AC_MSG_ERROR([Cannot set both --enable-HDF5-post-1.10 and --enable-HDF5-pre-1.10])
fi
BUILD_HDF5_PRE110=1
BUILD_HDF5_MODULE=1
DARSHAN_HDF5_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-hdf5-ld-opts"
fi]
......@@ -299,20 +356,23 @@ AS_IF([test "x$enable_mdhim" = "xyes"],
)
#
# Begin tests for MPI-enabled builds
#
AS_IF([test "x$ENABLE_MPI" = "x1"], [
# determine if the MPI library includes MPI-IO functions or not
AC_MSG_CHECKING(for MPI-IO support in MPI)
AC_TRY_LINK([#include <mpi.h>], [
MPI_Comm comm;
char* filename;
int amode;
MPI_Info info;
MPI_File fh;
MPI_File_open(comm, filename, amode, info, &fh);
],
AC_MSG_RESULT(yes),
AC_MSG_ERROR(Darshan requires a version of MPI with MPI-IO support)
)
MPI_Comm comm;
char* filename;
int amode;
MPI_Info info;
MPI_File fh;
MPI_File_open(comm, filename, amode, info, &fh);],
AC_MSG_RESULT(yes),
AC_MSG_ERROR(Darshan requires a version of MPI with MPI-IO support))
# At some point MPI-IO converted most API functions to be const correct. We
# need to test for this to determine how to define MPI-IO wrappers in
......@@ -386,6 +446,11 @@ AS_IF([$CC -show foo.c -o foo >& /dev/null],
MPICH_LIB_OLD=0),
MPICH_LIB_OLD=0)
])
#
# End of MPI-only checks
#
DARSHAN_VERSION="AC_PACKAGE_VERSION"
AC_SUBST(darshan_lib_path)
AC_SUBST(darshan_share_path)
......@@ -399,6 +464,15 @@ AC_SUBST(DARSHAN_USE_BGQ)
AC_SUBST(DARSHAN_USE_LUSTRE)
AC_SUBST(BUILD_HDF5_MODULE)
AC_SUBST(DARSHAN_HDF5_LD_OPTS)
AC_SUBST(BUILD_POSIX_MODULE)
AC_SUBST(DARSHAN_POSIX_LD_OPTS)
AC_SUBST(BUILD_MPIIO_MODULE)
AC_SUBST(DARSHAN_MPIIO_LD_OPTS)
AC_SUBST(BUILD_PNETCDF_MODULE)
AC_SUBST(DARSHAN_PNETCDF_LD_OPTS)
AC_SUBST(BUILD_STDIO_MODULE)
AC_SUBST(DARSHAN_STDIO_LD_OPTS)
AC_SUBST(BUILD_DXT_MODULE)
AC_SUBST(BUILD_MDHIM_MODULE)
AC_SUBST(DARSHAN_MDHIM_LD_OPTS)
AC_OUTPUT(Makefile
......@@ -418,3 +492,22 @@ share/mpi-profile/darshan-bg-cxx.conf
share/mpi-profile/darshan-bg-f.conf
share/ld-opts/darshan-base-ld-opts
)
AC_MSG_NOTICE(Building Darshan with the following features:)
AS_IF([test "x$BUILD_POSIX_MODULE" = "x"], [AC_MSG_NOTICE(POSIX module support: no)], [AC_MSG_NOTICE(POSIX module support: yes)])
AS_IF([test "x$BUILD_MPIIO_MODULE" = "x"], [AC_MSG_NOTICE(MPIIO module support: no)], [AC_MSG_NOTICE(MPIIO module support: yes)])
AS_IF([test "x$BUILD_STDIO_MODULE" = "x"], [AC_MSG_NOTICE(STDIO module support: no)], [AC_MSG_NOTICE(STDIO module support: yes)])
AS_IF([test "x$BUILD_PNETCDF_MODULE" = "x"], [AC_MSG_NOTICE(PnetCDF module support: no)], [AC_MSG_NOTICE(PnetCDF module support: yes)])
AS_IF(
[test "x$BUILD_HDF5_MODULE" = "x"],
[AC_MSG_NOTICE(HDF5 module support: no)],
[AS_IF(
[test "x$BUILD_HDF5_PRE110" = "x1"],
[AC_MSG_NOTICE(HDF5 module support: 1.10+)],
[AC_MSG_NOTICE(HDF5 module support: pre-1.10)]
)],
)
AS_IF([test "x$BUILD_DXT_MODULE" = "x"], [AC_MSG_NOTICE(DXT module support: no)], [AC_MSG_NOTICE(DXT module support: yes)])
AS_IF([test "x$DARSHAN_USE_MDHIM" = "x"], [AC_MSG_NOTICE(MDHIM module support: no)], [AC_MSG_NOTICE(MDHIM module support: yes)])
AS_IF([test "x$DARSHAN_USE_LUSTRE" = "x"], [AC_MSG_NOTICE(Lustre module support: no)], [AC_MSG_NOTICE(Lustre module support: yes)])
AS_IF([test "x$DARSHAN_USE_BGQ" = "x"], [AC_MSG_NOTICE(BG/Q module support: no)], [AC_MSG_NOTICE(BG/Q module support: yes)])
......@@ -263,6 +263,7 @@ void darshan_common_val_counter(
int64_t *val_p,
int64_t *cnt_p);
#ifdef HAVE_MPI
/* darshan_variance_reduce()
*
* MPI reduction operation to calculate variances on counters in
......@@ -279,5 +280,6 @@ void darshan_variance_reduce(
void *inoutvec,
int *len,
MPI_Datatype *dt);
#endif
#endif /* __DARSHAN_COMMON_H */
......@@ -30,6 +30,9 @@
/* Environment variable to override memory per module */
#define DARSHAN_MOD_MEM_OVERRIDE "DARSHAN_MODMEM"
/* Environment variable to enable profiling without MPI */
#define DARSHAN_ENABLE_NONMPI "DARSHAN_ENABLE_NONMPI"
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
/* Environment variable to override default mmap log path */
#define DARSHAN_MMAP_LOG_PATH_OVERRIDE "DARSHAN_MMAP_LOGPATH"
......
......@@ -12,6 +12,7 @@
#define DARSHAN_EXTERN_DECL(name,ret,args) \
extern ret (*__real_ ## name)args;
#ifdef HAVE_MPI
DARSHAN_EXTERN_DECL(PMPI_File_close, int, (MPI_File *fh));
DARSHAN_EXTERN_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
DARSHAN_EXTERN_DECL(PMPI_File_iread, int, (MPI_File fh, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
......@@ -133,5 +134,6 @@ DARSHAN_EXTERN_DECL(PMPI_Gather, int, (void *sendbuf, int sendcount, MPI_Datatyp
DARSHAN_EXTERN_DECL(PMPI_Barrier, int, (MPI_Comm comm));
#endif
#endif /* #ifdef HAVE_MPI */
#endif /* __DARSHAN_DYNAMIC_H */
#ifndef __DARSHAN_MPI_H
#define __DARSHAN_MPI_H
#ifdef HAVE_MPI
#include <mpi.h>
struct darshan_mpi_file {
enum {
FH_TYPE_MPI,
FH_TYPE_POSIX
} fhtype;
union {
MPI_File mpi;
int posix;
} fh;
};
int darshan_mpi_comm_size(MPI_Comm comm, int *_nprocs);
int darshan_mpi_comm_rank(MPI_Comm comm, int *me);
double darshan_mpi_wtime();
int darshan_mpi_allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
int darshan_mpi_reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
int darshan_mpi_reduce_records(void *sendbuf, void *recvbuf, int count, int record_size, MPI_Op op, int root, MPI_Comm comm);
int darshan_mpi_scan(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
int darshan_mpi_gather(void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
int darshan_mpi_barrier(MPI_Comm comm);
int darshan_mpi_bcast(void *buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm);
int darshan_mpi_op_create(MPI_User_function *user_fn, int commute, MPI_Op *op);
int darshan_mpi_op_free(MPI_Op *op);
int darshan_mpi_file_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, struct darshan_mpi_file *fh);
int darshan_mpi_file_close(struct darshan_mpi_file *fh);
int darshan_mpi_file_write_at(struct darshan_mpi_file fh, MPI_Offset offset,
const void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
int darshan_mpi_file_write_at_all(struct darshan_mpi_file fh, MPI_Offset offset,
const void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
int darshan_mpi_info_create(MPI_Info *info);
int darshan_mpi_info_set(MPI_Info info, char *key, char *value);
int darshan_mpi_info_free(MPI_Info *info);
int darshan_mpi_type_contiguous(int count, MPI_Datatype oldtype, MPI_Datatype *newtype);
int darshan_mpi_type_commit(MPI_Datatype *datatype);
int darshan_mpi_type_free(MPI_Datatype *datatype);
#endif /* #ifdef HAVE_MPI */
#endif /* __DARSHAN_MPI_H */
......@@ -21,6 +21,9 @@
/* Define to 1 if you have the <mntent.h> header file. */
#undef HAVE_MNTENT_H
/* Define if build is MPI-enabled */
#undef HAVE_MPI
/* Define if MPI-IO prototypes use const qualifier */
#undef HAVE_MPIIO_CONST
......
......@@ -10,7 +10,9 @@
#include <unistd.h>
#include <sys/types.h>
#include <stdint.h>
#ifdef HAVE_MPI
#include <mpi.h>
#endif
#include "darshan-log-format.h"
#include "darshan-common.h"
......@@ -86,7 +88,7 @@
* 'shared_recs' array) into a single data record.
*/
typedef void (*darshan_module_shutdown)(
MPI_Comm mod_comm, /* MPI communicator to run collectives with */
void *mod_comm, /* MPI communicator to run collectives with */
darshan_record_id *shared_recs, /* list of shared data record ids */
int shared_rec_count, /* count of shared data records */
void **mod_buf, /* output parameter to save module buffer address */
......
......@@ -17,6 +17,7 @@
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-mpi.h"
#include <mpix.h>
#include <spi/include/kernel/location.h>
......@@ -196,13 +197,13 @@ static void bgq_shutdown(
{
bgq_runtime->record->base_rec.rank = -1;
PMPI_Comm_size(mod_comm, &nprocs);
darshan_mpi_comm_size(mod_comm, &nprocs);
ion_ids = malloc(sizeof(*ion_ids)*nprocs);
result = (ion_ids != NULL);
if(!result)
bgq_runtime->record->counters[BGQ_INODES] = -1;
}
PMPI_Bcast(&result, 1, MPI_INT, 0, mod_comm);
darshan_mpi_bcast(&result, 1, MPI_INT, 0, mod_comm);
/* caclulate the number of I/O nodes */
if (result)
......@@ -210,14 +211,14 @@ static void bgq_shutdown(
int i, found;
uint64_t val;
PMPI_Gather(&bgq_runtime->record->counters[BGQ_INODES],
1,
MPI_LONG_LONG_INT,
ion_ids,
1,
MPI_LONG_LONG_INT,
0,
mod_comm);
darshan_mpi_gather(&bgq_runtime->record->counters[BGQ_INODES],
1,
MPI_LONG_LONG_INT,
ion_ids,
1,
MPI_LONG_LONG_INT,
0,
mod_comm);
if (my_rank == 0)
{
qsort(ion_ids, nprocs, sizeof(*ion_ids), cmpr);
......
......@@ -278,6 +278,7 @@ void darshan_common_val_counter(void **common_val_root, int *common_val_count,
return;
}
#ifdef HAVE_MPI
void darshan_variance_reduce(void *invec, void *inoutvec, int *len,
MPI_Datatype *dt)
{
......@@ -298,6 +299,7 @@ void darshan_variance_reduce(void *invec, void *inoutvec, int *len,
return;
}
#endif
/*
* Local variables:
......
......@@ -11,12 +11,12 @@
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include "darshan.h"
#include "darshan-core.h"
#include "darshan-dynamic.h"
#ifdef HAVE_MPI
DARSHAN_FORWARD_DECL(PMPI_Finalize, int, ());
DARSHAN_FORWARD_DECL(PMPI_Init, int, (int *argc, char ***argv));
DARSHAN_FORWARD_DECL(PMPI_Init_thread, int, (int *argc, char ***argv, int required, int *provided));
......@@ -85,6 +85,30 @@ int DARSHAN_DECL(MPI_Finalize)(void)
return(ret);
}
DARSHAN_WRAPPER_MAP(PMPI_Finalize, int, (void), MPI_Finalize())
#endif /* #ifdef HAVE_MPI */
/*
* Initialization hook that does not rely on MPI
*/
#ifdef __GNUC__
__attribute__((constructor)) void serial_init(void)
{
char *no_mpi;
no_mpi = getenv(DARSHAN_ENABLE_NONMPI);
if (no_mpi)
darshan_core_initialize(0, NULL);
return;
}
__attribute__((destructor)) void serial_finalize(void)
{
char *no_mpi;
no_mpi = getenv(DARSHAN_ENABLE_NONMPI);
if (no_mpi)
darshan_core_shutdown();
return;
}
#endif
/*
* Local variables:
......
This diff is collapsed.
......@@ -23,6 +23,7 @@
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-mpi.h"
/* hope this doesn't change any time soon */
typedef int herr_t; //hf5-1.10.0p1: H5public.h:126
......@@ -412,7 +413,6 @@ static void hdf5_shutdown(
int hdf5_rec_count;
struct darshan_hdf5_file *red_send_buf = NULL;
struct darshan_hdf5_file *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
int i;
......@@ -457,19 +457,11 @@ static void hdf5_shutdown(
}
}
/* construct a datatype for a HDF5 file record. This is serving no purpose
* except to make sure we can do a reduction on proper boundaries
*/
PMPI_Type_contiguous(sizeof(struct darshan_hdf5_file),
MPI_BYTE, &red_type);
PMPI_Type_commit(&red_type);
/* register a HDF5 file record reduction operator */
PMPI_Op_create(hdf5_record_reduction_op, 1, &red_op);
darshan_mpi_op_create(hdf5_record_reduction_op, 1, &red_op);
/* reduce shared HDF5 file records */
PMPI_Reduce(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, mod_comm);
darshan_mpi_reduce_records(red_send_buf, red_recv_buf, shared_rec_count, sizeof(struct darshan_hdf5_file), red_op, 0, mod_comm);
/* clean up reduction state */
if(my_rank == 0)
......@@ -484,8 +476,7 @@ static void hdf5_shutdown(
hdf5_rec_count -= shared_rec_count;
}
PMPI_Type_free(&red_type);
PMPI_Op_free(&red_op);
darshan_mpi_op_free(&red_op);
}
/* update output buffer size to account for shared file reduction */
......
......@@ -498,7 +498,6 @@ static void mdhim_shutdown(
*(struct darshan_mdhim_record **)mdhim_buf;
struct darshan_mdhim_record *red_send_buf = NULL;
struct darshan_mdhim_record *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
int mdhim_rec_count;
......@@ -556,12 +555,10 @@ static void mdhim_shutdown(
return;
}
}
PMPI_Type_contiguous(MDHIM_RECORD_SIZE(nr_servers),
MPI_BYTE, &red_type);
PMPI_Type_commit(&red_type);
PMPI_Op_create(mdhim_record_reduction_op, 1, &red_op);
PMPI_Reduce(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, mod_comm);
darshan_mpi_op_create(mdhim_record_reduction_op, 1, &red_op);
darshan_mpi_reduce(red_send_buf, red_recv_buf, shared_rec_count,
MDHIM_RECORD_SIZE(nr_servers), red_op, 0, mod_comm);
if (my_rank == 0)
{
......@@ -571,8 +568,7 @@ static void mdhim_shutdown(
free(red_recv_buf);
}
PMPI_Type_free(&red_type);
PMPI_Op_free(&red_op);
darshan_mpi_op_free(&red_op);
/* drop shared records from non-root ranks or we'll end up writing too
* much */
if (my_rank != 0)
......
This diff is collapsed.
......@@ -87,7 +87,7 @@ static void null_cleanup_runtime(
/* forward declaration for NULL shutdown function needed to interface
* with darshan-core
*/
static void null_shutdown(MPI_Comm mod_comm, darshan_record_id *shared_recs,
static void null_shutdown(void *mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **null_buf, int *null_buf_sz);
/* null_runtime is the global data structure encapsulating "NULL" module state */
......@@ -305,7 +305,7 @@ static void null_cleanup_runtime()
* and shutdown/free internal data structures.
*/
static void null_shutdown(
MPI_Comm mod_comm,
void *mod_comm,
darshan_record_id *shared_recs,
int shared_rec_count,
void **null_buf,
......
......@@ -23,6 +23,7 @@
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-mpi.h"
DARSHAN_FORWARD_DECL(ncmpi_create, int, (MPI_Comm comm, const char *path, int cmode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_open, int, (MPI_Comm comm, const char *path, int omode, MPI_Info info, int *ncidp));
......@@ -366,7 +367,6 @@ static void pnetcdf_shutdown(
int pnetcdf_rec_count;
struct darshan_pnetcdf_file *red_send_buf = NULL;
struct darshan_pnetcdf_file *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
int i;
......@@ -412,19 +412,12 @@ static void pnetcdf_shutdown(
}
}
/* construct a datatype for a PNETCDF file record. This is serving no purpose
* except to make sure we can do a reduction on proper boundaries
*/
PMPI_Type_contiguous(sizeof(struct darshan_pnetcdf_file),
MPI_BYTE, &red_type);
PMPI_Type_commit(&red_type);
/* register a PNETCDF file record reduction operator */
PMPI_Op_create(pnetcdf_record_reduction_op, 1, &red_op);
darshan_mpi_op_create(pnetcdf_record_reduction_op, 1, &red_op);
/* reduce shared PNETCDF file records */
PMPI_Reduce(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, mod_comm);
darshan_mpi_reduce_records(red_send_buf, red_recv_buf, shared_rec_count,
sizeof(struct darshan_pnetcdf_file), red_op, 0, mod_comm);
/* clean up reduction state */
if(my_rank == 0)
......@@ -439,8 +432,7 @@ static void pnetcdf_shutdown(
pnetcdf_rec_count -= shared_rec_count;
}
PMPI_Type_free(&red_type);
PMPI_Op_free(&red_op);
darshan_mpi_op_free(&red_op);
}
/* update output buffer size to account for shared file reduction */
......
This diff is collapsed.
......@@ -83,6 +83,7 @@
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-mpi.h"
#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
......@@ -1150,7 +1151,6 @@ static void stdio_shutdown(
int i;
struct darshan_stdio_file *red_send_buf = NULL;
struct darshan_stdio_file *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
int stdio_rec_count;
double stdio_time;
......@@ -1220,19 +1220,12 @@ static void stdio_shutdown(
}
}
/* construct a datatype for a STDIO file record. This is serving no purpose
* except to make sure we can do a reduction on proper boundaries
*/
PMPI_Type_contiguous(sizeof(struct darshan_stdio_file),
MPI_BYTE, &red_type);
PMPI_Type_commit(&red_type);
/* register a STDIO file record reduction operator */
PMPI_Op_create(stdio_record_reduction_op, 1, &red_op);
darshan_mpi_op_create(stdio_record_reduction_op, 1, &red_op);
/* reduce shared STDIO file records */
PMPI_Reduce(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, mod_comm);
darshan_mpi_reduce_records(red_send_buf, red_recv_buf, shared_rec_count,
sizeof(struct darshan_stdio_file), red_op, 0, mod_comm);
/* get the time and byte variances for shared files */
stdio_shared_record_variance(mod_comm, red_send_buf, red_recv_buf,
......@@ -1251,8 +1244,7 @@ static void stdio_shutdown(
stdio_rec_count -= shared_rec_count;
}
PMPI_Type_free(&red_type);
PMPI_Op_free(&red_op);
darshan_mpi_op_free(&red_op);
}
/* filter out any records that have no activity on them; this is
......@@ -1364,17 +1356,13 @@ static void stdio_shared_record_variance(MPI_Comm mod_comm,
struct darshan_stdio_file *inrec_array, struct darshan_stdio_file *outrec_array,
int shared_rec_count)
{
MPI_Datatype var_dt;
MPI_Datatype var_dt = MPI_DATATYPE_NULL;
MPI_Op var_op;
int i;
struct darshan_variance_dt *var_send_buf = NULL;
struct darshan_variance_dt *var_recv_buf = NULL;
PMPI_Type_contiguous(sizeof(struct darshan_variance_dt),
MPI_BYTE, &var_dt);
PMPI_Type_commit(&var_dt);
PMPI_Op_create(darshan_variance_reduce, 1, &var_op);
darshan_mpi_op_create(darshan_variance_reduce, 1, &var_op);
var_send_buf = malloc(shared_rec_count * sizeof(struct darshan_variance_dt));
if(!var_send_buf)
......@@ -1399,8 +1387,17 @@ static void stdio_shared_record_variance(MPI_Comm mod_comm,
inrec_array[i].fcounters[STDIO_F_META_TIME];
}
PMPI_Reduce(var_send_buf, var_recv_buf, shared_rec_count,
var_dt, var_op, 0, mod_comm);
/* If MPI is enabled, use an MPI datatype. Otherwise var_dt will come back
* as invalid; then we just use the record size directly
*/
darshan_mpi_type_contiguous(sizeof(struct darshan_variance_dt), MPI_BYTE, &var_dt);
darshan_mpi_type_commit(&var_dt);
if (var_dt != MPI_DATATYPE_NULL)
darshan_mpi_reduce(var_send_buf, var_recv_buf, shared_rec_count,
var_dt, var_op, 0, mod_comm);
else
darshan_mpi_reduce_records(var_send_buf, var_recv_buf, shared_rec_count,
sizeof(struct darshan_variance_dt), var_op, 0, mod_comm);
if(my_rank == 0)
{
......@@ -1422,8 +1419,15 @@ static void stdio_shared_record_variance(MPI_Comm mod_comm,
inrec_array[i].counters[STDIO_BYTES_WRITTEN];
}
PMPI_Reduce(var_send_buf, var_recv_buf, shared_rec_count,
var_dt, var_op, 0, mod_comm);
/* If MPI is enabled, use an MPI datatype. Otherwise just use the record
* size directly
*/
if (var_dt != MPI_DATATYPE_NULL)
darshan_mpi_reduce(var_send_buf, var_recv_buf, shared_rec_count,
var_dt, var_op, 0, mod_comm);
else
darshan_mpi_reduce(var_send_buf, var_recv_buf, shared_rec_count,
sizeof(struct darshan_variance_dt), var_op, 0, mod_comm);
if(my_rank == 0)
{
......@@ -1434,8 +1438,8 @@ static void stdio_shared_record_variance(MPI_Comm mod_comm,
}
}
PMPI_Type_free(&var_dt);
PMPI_Op_free(&var_op);
darshan_mpi_type_free(&var_dt);
darshan_mpi_op_free(&var_op);
free(var_send_buf);
free(var_recv_buf);
......
......@@ -7,9 +7,9 @@
--wrap=PMPI_Init
--wrap=PMPI_Init_thread
--wrap=PMPI_Finalize
@@darshan_share_path@/ld-opts/darshan-posix-ld-opts
@@darshan_share_path@/ld-opts/darshan-pnetcdf-ld-opts
@@darshan_share_path@/ld-opts/darshan-stdio-ld-opts
@@darshan_share_path@/ld-opts/darshan-mpiio-ld-opts
@DARSHAN_POSIX_LD_OPTS@
@DARSHAN_MPIIO_LD_OPTS@
@DARSHAN_STDIO_LD_OPTS@
@DARSHAN_PNETCDF_LD_OPTS@
@DARSHAN_HDF5_LD_OPTS@
@DARSHAN_MDHIM_LD_OPTS@
This diff is collapsed.