Commit 146a6ccb authored by Shane Snyder's avatar Shane Snyder
Browse files

port hdf5 mod to recent darshan changes

parent 4fd1d6d3
...@@ -679,7 +679,6 @@ infodir ...@@ -679,7 +679,6 @@ infodir
docdir docdir
oldincludedir oldincludedir
includedir includedir
runstatedir
localstatedir localstatedir
sharedstatedir sharedstatedir
sysconfdir sysconfdir
...@@ -775,7 +774,6 @@ datadir='${datarootdir}' ...@@ -775,7 +774,6 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc' sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com' sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var' localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include' includedir='${prefix}/include'
oldincludedir='/usr/include' oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
...@@ -1028,15 +1026,6 @@ do ...@@ -1028,15 +1026,6 @@ do
| -silent | --silent | --silen | --sile | --sil) | -silent | --silent | --silen | --sile | --sil)
silent=yes ;; silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;; ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
...@@ -1174,7 +1163,7 @@ fi ...@@ -1174,7 +1163,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \ datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir runstatedir libdir localedir mandir
do do
eval ac_val=\$$ac_var eval ac_val=\$$ac_var
# Remove trailing slashes. # Remove trailing slashes.
...@@ -1327,7 +1316,6 @@ Fine tuning of the installation directories: ...@@ -1327,7 +1316,6 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var] --localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib] --libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include] --includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include] --oldincludedir=DIR C header files for non-gcc [/usr/include]
...@@ -4102,6 +4090,7 @@ fi ...@@ -4102,6 +4090,7 @@ fi
# HDF5 module # HDF5 module
BUILD_HDF5_MODULE= BUILD_HDF5_MODULE=
BUILD_HDF5_POST110=
DARSHAN_HDF5_LD_OPTS= DARSHAN_HDF5_LD_OPTS=
# see if user explicitly enabled support for an HDF5 API # see if user explicitly enabled support for an HDF5 API
...@@ -4115,6 +4104,7 @@ $as_echo "#define __DARSHAN_ENABLE_HDF5110 1" >>confdefs.h ...@@ -4115,6 +4104,7 @@ $as_echo "#define __DARSHAN_ENABLE_HDF5110 1" >>confdefs.h
as_fn_error $? "Cannot set both --enable-HDF5-post-1.10 and --enable-HDF5-pre-1.10" "$LINENO" 5 as_fn_error $? "Cannot set both --enable-HDF5-post-1.10 and --enable-HDF5-pre-1.10" "$LINENO" 5
fi fi
BUILD_HDF5_MODULE=1 BUILD_HDF5_MODULE=1
BUILD_HDF5_POST110=1
DARSHAN_HDF5_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-hdf5-ld-opts" DARSHAN_HDF5_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-hdf5-ld-opts"
fi fi
...@@ -6202,7 +6192,7 @@ fi ...@@ -6202,7 +6192,7 @@ fi
if test "x$BUILD_HDF5_MODULE" = "x"; then : if test "x$BUILD_HDF5_MODULE" = "x"; then :
{ $as_echo "$as_me:${as_lineno-$LINENO}: HDF5 module support: no" >&5 { $as_echo "$as_me:${as_lineno-$LINENO}: HDF5 module support: no" >&5
$as_echo "$as_me: HDF5 module support: no" >&6;} $as_echo "$as_me: HDF5 module support: no" >&6;}
elif if test "x$BUILD_HDF5_PRE110" = "x1"; then : elif if test "x$BUILD_HDF5_POST110" = "x1"; then :
{ $as_echo "$as_me:${as_lineno-$LINENO}: HDF5 module support: 1.10+" >&5 { $as_echo "$as_me:${as_lineno-$LINENO}: HDF5 module support: 1.10+" >&5
$as_echo "$as_me: HDF5 module support: 1.10+" >&6;} $as_echo "$as_me: HDF5 module support: 1.10+" >&6;}
else else
......
...@@ -211,6 +211,7 @@ AS_IF([test "x$enable_pnetcdf_mod" = "xno" || test "x$ENABLE_MPI" = "x"], ...@@ -211,6 +211,7 @@ AS_IF([test "x$enable_pnetcdf_mod" = "xno" || test "x$ENABLE_MPI" = "x"],
# HDF5 module # HDF5 module
BUILD_HDF5_MODULE= BUILD_HDF5_MODULE=
BUILD_HDF5_POST110=
DARSHAN_HDF5_LD_OPTS= DARSHAN_HDF5_LD_OPTS=
# see if user explicitly enabled support for an HDF5 API # see if user explicitly enabled support for an HDF5 API
...@@ -223,6 +224,7 @@ AC_ARG_ENABLE(HDF5-post-1.10, ...@@ -223,6 +224,7 @@ AC_ARG_ENABLE(HDF5-post-1.10,
AC_MSG_ERROR([Cannot set both --enable-HDF5-post-1.10 and --enable-HDF5-pre-1.10]) AC_MSG_ERROR([Cannot set both --enable-HDF5-post-1.10 and --enable-HDF5-pre-1.10])
fi fi
BUILD_HDF5_MODULE=1 BUILD_HDF5_MODULE=1
BUILD_HDF5_POST110=1
DARSHAN_HDF5_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-hdf5-ld-opts" DARSHAN_HDF5_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-hdf5-ld-opts"
fi] fi]
,) ,)
...@@ -535,7 +537,7 @@ AS_IF( ...@@ -535,7 +537,7 @@ AS_IF(
[test "x$BUILD_HDF5_MODULE" = "x"], [test "x$BUILD_HDF5_MODULE" = "x"],
[AC_MSG_NOTICE(HDF5 module support: no)], [AC_MSG_NOTICE(HDF5 module support: no)],
[AS_IF( [AS_IF(
[test "x$BUILD_HDF5_PRE110" = "x1"], [test "x$BUILD_HDF5_POST110" = "x1"],
[AC_MSG_NOTICE(HDF5 module support: 1.10+)], [AC_MSG_NOTICE(HDF5 module support: 1.10+)],
[AC_MSG_NOTICE(HDF5 module support: pre-1.10)] [AC_MSG_NOTICE(HDF5 module support: pre-1.10)]
)], )],
......
...@@ -60,14 +60,17 @@ static void hdf5_runtime_initialize( ...@@ -60,14 +60,17 @@ static void hdf5_runtime_initialize(
void); void);
static struct hdf5_file_record_ref *hdf5_track_new_file_record( static struct hdf5_file_record_ref *hdf5_track_new_file_record(
darshan_record_id rec_id, const char *path); darshan_record_id rec_id, const char *path);
static void hdf5_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void hdf5_cleanup_runtime( static void hdf5_cleanup_runtime(
void); void);
#ifdef HAVE_MPI
static void hdf5_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void hdf5_mpi_redux(
void *hdf5_buf, MPI_Comm mod_comm,
darshan_record_id *shared_recs, int shared_rec_count);
#endif
static void hdf5_shutdown( static void hdf5_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs, void **hdf5_buf, int *hdf5_buf_sz);
int shared_rec_count, void **hdf5_buf, int *hdf5_buf_sz);
static struct hdf5_runtime *hdf5_runtime = NULL; static struct hdf5_runtime *hdf5_runtime = NULL;
static pthread_mutex_t hdf5_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; static pthread_mutex_t hdf5_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
...@@ -257,6 +260,12 @@ herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id) ...@@ -257,6 +260,12 @@ herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
static void hdf5_runtime_initialize() static void hdf5_runtime_initialize()
{ {
int hdf5_buf_size; int hdf5_buf_size;
darshan_module_funcs mod_funcs = {
#ifdef HAVE_MPI
.mod_redux_func = &hdf5_mpi_redux,
#endif
.mod_shutdown_func = &hdf5_shutdown
};
/* try and store the default number of records for this module */ /* try and store the default number of records for this module */
hdf5_buf_size = DARSHAN_DEF_MOD_REC_COUNT * sizeof(struct darshan_hdf5_file); hdf5_buf_size = DARSHAN_DEF_MOD_REC_COUNT * sizeof(struct darshan_hdf5_file);
...@@ -264,7 +273,7 @@ static void hdf5_runtime_initialize() ...@@ -264,7 +273,7 @@ static void hdf5_runtime_initialize()
/* register hdf5 module with darshan-core */ /* register hdf5 module with darshan-core */
darshan_core_register_module( darshan_core_register_module(
DARSHAN_HDF5_MOD, DARSHAN_HDF5_MOD,
&hdf5_shutdown, mod_funcs,
&hdf5_buf_size, &hdf5_buf_size,
&my_rank, &my_rank,
NULL); NULL);
...@@ -335,6 +344,18 @@ static struct hdf5_file_record_ref *hdf5_track_new_file_record( ...@@ -335,6 +344,18 @@ static struct hdf5_file_record_ref *hdf5_track_new_file_record(
return(rec_ref); return(rec_ref);
} }
static void hdf5_cleanup_runtime()
{
darshan_clear_record_refs(&(hdf5_runtime->hid_hash), 0);
darshan_clear_record_refs(&(hdf5_runtime->rec_id_hash), 1);
free(hdf5_runtime);
hdf5_runtime = NULL;
return;
}
#ifdef HAVE_MPI
static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v, static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype) int *len, MPI_Datatype *datatype)
{ {
...@@ -384,32 +405,22 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v, ...@@ -384,32 +405,22 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
return; return;
} }
#endif
static void hdf5_cleanup_runtime()
{
darshan_clear_record_refs(&(hdf5_runtime->hid_hash), 0);
darshan_clear_record_refs(&(hdf5_runtime->rec_id_hash), 1);
free(hdf5_runtime);
hdf5_runtime = NULL;
return;
}
/************************************************************************ /************************************************************************
* Functions exported by HDF5 module for coordinating with darshan-core * * Functions exported by HDF5 module for coordinating with darshan-core *
************************************************************************/ ************************************************************************/
static void hdf5_shutdown( #ifdef HAVE_MPI
static void hdf5_mpi_redux(
void *hdf5_buf,
MPI_Comm mod_comm, MPI_Comm mod_comm,
darshan_record_id *shared_recs, darshan_record_id *shared_recs,
int shared_rec_count, int shared_rec_count)
void **hdf5_buf,
int *hdf5_buf_sz)
{ {
int hdf5_rec_count;
struct hdf5_file_record_ref *rec_ref; struct hdf5_file_record_ref *rec_ref;
struct darshan_hdf5_file *hdf5_rec_buf = *(struct darshan_hdf5_file **)hdf5_buf; struct darshan_hdf5_file *hdf5_rec_buf = *(struct darshan_hdf5_file **)hdf5_buf;
int hdf5_rec_count;
struct darshan_hdf5_file *red_send_buf = NULL; struct darshan_hdf5_file *red_send_buf = NULL;
struct darshan_hdf5_file *red_recv_buf = NULL; struct darshan_hdf5_file *red_recv_buf = NULL;
MPI_Datatype red_type; MPI_Datatype red_type;
...@@ -419,81 +430,88 @@ static void hdf5_shutdown( ...@@ -419,81 +430,88 @@ static void hdf5_shutdown(
HDF5_LOCK(); HDF5_LOCK();
assert(hdf5_runtime); assert(hdf5_runtime);
hdf5_rec_count = hdf5_runtime->file_rec_count; /* necessary initialization of shared records */
for(i = 0; i < shared_rec_count; i++)
{
rec_ref = darshan_lookup_record_ref(hdf5_runtime->rec_id_hash,
&shared_recs[i], sizeof(darshan_record_id));
assert(rec_ref);
rec_ref->file_rec->base_rec.rank = -1;
}
/* if there are globally shared files, do a shared file reduction */ /* sort the array of records so we get all of the shared records
/* NOTE: the shared file reduction is also skipped if the * (marked by rank -1) in a contiguous portion at end of the array
* DARSHAN_DISABLE_SHARED_REDUCTION environment variable is set.
*/ */
if(shared_rec_count && !getenv("DARSHAN_DISABLE_SHARED_REDUCTION")) darshan_record_sort(hdf5_rec_buf, hdf5_rec_count,
sizeof(struct darshan_hdf5_file));
/* make *send_buf point to the shared files at the end of sorted array */
red_send_buf = &(hdf5_rec_buf[hdf5_rec_count-shared_rec_count]);
/* allocate memory for the reduction output on rank 0 */
if(my_rank == 0)
{ {
/* necessary initialization of shared records */ red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_hdf5_file));
for(i = 0; i < shared_rec_count; i++) if(!red_recv_buf)
{ {
rec_ref = darshan_lookup_record_ref(hdf5_runtime->rec_id_hash, HDF5_UNLOCK();
&shared_recs[i], sizeof(darshan_record_id)); return;
assert(rec_ref);
rec_ref->file_rec->base_rec.rank = -1;
} }
}
/* sort the array of records so we get all of the shared records /* construct a datatype for a HDF5 file record. This is serving no purpose
* (marked by rank -1) in a contiguous portion at end of the array * except to make sure we can do a reduction on proper boundaries
*/ */
darshan_record_sort(hdf5_rec_buf, hdf5_rec_count, PMPI_Type_contiguous(sizeof(struct darshan_hdf5_file),
sizeof(struct darshan_hdf5_file)); MPI_BYTE, &red_type);
PMPI_Type_commit(&red_type);
/* make *send_buf point to the shared files at the end of sorted array */ /* register a HDF5 file record reduction operator */
red_send_buf = &(hdf5_rec_buf[hdf5_rec_count-shared_rec_count]); PMPI_Op_create(hdf5_record_reduction_op, 1, &red_op);
/* allocate memory for the reduction output on rank 0 */ /* reduce shared HDF5 file records */
if(my_rank == 0) PMPI_Reduce(red_send_buf, red_recv_buf,
{ shared_rec_count, red_type, red_op, 0, mod_comm);
red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_hdf5_file));
if(!red_recv_buf)
{
HDF5_UNLOCK();
return;
}
}
/* construct a datatype for a HDF5 file record. This is serving no purpose /* clean up reduction state */
* except to make sure we can do a reduction on proper boundaries if(my_rank == 0)
*/ {
PMPI_Type_contiguous(sizeof(struct darshan_hdf5_file), int tmp_ndx = hdf5_rec_count - shared_rec_count;
MPI_BYTE, &red_type); memcpy(&(hdf5_rec_buf[tmp_ndx]), red_recv_buf,
PMPI_Type_commit(&red_type); shared_rec_count * sizeof(struct darshan_hdf5_file));
free(red_recv_buf);
}
else
{
hdf5_rec_count -= shared_rec_count;
}
/* register a HDF5 file record reduction operator */ PMPI_Type_free(&red_type);
PMPI_Op_create(hdf5_record_reduction_op, 1, &red_op); PMPI_Op_free(&red_op);
/* reduce shared HDF5 file records */ HDF5_UNLOCK();
PMPI_Reduce(red_send_buf, red_recv_buf, return;
shared_rec_count, red_type, red_op, 0, mod_comm); }
#endif
/* clean up reduction state */ static void hdf5_shutdown(
if(my_rank == 0) void **hdf5_buf,
{ int *hdf5_buf_sz)
int tmp_ndx = hdf5_rec_count - shared_rec_count; {
memcpy(&(hdf5_rec_buf[tmp_ndx]), red_recv_buf, int hdf5_rec_count;
shared_rec_count * sizeof(struct darshan_hdf5_file));
free(red_recv_buf);
}
else
{
hdf5_rec_count -= shared_rec_count;
}
PMPI_Type_free(&red_type); HDF5_LOCK();
PMPI_Op_free(&red_op); assert(hdf5_runtime);
}
/* update output buffer size to account for shared file reduction */ hdf5_rec_count = hdf5_runtime->file_rec_count;
*hdf5_buf_sz = hdf5_rec_count * sizeof(struct darshan_hdf5_file);
/* shutdown internal structures used for instrumenting */ /* shutdown internal structures used for instrumenting */
hdf5_cleanup_runtime(); hdf5_cleanup_runtime();
/* update output buffer size to account for shared file reduction */
*hdf5_buf_sz = hdf5_rec_count * sizeof(struct darshan_hdf5_file);
HDF5_UNLOCK(); HDF5_UNLOCK();
return; return;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment