Commit 1a67e237 authored by Philip Carns's avatar Philip Carns

Merge remote-tracking branch 'origin/master' into dev-stdio

- note: darshan-stdio module updates are still needed; this merge just
  updates the rest of the code and resolves conflicts
parents 9cd5e5c0 eca097a2
...@@ -2,6 +2,27 @@ ...@@ -2,6 +2,27 @@
Darshan Release Change Log Darshan Release Change Log
-------------------------- --------------------------
Darshan-3.0.1
=============
* bug fix in darshan logutil mount parsing code that was
causing file paths to be matched to the first mount point
with a common prefix rather than the one with the longest
common prefix
* bug fix in the darshan-util bzip2 configure check that
was accidentally overwriting Darshan's LDFLAGS
* minor bug fixes to IO start time counters in all modules
to set IO start time to the actual first start time rather
than the first IO op to complete
* update darshan-util perl scripts to get perl bin from
user's path, rather than from /usr/lib (reported by
Kay Thust)
* update Darshan's fortran and cxx compiler wrapper generators
to automatically detect MPICH library names on BG/Q
* fix bug that was calculating Darshan's agg_perf_by_slowest
performance metric incorrectly
* add performance estimate info to darshan-job-summary
graphs
Darshan-3.0.0 Darshan-3.0.0
============= =============
* install darshan-null-log-format.h header when installing * install darshan-null-log-format.h header when installing
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
#define DARSHAN_BGQ_VER 1 #define DARSHAN_BGQ_VER 1
#define BGQ_COUNTERS \ #define BGQ_COUNTERS \
/* control system jobid*/\ /* control system jobid */\
X(BGQ_CSJOBID) \ X(BGQ_CSJOBID) \
/* number of BGQ compute nodes */\ /* number of BGQ compute nodes */\
X(BGQ_NNODES) \ X(BGQ_NNODES) \
...@@ -60,16 +60,13 @@ enum darshan_bgq_f_indices ...@@ -60,16 +60,13 @@ enum darshan_bgq_f_indices
* which would actually be logged to file by Darshan for the "BGQ" example * which would actually be logged to file by Darshan for the "BGQ" example
* module. This example implementation logs the following data for each * module. This example implementation logs the following data for each
* record: * record:
* - a corresponding Darshan record identifier * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process responsible for the record
* - integer I/O counters (operation counts, I/O sizes, etc.) * - integer I/O counters (operation counts, I/O sizes, etc.)
* - floating point I/O counters (timestamps, cumulative timers, etc.) * - floating point I/O counters (timestamps, cumulative timers, etc.)
*/ */
struct darshan_bgq_record struct darshan_bgq_record
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int alignment;
int64_t counters[BGQ_NUM_INDICES]; int64_t counters[BGQ_NUM_INDICES];
double fcounters[BGQ_F_NUM_INDICES]; double fcounters[BGQ_F_NUM_INDICES];
}; };
......
...@@ -41,15 +41,13 @@ enum darshan_hdf5_f_indices ...@@ -41,15 +41,13 @@ enum darshan_hdf5_f_indices
/* file record structure for HDF5 files. a record is created and stored for /* file record structure for HDF5 files. a record is created and stored for
* every HDF5 file opened by the original application. For the HDF5 module, * every HDF5 file opened by the original application. For the HDF5 module,
* the record includes: * the record includes:
* - a corresponding record identifier (created by hashing the file path) * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (open, read/write counts, etc) * - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.) * - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/ */
struct darshan_hdf5_file struct darshan_hdf5_file
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int64_t counters[HDF5_NUM_INDICES]; int64_t counters[HDF5_NUM_INDICES];
double fcounters[HDF5_F_NUM_INDICES]; double fcounters[HDF5_F_NUM_INDICES];
}; };
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#endif #endif
/* update this on file format changes */ /* update this on file format changes */
#define DARSHAN_LOG_VERSION "3.00" #define DARSHAN_LOG_VERSION "3.01"
/* magic number for validating output files and checking byte order */ /* magic number for validating output files and checking byte order */
#define DARSHAN_MAGIC_NR 6567223 #define DARSHAN_MAGIC_NR 6567223
...@@ -44,6 +44,7 @@ enum darshan_comp_type ...@@ -44,6 +44,7 @@ enum darshan_comp_type
{ {
DARSHAN_ZLIB_COMP, DARSHAN_ZLIB_COMP,
DARSHAN_BZIP2_COMP, DARSHAN_BZIP2_COMP,
DARSHAN_NO_COMP,
}; };
typedef uint64_t darshan_record_id; typedef uint64_t darshan_record_id;
...@@ -68,7 +69,7 @@ struct darshan_header ...@@ -68,7 +69,7 @@ struct darshan_header
int64_t magic_nr; int64_t magic_nr;
unsigned char comp_type; unsigned char comp_type;
uint32_t partial_flag; uint32_t partial_flag;
struct darshan_log_map rec_map; struct darshan_log_map name_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS]; struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
uint32_t mod_ver[DARSHAN_MAX_MODS]; uint32_t mod_ver[DARSHAN_MAX_MODS];
}; };
...@@ -85,11 +86,18 @@ struct darshan_job ...@@ -85,11 +86,18 @@ struct darshan_job
char metadata[DARSHAN_JOB_METADATA_LEN]; char metadata[DARSHAN_JOB_METADATA_LEN];
}; };
/* minimal record stored for each file/object accessed by Darshan */ /* record to store name->darshan_id mapping for each registered record */
struct darshan_record struct darshan_name_record
{ {
char* name;
darshan_record_id id; darshan_record_id id;
char name[1];
};
/* base record definition that can be used by modules */
struct darshan_base_record
{
darshan_record_id id;
int64_t rank;
}; };
......
...@@ -140,15 +140,13 @@ enum darshan_mpiio_f_indices ...@@ -140,15 +140,13 @@ enum darshan_mpiio_f_indices
/* file record structure for MPI-IO files. a record is created and stored for /* file record structure for MPI-IO files. a record is created and stored for
* every MPI-IO file opened by the original application. For the MPI-IO module, * every MPI-IO file opened by the original application. For the MPI-IO module,
* the record includes: * the record includes:
* - a corresponding record identifier (created by hashing the file path) * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (open, read/write counts, etc) * - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.) * - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/ */
struct darshan_mpiio_file struct darshan_mpiio_file
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int64_t counters[MPIIO_NUM_INDICES]; int64_t counters[MPIIO_NUM_INDICES];
double fcounters[MPIIO_F_NUM_INDICES]; double fcounters[MPIIO_F_NUM_INDICES];
}; };
......
...@@ -44,15 +44,13 @@ enum darshan_null_f_indices ...@@ -44,15 +44,13 @@ enum darshan_null_f_indices
* which would actually be logged to file by Darshan for the "NULL" example * which would actually be logged to file by Darshan for the "NULL" example
* module. This example implementation logs the following data for each * module. This example implementation logs the following data for each
* record: * record:
* - a corresponding Darshan record identifier * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process responsible for the record
* - integer I/O counters (operation counts, I/O sizes, etc.) * - integer I/O counters (operation counts, I/O sizes, etc.)
* - floating point I/O counters (timestamps, cumulative timers, etc.) * - floating point I/O counters (timestamps, cumulative timers, etc.)
*/ */
struct darshan_null_record struct darshan_null_record
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int64_t counters[NULL_NUM_INDICES]; int64_t counters[NULL_NUM_INDICES];
double fcounters[NULL_F_NUM_INDICES]; double fcounters[NULL_F_NUM_INDICES];
}; };
......
...@@ -43,15 +43,13 @@ enum darshan_pnetcdf_f_indices ...@@ -43,15 +43,13 @@ enum darshan_pnetcdf_f_indices
/* file record structure for PNETCDF files. a record is created and stored for /* file record structure for PNETCDF files. a record is created and stored for
* every PNETCDF file opened by the original application. For the PNETCDF module, * every PNETCDF file opened by the original application. For the PNETCDF module,
* the record includes: * the record includes:
* - a corresponding record identifier (created by hashing the file path) * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (open, read/write counts, etc) * - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.) * - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/ */
struct darshan_pnetcdf_file struct darshan_pnetcdf_file
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int64_t counters[PNETCDF_NUM_INDICES]; int64_t counters[PNETCDF_NUM_INDICES];
double fcounters[PNETCDF_F_NUM_INDICES]; double fcounters[PNETCDF_F_NUM_INDICES];
}; };
......
...@@ -164,15 +164,13 @@ enum darshan_posix_f_indices ...@@ -164,15 +164,13 @@ enum darshan_posix_f_indices
/* file record structure for POSIX files. a record is created and stored for /* file record structure for POSIX files. a record is created and stored for
* every POSIX file opened by the original application. For the POSIX module, * every POSIX file opened by the original application. For the POSIX module,
* the record includes: * the record includes:
* - a corresponding record identifier (created by hashing the file path) * - a darshan_base_record structure, which contains the record id & rank
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (open, read/write counts, etc) * - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.) * - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/ */
struct darshan_posix_file struct darshan_posix_file
{ {
darshan_record_id f_id; struct darshan_base_record base_rec;
int64_t rank;
int64_t counters[POSIX_NUM_INDICES]; int64_t counters[POSIX_NUM_INDICES];
double fcounters[POSIX_F_NUM_INDICES]; double fcounters[POSIX_F_NUM_INDICES];
}; };
......
...@@ -17,6 +17,7 @@ CC = @CC@ ...@@ -17,6 +17,7 @@ CC = @CC@
LD = @LD@ LD = @LD@
DISABLE_LDPRELOAD = @DISABLE_LDPRELOAD@ DISABLE_LDPRELOAD = @DISABLE_LDPRELOAD@
ENABLE_MMAP_LOGS = @ENABLE_MMAP_LOGS@
DARSHAN_USE_BGQ = @DARSHAN_USE_BGQ@ DARSHAN_USE_BGQ = @DARSHAN_USE_BGQ@
DARSHAN_LOG_FORMAT = $(srcdir)/../darshan-log-format.h DARSHAN_LOG_FORMAT = $(srcdir)/../darshan-log-format.h
DARSHAN_VERSION = @DARSHAN_VERSION@ DARSHAN_VERSION = @DARSHAN_VERSION@
...@@ -164,6 +165,9 @@ endif ...@@ -164,6 +165,9 @@ endif
install -m 644 $(srcdir)/share/ld-opts/darshan-hdf5-ld-opts $(datarootdir)/ld-opts/darshan-hdf5-ld-opts install -m 644 $(srcdir)/share/ld-opts/darshan-hdf5-ld-opts $(datarootdir)/ld-opts/darshan-hdf5-ld-opts
install -m 644 $(srcdir)/share/ld-opts/darshan-pnetcdf-ld-opts $(datarootdir)/ld-opts/darshan-pnetcdf-ld-opts install -m 644 $(srcdir)/share/ld-opts/darshan-pnetcdf-ld-opts $(datarootdir)/ld-opts/darshan-pnetcdf-ld-opts
install -m 644 $(srcdir)/share/ld-opts/darshan-stdio-ld-opts $(datarootdir)/ld-opts/darshan-stdio-ld-opts install -m 644 $(srcdir)/share/ld-opts/darshan-stdio-ld-opts $(datarootdir)/ld-opts/darshan-stdio-ld-opts
ifdef ENABLE_MMAP_LOGS
install -m 755 share/darshan-mmap-epilog.sh $(datarootdir)/darshan-mmap-epilog.sh
endif
install -d $(libdir)/pkgconfig install -d $(libdir)/pkgconfig
install -m 644 lib/pkgconfig/darshan-runtime.pc $(libdir)/pkgconfig/darshan-runtime.pc install -m 644 lib/pkgconfig/darshan-runtime.pc $(libdir)/pkgconfig/darshan-runtime.pc
...@@ -171,5 +175,5 @@ clean:: ...@@ -171,5 +175,5 @@ clean::
rm -f *.o *.a lib/*.o lib/*.po lib/*.a lib/*.so rm -f *.o *.a lib/*.o lib/*.po lib/*.a lib/*.so
distclean:: clean distclean:: clean
rm -f darshan-runtime-config.h darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-gen-cc.pl darshan-mk-log-dirs.pl darshan-config darshan-base-ld-opts aclocal.m4 autom4te.cache/* config.status config.log Makefile rm -f darshan-runtime-config.h darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-gen-cc.pl darshan-mk-log-dirs.pl darshan-config lib/pkgconfig/darshan-runtime.pc share/craype-1.x/darshan-module share/craype-2.x/darshan-module share/darshan-mmap-epilog.sh share/ld-opts/darshan-base-ld-opts share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf aclocal.m4 autom4te.cache/* config.status config.log Makefile
rm -rf autom4te.cache rm -rf autom4te.cache
This diff is collapsed.
...@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script. ...@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl You may need to use autoheader as well if changing any DEFINEs dnl You may need to use autoheader as well if changing any DEFINEs
dnl sanity checks, output header, location of scripts used here dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-runtime], [3.0.0]) AC_INIT([darshan-runtime], [3.0.1])
AC_CONFIG_SRCDIR([darshan.h]) AC_CONFIG_SRCDIR([darshan.h])
AC_CONFIG_AUX_DIR(../maint/config) AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-runtime-config.h) AC_CONFIG_HEADER(darshan-runtime-config.h)
...@@ -24,27 +24,44 @@ AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)], ...@@ -24,27 +24,44 @@ AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
dnl runtime libraries require zlib dnl runtime libraries require zlib
CHECK_ZLIB CHECK_ZLIB
AC_ARG_ENABLE(cuserid, AC_ARG_ENABLE(ld-preload,
[ --disable-cuserid Disables attempted use of cuserid() at run time], [ --disable-ld-preload Disables support for LD_PRELOAD library],
[if test "x$enableval" = "xno" ; then [if test "x$enableval" = "xno" ; then
AC_DEFINE(DARSHAN_DISABLE_CUSERID, 1, Define if cuserid() should be disabled) DISABLE_LDPRELOAD="1"
fi] fi]
,) ,)
AC_ARG_ENABLE(ld-preload, AC_ARG_ENABLE(cuserid,
[ --disable-ld-preload Disables support for LD_PRELOAD library], [ --disable-cuserid Disables attempted use of cuserid() at run time],
[if test "x$enableval" = "xno" ; then [if test "x$enableval" = "xno" ; then
DISABLE_LDPRELOAD="1" AC_DEFINE(__DARSHAN_DISABLE_CUSERID, 1, Define if cuserid() should be disabled)
fi] fi]
,) ,)
AC_ARG_ENABLE(group-readable-logs, AC_ARG_ENABLE(group-readable-logs,
[ --enable-group-readable-logs Set log files to be group readable], [ --enable-group-readable-logs
Set log files to be group readable],
[if test "x$enableval" = "xyes" ; then [if test "x$enableval" = "xyes" ; then
AC_DEFINE(__DARSHAN_GROUP_READABLE_LOGS, 1, Define if Darshan should set log files to be group readable) AC_DEFINE(__DARSHAN_GROUP_READABLE_LOGS, 1, Define if Darshan should set log files to be group readable)
fi] fi]
,) ,)
AC_ARG_ENABLE(
[bgq_mod],
[ --disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)],
[],
[enable_bgq_mod=check]
)
AC_ARG_ENABLE(mmap-logs,
[ --enable-mmmap-logs Enables ability to mmap I/O data to log file],
[if test "x$enableval" = "xyes" ; then
AC_DEFINE(__DARSHAN_ENABLE_MMAP_LOGS, 1, Define if Darshan should mmap data structures to log file)
ENABLE_MMAP_LOGS=1
AC_CONFIG_FILES(share/darshan-mmap-epilog.sh)
fi]
,)
AC_ARG_WITH(mem-align, AC_ARG_WITH(mem-align,
[ --with-mem-align=<num> Memory alignment in bytes], [ --with-mem-align=<num> Memory alignment in bytes],
if test x$withval = xyes; then if test x$withval = xyes; then
...@@ -56,7 +73,9 @@ AC_ARG_WITH(mem-align, ...@@ -56,7 +73,9 @@ AC_ARG_WITH(mem-align,
) )
AC_ARG_WITH(log-path-by-env, AC_ARG_WITH(log-path-by-env,
[ --with-log-path-by-env=<env var list> Comma separated list of environment variables to check for log path location before --with-log-path path], [ --with-log-path-by-env=<env var list>
Comma separated list of environment variables to check for
log path location before --with-log-path path],
if test x$withval = xyes; then if test x$withval = xyes; then
AC_MSG_ERROR(--with-log-path-by-env must be given at least one variable name) AC_MSG_ERROR(--with-log-path-by-env must be given at least one variable name)
else else
...@@ -67,7 +86,8 @@ AC_ARG_WITH(log-path-by-env, ...@@ -67,7 +86,8 @@ AC_ARG_WITH(log-path-by-env,
) )
AC_ARG_WITH(log-hints, AC_ARG_WITH(log-hints,
[ --with-log-hints=<hint1=x;hint2=y> Semicolon-separated list of MPI-IO hints for log file write], [ --with-log-hints=<hint1=x;hint2=y>
Semicolon-separated list of MPI-IO hints for log file write],
if test x$withval = xyes; then if test x$withval = xyes; then
AC_MSG_ERROR(--with-log-hints must be given an argument) AC_MSG_ERROR(--with-log-hints must be given an argument)
else else
...@@ -90,8 +110,8 @@ AC_ARG_WITH(log-path, ...@@ -90,8 +110,8 @@ AC_ARG_WITH(log-path,
AC_ARG_WITH(jobid-env, AC_ARG_WITH(jobid-env,
[ --with-jobid-env=<name> Name of environment variable that stores the jobid [ --with-jobid-env=<name> Name of environment variable that stores the jobid
(specify "NONE" if no appropriate environment variable is available: (specify "NONE" if no appropriate environment variable
Darshan will use rank 0's pid instead)], is available: Darshan will use rank 0's pid instead)],
if test x$withval = xyes; then if test x$withval = xyes; then
AC_MSG_ERROR(--with-jobid-env must be given a name) AC_MSG_ERROR(--with-jobid-env must be given a name)
else else
...@@ -101,6 +121,30 @@ AC_ARG_WITH(jobid-env, ...@@ -101,6 +121,30 @@ AC_ARG_WITH(jobid-env,
fi fi
) )
AC_ARG_WITH(mod-mem,
[ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module],
if test x$withval = xyes; then
AC_MSG_ERROR(--with-mod-mem must be given a number)
else
AC_DEFINE_UNQUOTED(__DARSHAN_MOD_MEM_MAX, ${withval}, Maximum memory (in MiB) for each Darshan module)
fi
)
if test x$enable_bgq_mod != xno; then
AC_MSG_CHECKING(for BG/Q environment)
AC_TRY_COMPILE([
#ifndef __bgq__
#error __bgq__ not set
#endif
], [],
AC_MSG_RESULT(yes)
DARSHAN_USE_BGQ=1,
AC_MSG_RESULT(no)
AS_IF([test "x$enable_bgq_mod" = xyes],
AC_MSG_ERROR(BG/Q module enabled in non-BG/Q environment),
[]))
fi
if test x$GOT_ALIGNMENT != x1; then if test x$GOT_ALIGNMENT != x1; then
AC_MSG_ERROR(must provide --with-mem-align=<num> argument to configure.) AC_MSG_ERROR(must provide --with-mem-align=<num> argument to configure.)
fi fi
...@@ -280,43 +324,13 @@ AS_IF([$CC -show foo.c -o foo >& /dev/null], ...@@ -280,43 +324,13 @@ AS_IF([$CC -show foo.c -o foo >& /dev/null],
MPICH_LIB_OLD=0), MPICH_LIB_OLD=0),
MPICH_LIB_OLD=0) MPICH_LIB_OLD=0)
# check to see whether the bgq instrumentation module should be built
AC_ARG_ENABLE(
[bgq_mod],
[ --disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)],
[],
[enable_bgq_mod=check]
)
if test x$enable_bgq_mod != xno; then
AC_MSG_CHECKING(for BG/Q environment)
AC_TRY_COMPILE([
#ifndef __bgq__
#error __bgq__ not set
#endif
], [],
AC_MSG_RESULT(yes)
DARSHAN_USE_BGQ=1,
AC_MSG_RESULT(no)
AS_IF([test "x$enable_bgq_mod" = xyes],
AC_MSG_ERROR(BG/Q module enabled in non-BG/Q environment),
[]))
fi
AC_ARG_WITH(mod-mem,
[ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module],
if test x$withval = xyes; then
AC_MSG_ERROR(--with-mod-mem must be given a number)
else
AC_DEFINE_UNQUOTED(__DARSHAN_MOD_MEM_MAX, ${withval}, Maximum memory (in MiB) for each Darshan module)
fi
)
DARSHAN_VERSION="AC_PACKAGE_VERSION" DARSHAN_VERSION="AC_PACKAGE_VERSION"
AC_SUBST(darshan_lib_path) AC_SUBST(darshan_lib_path)
AC_SUBST(darshan_share_path) AC_SUBST(darshan_share_path)
AC_SUBST(LDFLAGS) AC_SUBST(LDFLAGS)
AC_SUBST(__DARSHAN_LOG_PATH) AC_SUBST(__DARSHAN_LOG_PATH)
AC_SUBST(DISABLE_LDPRELOAD) AC_SUBST(DISABLE_LDPRELOAD)
AC_SUBST(ENABLE_MMAP_LOGS)
AC_SUBST(DARSHAN_VERSION) AC_SUBST(DARSHAN_VERSION)
AC_SUBST(MPICH_LIB_OLD) AC_SUBST(MPICH_LIB_OLD)
AC_SUBST(DARSHAN_USE_BGQ) AC_SUBST(DARSHAN_USE_BGQ)
......
...@@ -66,44 +66,68 @@ ...@@ -66,44 +66,68 @@
} while(0) } while(0)
/* potentially set or increment a common value counter, depending on the __count /* potentially set or increment a common value counter, depending on the __count
* for the given __value * for the given __value. This macro ensures common values are stored first in
* decreasing order of their total count, and second by decreasing order of
* their value.
* *
* NOTE: This macro is hardcoded to expect that Darshan will only track the 4 * NOTE: This macro is hardcoded to expect that Darshan will only track the 4
* most common (i.e., frequently occuring) values. __val_p is a pointer to the * most common (i.e., frequently occuring) values. __val_p is a pointer to the
* base of the value counters (i.e., the first of 4 contiguous common value * base of the value counters (i.e., the first of 4 contiguous common value
* counters) and __cnt_p is a pointer to the base of the count counters (i.e. * counters) and __cnt_p is a pointer to the base of the count counters (i.e.
* the first of 4 contiguous common count counters). It is assumed your counters * the first of 4 contiguous common count counters). It is assumed your counters
* are stored as int64_t types. * are stored as int64_t types. __add_flag is set if the given count should be
* added to the common access counter, rather than just incrementing it.
*/ */
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count) do {\ #define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count, __add_flag) do {\
int i; \ int i; \
int set = 0; \ int inc_count, total_count; \
int64_t min = *(__cnt_p); \ int64_t tmp_val[4] = {0}; \
int min_index = 0; \ int64_t tmp_cnt[4] = {0}; \
int tmp_ndx = 0; \
if(__value == 0) break; \ if(__value == 0) break; \
if(__add_flag) \
inc_count = 1; \
else \
inc_count = __count; \
for(i=0; i<4; i++) { \ for(i=0; i<4; i++) { \
/* increment bucket if already exists */ \
if(*(__val_p + i) == __value) { \ if(*(__val_p + i) == __value) { \
*(__cnt_p + i) += __count; \ total_count = *(__cnt_p + i) + inc_count; \
set = 1; \
break; \ break; \
} \ } \
/* otherwise find the least frequently used bucket */ \ } \
else if(*(__cnt_p + i) < min) { \ if(i == 4) total_count = __count; \
min = *(__cnt_p + i); \ /* first, copy over any counters that should be sorted above this one \
min_index = i; \ * (counters with higher counts or equal counts and larger values) \
*/ \
for(i=0;i < 4; i++) { \
if((*(__cnt_p + i) > total_count) || \
((*(__cnt_p + i) == total_count) && (*(__val_p + i) > __value))) { \
tmp_val[tmp_ndx] = *(__val_p + i); \
tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
tmp_ndx++; \
} \ } \
else break; \
} \ } \
if(!set && (__count > min)) { \ if(tmp_ndx == 4) break; /* all done, updated counter is not added */ \
*(__cnt_p + min_index) = __count; \ /* next, add the updated counter */ \
*(__val_p + min_index) = __value; \ tmp_val[tmp_ndx] = __value; \
tmp_cnt[tmp_ndx] = total_count; \
tmp_ndx++; \
/* last, copy over any remaining counters to make sure we have 4 sets total */ \
while(tmp_ndx != 4) { \
if(*(__val_p + i) != __value) { \
tmp_val[tmp_ndx] = *(__val_p + i); \
tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
tmp_ndx++; \
} \
i++; \
} \ } \
memcpy(__val_p, tmp_val, 4*sizeof(int64_t)); \
memcpy(__cnt_p, tmp_cnt, 4*sizeof(int64_t)); \
} while(0) } while(0)
/* maximum number of common values that darshan will track per file at /* maximum number of common values that darshan will track per file at runtime */
* runtime; at shutdown time these will be reduced to the 4 most
* frequently occuring ones
*/
#define DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT 32 #define DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT 32
struct darshan_common_val_counter struct darshan_common_val_counter
{ {
...@@ -130,6 +154,67 @@ struct darshan_variance_dt ...@@ -130,6 +154,67 @@ struct darshan_variance_dt
* darshan-common functions for darshan modules * * darshan-common functions for darshan modules *
***********************************************/ ***********************************************/
/* darshan_lookup_record_ref()
*
* Lookup a record reference pointer using the given 'handle'.
* 'handle_sz' is the size of the handle structure, and 'hash_head'
* is the pointer to the hash table to search.
* If the handle is found, the corresponding record reference pointer
* is returned, otherwise NULL is returned.
*/
void *darshan_lookup_record_ref(
void *hash_head,
void *handle,
size_t handle_sz);
/* darshan_add_record_ref()
*
* Add the given record reference pointer, 'rec_ref_p' to the hash
* table whose address is stored in the 'hash_head_p' pointer. The
* hash is generated from the given 'handle', with size 'handle_sz'.
* If the record reference is successfully added, 1 is returned,
* otherwise, 0 is returned.
*/
int darshan_add_record_ref(
void **hash_head_p,
void *handle,
size_t handle_sz,
void *rec_ref_p);
/* darshan_delete_record_ref()
*
* Delete the record reference for the given 'handle', with size
* 'handle_sz', from the hash table whose address is stored in
* the 'hash_head_p' pointer.
* On success deletion, the corresponding record reference pointer
* is returned, otherwise NULL is returned.
*/
void *darshan_delete_record_ref(
void **hash_head_p,
void *handle,
size_t handle_sz);
/* darshan_clear_record_refs()
*
* Clear all record references from the hash table stored in the
* 'hash_head_p' pointer. If 'free_flag' is set, the corresponding
* record_reference_pointer is also freed.
*/
void darshan_clear_record_refs(
void **hash_head_p,
int free_flag);
/* darshan_iter_record_ref()
*
* Iterate each record reference stored in the hash table pointed
* to by 'hash_head' and perform the given action 'iter_action'.
* The action function takes a single pointer which points to the
* corresponding record reference pointer.
*/
void darshan_iter_record_refs(
void *hash_head,
void (*iter_action)(void *));
/* darshan_clean_file_path() /* darshan_clean_file_path()
* *
* Allocate a new string that contains a new cleaned-up version of * Allocate a new string that contains a new cleaned-up version of
...@@ -138,7 +223,21 @@ struct darshan_variance_dt ...@@ -138,7 +223,21 @@ struct darshan_variance_dt
* path string. * path string.
*/ */
char* darshan_clean_file_path( char* darshan_clean_file_path(
const char* path); const char *path);
/* darshan_record_sort()
*
* Sort the records in 'rec_buf' by descending rank to get all
* shared records in a contiguous region at the end of the buffer.
* Records are secondarily sorted by ascending record identifiers.
* 'rec_count' is the number of records in the buffer, and 'rec_size'
* is the size of the record structure.
* NOTE: this function only works on fixed-length records.
*/
void darshan_record_sort(
void *rec_buf,
int rec_count,
int rec_size);
/* darshan_common_val_counter() /* darshan_common_val_counter()
* *
...@@ -148,29 +247,19 @@ char* darshan_clean_file_path( ...@@ -148,29 +247,19 @@ char* darshan_clean_file_path(
* used by a specific module, for instance. 'common_val_root' is the * used by a specific module, for instance. 'common_val_root' is the
* root pointer for the tree which stores common value info,