Merge branch 'master' into mmap-dev

Conflicts:
	darshan-runtime/Makefile.in
	darshan-runtime/configure
	darshan-runtime/configure.in
	darshan-runtime/darshan.h
	darshan-runtime/lib/darshan-core.c
	darshan-runtime/lib/darshan-hdf5.c
	darshan-runtime/lib/darshan-mpiio.c
	darshan-runtime/lib/darshan-pnetcdf.c
	darshan-runtime/lib/darshan-posix.c
	darshan-util/Makefile.in
	darshan-util/darshan-convert.c
	darshan-util/darshan-logutils.c
	darshan-util/darshan-logutils.h
	darshan-util/darshan-parser.c

ChangeLog

@@ -2,6 +2,51 @@
 Darshan Release Change Log
 --------------------------
 
+Darshan-3.0.1
+=============
+* bug fix in darshan logutil mount parsing code that was
+  causing file paths to be matched to the first mount point
+  with a common prefix rather than the one with the longest
+  common prefix
+* bug fix in the darshan-util bzip2 configure check that
+  was accidentally overwriting Darshan's LDFLAGS
+* minor bug fixes to IO start time counters in all modules
+  to set IO start time to the actual first start time rather
+  than the first IO op to complete
+* update darshan-util perl scripts to get perl bin from
+  user's path, rather than from /usr/lib (reported by
+  Kay Thust)
+* update Darshan's fortran and cxx compiler wrapper generators
+  to automatically detect MPICH library names on BG/Q
+* fix bug that was calculating Darshan's agg_perf_by_slowest
+  performance metric incorrectly
+* add performance estimate info to darshan-job-summary
+  graphs
+
+Darshan-3.0.0
+=============
+* install darshan-null-log-format.h header when installing
+  darshan-util component, otherwise compiler errors are
+  generated when building external tools that use
+  libdarshan-util
+* update docs to give debugging tips for cases where
+  Darshan logs are not generated
+* fix shared library regression test script to check for
+  potential errors with Darshan symbols rather than
+  failing silently in these cases
+* bug fix for determining minimum non-zero counters in
+shared file reductions in all modules
+* loosen Darshan's PMPI symbol check to prevent inadvertent
+  disabling of Darshan for some MPICH builds
+* update runtime docs to give information on upgrading Darshan
+* bug fix for resolving MPI_Gather and MPI_Barrier when LDPRELOADing
+  Darshan's shared libraries (reported by Richard Hedges and Rob Latham)
+* add more helpful error handling when opening 2.x version log files
+* port darshan-diff utility over to new log file format
+* fix numerous configure bugs on Cray systems
+* add synthetic benchmarking hooks for testing Darshan's shutdown
+  procedure
+
 Darshan-3.0.0-pre3
 =============
 * add module-specific version fields to header to allow utilities

darshan-runtime/Makefile.in

@@ -155,10 +155,11 @@ endif
 	install -m 755 share/mpi-profile/darshan-bg-cxx.conf $(datarootdir)/mpi-profile/darshan-bg-cxx.conf
 	install -m 755 share/mpi-profile/darshan-bg-f.conf $(datarootdir)/mpi-profile/darshan-bg-f.conf
 	install -m 755 share/darshan-mmap-epilog.sh $(datarootdir)/darshan-mmap-epilog.sh
-	install -m 644 darshan-base-ld-opts $(datarootdir)/darshan-base-ld-opts
-	install -m 644 $(srcdir)/darshan-posix-ld-opts $(datarootdir)/darshan-posix-ld-opts
-	install -m 644 $(srcdir)/darshan-hdf5-ld-opts $(datarootdir)/darshan-hdf5-ld-opts
-	install -m 644 $(srcdir)/darshan-pnetcdf-ld-opts $(datarootdir)/darshan-pnetcdf-ld-opts
+	install -d $(datarootdir)/ld-opts
+	install -m 644 share/ld-opts/darshan-base-ld-opts $(datarootdir)/ld-opts/darshan-base-ld-opts
+	install -m 644 $(srcdir)/share/ld-opts/darshan-posix-ld-opts $(datarootdir)/ld-opts/darshan-posix-ld-opts
+	install -m 644 $(srcdir)/share/ld-opts/darshan-hdf5-ld-opts $(datarootdir)/ld-opts/darshan-hdf5-ld-opts
+	install -m 644 $(srcdir)/share/ld-opts/darshan-pnetcdf-ld-opts $(datarootdir)/ld-opts/darshan-pnetcdf-ld-opts
 	install -d $(libdir)/pkgconfig
 	install -m 644 lib/pkgconfig/darshan-runtime.pc $(libdir)/pkgconfig/darshan-runtime.pc
 

darshan-runtime/configure

 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for darshan-runtime 3.0.0-pre3.
+# Generated by GNU Autoconf 2.69 for darshan-runtime 3.0.1.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -577,8 +577,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='darshan-runtime'
 PACKAGE_TARNAME='darshan-runtime'
-PACKAGE_VERSION='3.0.0-pre3'
-PACKAGE_STRING='darshan-runtime 3.0.0-pre3'
+PACKAGE_VERSION='3.0.1'
+PACKAGE_STRING='darshan-runtime 3.0.1'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1245,7 +1245,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures darshan-runtime 3.0.0-pre3 to adapt to many kinds of systems.
+\`configure' configures darshan-runtime 3.0.1 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1306,7 +1306,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of darshan-runtime 3.0.0-pre3:";;
+     short | recursive ) echo "Configuration of darshan-runtime 3.0.1:";;
    esac
   cat <<\_ACEOF
 
@@ -1415,7 +1415,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-darshan-runtime configure 3.0.0-pre3
+darshan-runtime configure 3.0.1
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1767,7 +1767,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by darshan-runtime $as_me 3.0.0-pre3, which was
+It was created by darshan-runtime $as_me 3.0.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -4300,20 +4300,24 @@ $as_echo "#define __D_MPI_REQUEST MPI_Request" >>confdefs.h
 fi
 
 
-# attempt to detect librarly naming convention in mpi compiler script
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for old (pre-3.1.1) style MPICH library naming convention" >&5
+# attempt to detect library naming convention in mpi compiler script
+if $CC -show foo.c -o foo >& /dev/null; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for old (pre-3.1.1) style MPICH library naming convention" >&5
 $as_echo_n "checking for old (pre-3.1.1) style MPICH library naming convention... " >&6; }
-if $CC -show foo.c -o foo |grep lmpich >& /dev/null; then :
+	if $CC -show foo.c -o foo |grep lmpich >& /dev/null; then :
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
-    MPICH_LIB_OLD=1
+	    MPICH_LIB_OLD=1
 else
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 $as_echo "no" >&6; }
-    MPICH_LIB_OLD=0
+	    MPICH_LIB_OLD=0
+fi
+else
+  MPICH_LIB_OLD=0
 fi
 
-DARSHAN_VERSION="3.0.0-pre3"
+DARSHAN_VERSION="3.0.1"
 
 
 
@@ -4322,7 +4326,7 @@ DARSHAN_VERSION="3.0.0-pre3"
 
 
 
-ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-base-ld-opts darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/darshan-mmap-epilog.sh"
+ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/ld-opts/darshan-base-ld-opts share/darshan-mmap-epilog.sh"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -4831,7 +4835,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by darshan-runtime $as_me 3.0.0-pre3, which was
+This file was extended by darshan-runtime $as_me 3.0.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -4893,7 +4897,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-darshan-runtime config.status 3.0.0-pre3
+darshan-runtime config.status 3.0.1
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
@@ -5021,7 +5025,6 @@ do
     "darshan-gen-cc.pl") CONFIG_FILES="$CONFIG_FILES darshan-gen-cc.pl" ;;
     "darshan-gen-cxx.pl") CONFIG_FILES="$CONFIG_FILES darshan-gen-cxx.pl" ;;
     "darshan-gen-fortran.pl") CONFIG_FILES="$CONFIG_FILES darshan-gen-fortran.pl" ;;
-    "darshan-base-ld-opts") CONFIG_FILES="$CONFIG_FILES darshan-base-ld-opts" ;;
     "darshan-config") CONFIG_FILES="$CONFIG_FILES darshan-config" ;;
     "share/craype-1.x/darshan-module") CONFIG_FILES="$CONFIG_FILES share/craype-1.x/darshan-module" ;;
     "share/craype-2.x/darshan-module") CONFIG_FILES="$CONFIG_FILES share/craype-2.x/darshan-module" ;;
@@ -5032,6 +5035,7 @@ do
     "share/mpi-profile/darshan-bg-cc.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-cc.conf" ;;
     "share/mpi-profile/darshan-bg-cxx.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-cxx.conf" ;;
     "share/mpi-profile/darshan-bg-f.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-f.conf" ;;
+    "share/ld-opts/darshan-base-ld-opts") CONFIG_FILES="$CONFIG_FILES share/ld-opts/darshan-base-ld-opts" ;;
     "share/darshan-mmap-epilog.sh") CONFIG_FILES="$CONFIG_FILES share/darshan-mmap-epilog.sh" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;

darshan-runtime/configure.in

@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
 dnl You may need to use autoheader as well if changing any DEFINEs
 
 dnl sanity checks, output header, location of scripts used here
-AC_INIT([darshan-runtime], [3.0.0-pre3])
+AC_INIT([darshan-runtime], [3.0.1])
 AC_CONFIG_SRCDIR([darshan.h])
 AC_CONFIG_AUX_DIR(../maint/config)
 AC_CONFIG_HEADER(darshan-runtime-config.h)
@@ -312,13 +312,15 @@ AC_CHECK_TYPE([MPIO_Request],
 		AC_DEFINE(__D_MPI_REQUEST, MPI_Request, Generalized request type for MPI-IO)
 	,[#include <mpi.h>])
 
-# attempt to detect librarly naming convention in mpi compiler script
-AC_MSG_CHECKING(for old (pre-3.1.1) style MPICH library naming convention)
-AS_IF([$CC -show foo.c -o foo |grep lmpich >& /dev/null], 
-    AC_MSG_RESULT(yes)
-    MPICH_LIB_OLD=1,
-    AC_MSG_RESULT(no)
-    MPICH_LIB_OLD=0)
+# attempt to detect library naming convention in mpi compiler script
+AS_IF([$CC -show foo.c -o foo >& /dev/null],
+	AC_MSG_CHECKING(for old (pre-3.1.1) style MPICH library naming convention)
+	AS_IF([$CC -show foo.c -o foo |grep lmpich >& /dev/null], 
+	    AC_MSG_RESULT(yes)
+	    MPICH_LIB_OLD=1,
+	    AC_MSG_RESULT(no)
+	    MPICH_LIB_OLD=0),
+	MPICH_LIB_OLD=0)
 
 DARSHAN_VERSION="AC_PACKAGE_VERSION"
 AC_SUBST(darshan_lib_path)
@@ -334,7 +336,6 @@ darshan-mk-log-dirs.pl
 darshan-gen-cc.pl
 darshan-gen-cxx.pl
 darshan-gen-fortran.pl
-darshan-base-ld-opts
 darshan-config
 share/craype-1.x/darshan-module
 share/craype-2.x/darshan-module
@@ -345,5 +346,6 @@ share/mpi-profile/darshan-f.conf
 share/mpi-profile/darshan-bg-cc.conf
 share/mpi-profile/darshan-bg-cxx.conf
 share/mpi-profile/darshan-bg-f.conf
+share/ld-opts/darshan-base-ld-opts
 share/darshan-mmap-epilog.sh
 )

darshan-runtime/darshan-base-ld-opts.in

---undefined=MPI_Init
---undefined=MPI_Wtime
-@@darshan_share_path@/darshan-posix-ld-opts
-@@darshan_share_path@/darshan-hdf5-ld-opts
-@@darshan_share_path@/darshan-pnetcdf-ld-opts

darshan-runtime/darshan-config.in

@@ -11,8 +11,8 @@ DARSHAN_LD_FLAGS="@LDFLAGS@"
 #   dependencies on PnetCDF and HDF5 symbols (if the app used a library which 
 #   in turn used one of those HLLs).
 
-PRE_LD_FLAGS="-L$DARSHAN_LIB_PATH $DARSHAN_LD_FLAGS -ldarshan -lz -Wl,@$DARSHAN_SHARE_PATH/darshan-base-ld-opts"
-POST_LD_FLAGS="-L$DARSHAN_LIB_PATH -ldarshan -lz -lrt -lpthread"
+PRE_LD_FLAGS="-L$DARSHAN_LIB_PATH $DARSHAN_LD_FLAGS -ldarshan -lz -Wl,@$DARSHAN_SHARE_PATH/ld-opts/darshan-base-ld-opts"
+POST_LD_FLAGS="-L$DARSHAN_LIB_PATH -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group -lz -lrt -lpthread"
 
 usage="\
 Usage: darshan-config [--pre-ld-flags] [--post-ld-flags]"

darshan-runtime/darshan-dynamic.h

@@ -130,6 +130,7 @@ DARSHAN_EXTERN_DECL(PMPI_Gather, int, (const void *sendbuf, int sendcount, MPI_D
 #else
 DARSHAN_EXTERN_DECL(PMPI_Gather, int, (void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm));
 #endif
+DARSHAN_EXTERN_DECL(PMPI_Barrier, int, (MPI_Comm comm));
 
 #endif
 

darshan-runtime/darshan-gen-cxx.pl.in

@@ -282,6 +282,10 @@ print OUTPUT<<"EOF";
     if [ \$? -eq 0 ] ; then
         CXXMPICH=-lmpicxx
     fi
+    bgqlib_out=`grep -m 1 -Po "libmpichcxx-\\S+" \$tmpfile`
+    if [ \$? -eq 0 ] ; then
+        CXXMPICH=`echo \$bgqlib_out | sed 's/libmpichcxx-\\(.*\\)\\.a.*/-lmpichcxx-\\1/'`
+    fi
 
     rm \$tmpfile >& /dev/null
 

darshan-runtime/darshan-gen-fortran.pl.in

@@ -287,7 +287,13 @@ print OUTPUT<<"EOF";
         if [ \$rc_cnk_check -eq 0 ] ; then
             FMPICH=-lfmpich.cnk
         else
-            FMPICH=-lfmpich
+            bgqlib_out=`grep -m 1 -Po "libmpichf\\S\+\-\\S\+" \$tmpfile 2>/dev/null`
+            rc_bgqlib_check=\$?
+            if [ \$rc_bgqlib_check -eq 0 ] ; then
+                FMPICH=`echo \$bgqlib_out | sed 's/libmpich\\(.*\\)-\\(.*\\)\\.a.*/-lmpich\\1-\\2/'`
+            else
+                FMPICH=-lfmpich
+            fi
         fi
     fi
 

darshan-runtime/doc/darshan-runtime.txt

@@ -169,8 +169,8 @@ Example for MPICH 3.1 or earlier:
 ----
 export MPICC_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-cc
 export MPICXX_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-cxx
-export MPICF77_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-f
-export MPICF90_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-f
+export MPIF77_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-f
+export MPIF90_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-f
 ----
 
 Examples for command line use:
@@ -213,17 +213,36 @@ environment variable to insert instrumentation at run time.  The executables
 should be compiled using the normal, unmodified MPI compiler.
 
 To use this mechanism, set the `LD_PRELOAD` environment variable to the full
-path to the Darshan shared library, as in this example:
+path to the Darshan shared library. The preferred method of inserting Darshan
+instrumentation in this case is to set the LD_PRELOAD variable specifically
+for the application of interest. Typically this is possible using
+command line arguments offered by the `mpirun` or `mpiexec` scripts or by
+the job scheduler:
 
 ----
-export LD_PRELOAD=/home/carns/darshan-install/lib/libdarshan.so
+mpiexec -n 4 -env LD_PRELOAD /home/carns/darshan-install/lib/libdarshan.so mpi-io-test
+----
+
+----
+srun -n 4 --export=LD_PRELOAD=/home/carns/darshan-install/lib/libdarshan.so mpi-io-test
+----
+
+For sequential programs, the following will set LD_PRELOAD for process duration only:
+
 ----
+env LD_PRELOAD=/home/carns/darshan-install/lib/libdarshan.so mpi-io-test
+----
+
+Other environments may have other specific options for controlling this behavior.
+Please check your local site documentation for details.
 
-You can then run your application as usual.  Some environments may require a
-special `mpirun` or `mpiexec` command line argument to propagate the
-environment variable to all processes.  Other environments may require a
-scheduler submission option to control this behavior.  Please check your
-local site documentation for details.
+It is also possible to just export LD_PRELOAD as follows, but it is recommended
+against doing that to prevent Darshan and MPI symbols from being pulled into
+unrelated binaries:
+
+----
+export LD_PRELOAD=/home/carns/darshan-install/lib/libdarshan.so
+----
 
 === Instrumenting dynamically-linked Fortran applications
 
@@ -294,8 +313,8 @@ Blue Gene profiling configuration example:
 ----
 export MPICC_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-cc
 export MPICXX_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-cxx
-export MPICF77_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-f
-export MPICF90_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-f
+export MPIF77_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-f
+export MPIF90_PROFILE=$DARSHAN_PREFIX/share/mpi-profile/darshan-bg-f
 ----
 
 === Cray platforms (XE, XC, or similar)
@@ -483,20 +502,17 @@ runtime environment variables] are equally applicable to both versions.
 However, we do provide some suggestions and expectations for system administrators to keep in
 mind when upgrading to Darshan 3.x:
 
-* Log file compatibility was broken in the upgrade, and thus 3.x log utilities do not
-work directly with logs generated by 2.x versions of Darshan (and vice versa).
-    - There is currently no tool for converting 2.x logs into the 3.x log format.
-    - The `darshan-logutils` library will provide error messages to indicate whether a given
+* Darshan 2.x and Darshan 3.x produce incompatible log file formats
+    ** log files named *.darshan.gz or *.darshan.bz2: Darshan 2.x format
+    ** log files named *.darshan: Darshan 3.x format
+        *** a field in the log file header indicates underlying compression method in 3.x logs
+    ** There is currently no tool for converting 2.x logs into the 3.x log format.
+    ** The `darshan-logutils` library will provide error messages to indicate whether a given
 log file is incompatible with the correspnonding library version. 
 
-* The darshan log file extension has been changed from `.darshan.gz` (or `.darshan.bz2` for
-log files converted to use bzip2 compression) to `.darshan`.
-    - A field in the Darshan log header is used to indicate whether a log is compressed using
-libz or bzip2 compression.
-
 * We encourage administrators to use the same log file directory for version 3.x as had been
 used for version 2.x.
-    - Within this directory, the determination on which set of log utilities (version 2.x
+    ** Within this directory, the determination on which set of log utilities (version 2.x
 or version 3.x) to use can be based on the file extension for a given log (as explained
 above).
 
@@ -514,3 +530,47 @@ behavior at runtime:
 * DARSHAN_LOGPATH: specifies the path to write Darshan log files to. Note that this directory needs to be formatted using the darshan-mk-log-dirs script.
 * DARSHAN_LOGFILE: specifies the path (directory + Darshan log file name) to write the output Darshan log to. This overrides the default Darshan behavior of automatically generating a log file name and adding it to a log file directory formatted using darshan-mk-log-dirs script.
 * DARSHAN_MODMEM: specifies the maximum amount of memory (in MiB) a Darshan instrumentation module can consume at runtime.
+
+== Debugging
+
+=== No log file
+
+In cases where Darshan is not generating a log file for an application, some common things to check are:
+
+* Check stderr to ensure Darshan isn't indicating any internal errors (e.g., invalid log file path)
+
+For statically linked executables:
+
+* Ensure that Darshan symbols are present in the underlying executable by running `nm` on it:
+----
+> nm test | grep darshan
+0000000000772260 b darshan_core
+0000000000404440 t darshan_core_cleanup
+00000000004049b0 T darshan_core_initialize
+000000000076b660 d darshan_core_mutex
+00000000004070a0 T darshan_core_register_module
+----
+
+* Make sure the application executable is statically linked:
+    ** In general, we encourage the use of purely statically linked executables when using the static
+instrumentation method given in link:darshan-runtime.html#_instrumenting_statically_linked_applications[Section 5]
+    ** If purely static executables are not an option, we encourage users to use the LD_PRELOAD method of
+instrumentation given in link:darshan-runtime.html#_instrumenting_dynamically_linked_applications[Section 6]
+    ** Statically linked executables are the default on Cray platforms and the IBM BG platforms; 
+statically linked executables can be explicitly requested using the `-static` compile option to most compilers
+    ** You can verify that an executable is purely statically linked by using the `file` command:
+----
+> file mpi-io-test
+mpi-io-test: ELF 64-bit LSB  executable, x86-64, version 1 (GNU/Linux), statically linked, for GNU/Linux 2.6.24, BuildID[sha1]=9893e599e7a560159ccf547b4c4ba5671f65ba32, not stripped
+----
+
+* Ensure that the linker is correctly linking in Darshan's runtime libraries:
+    ** A common mistake is to explicitly link in the underlying MPI libraries (e.g., `-lmpich` or `-lmpichf90`)
+in the link command, which can interfere with Darshan's instrumentation
+        *** These libraries are usually linked in automatically by the compiler
+        *** MPICH's `mpicc` comipler's `-show` flag can be used to examine the invoked link command, for instance
+    ** The linker's `-y` option can be used to verify that Darshan is properly intercepting MPI_Init
+function (e.g. by setting `CFLAGS='-Wl,-yMPI_Init'`), which it uses to initialize its runtime structures
+----
+/usr/common/software/darshan/3.0.0-pre3/lib/libdarshan.a(darshan-core-init-finalize.o): definition of MPI_Init
+----

darshan-runtime/lib/darshan-core-init-finalize.c

@@ -136,6 +136,7 @@ DARSHAN_FORWARD_DECL(PMPI_Gather, int, (const void *sendbuf, int sendcount, MPI_
 #else
 DARSHAN_FORWARD_DECL(PMPI_Gather, int, (void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm));
 #endif
+DARSHAN_FORWARD_DECL(PMPI_Barrier, int, (MPI_Comm comm));
 
 void resolve_mpi_symbols (void)
 {
@@ -196,6 +197,7 @@ void resolve_mpi_symbols (void)
     MAP_OR_FAIL(PMPI_Send);
     MAP_OR_FAIL(PMPI_Recv);
     MAP_OR_FAIL(PMPI_Gather);
+    MAP_OR_FAIL(PMPI_Barrier);
 
     return;
 }

darshan-runtime/lib/darshan-core.c

@@ -55,6 +55,7 @@ char* darshan_path_exclusions[] = {
 "/sbin/",
 "/sys/",
 "/proc/",
+"/var/",
 NULL
 };
 
@@ -292,6 +293,10 @@ void darshan_core_initialize(int argc, char **argv)
         }
     }
 
+    /* if darshan was successfully initialized, set the global pointer */
+    if(init_core)
+        darshan_core = init_core;
+
     return;
 }
 
@@ -987,6 +992,7 @@ static void darshan_get_exe_and_mounts(struct darshan_core_runtime *core,
      * grab any non-nfs mount points, then on the second pass we grab nfs
      * mount points
      */
+    mnt_data_count = 0;
 
     tab = setmntent("/etc/mtab", "r");
     if(!tab)
@@ -1680,6 +1686,85 @@ static void darshan_core_cleanup(struct darshan_core_runtime* core)
     return;
 }
 
+/* crude benchmarking hook into darshan-core to benchmark Darshan
+ * shutdown overhead using a variety of application I/O workloads
+ */
+extern void darshan_posix_shutdown_bench_setup();
+extern void darshan_mpiio_shutdown_bench_setup();
+void darshan_shutdown_bench(int argc, char **argv)
+{
+    /* clear out existing core runtime structure */
+    if(darshan_core)
+    {
+        darshan_core_cleanup(darshan_core);
+        darshan_core = NULL;
+    }
+
+    /***********************************************************/
+    /* restart darshan */
+    darshan_core_initialize(argc, argv);
+
+    darshan_posix_shutdown_bench_setup(1);
+    darshan_mpiio_shutdown_bench_setup(1);
+
+    if(my_rank == 0)
+        printf("# 1 unique file per proc\n");
+    DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
+    darshan_core_shutdown();
+    darshan_core = NULL;
+
+    sleep(1);
+
+    /***********************************************************/
+    /* restart darshan */
+    darshan_core_initialize(argc, argv);
+
+    darshan_posix_shutdown_bench_setup(2);
+    darshan_mpiio_shutdown_bench_setup(2);
+
+    if(my_rank == 0)
+        printf("# 1 shared file per proc\n");
+    DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
+    darshan_core_shutdown();
+    darshan_core = NULL;
+
+    sleep(1);
+
+    /***********************************************************/
+    /* restart darshan */
+    darshan_core_initialize(argc, argv);
+
+    darshan_posix_shutdown_bench_setup(3);
+    darshan_mpiio_shutdown_bench_setup(3);
+
+    if(my_rank == 0)
+        printf("# 1024 unique files per proc\n");
+    DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
+    darshan_core_shutdown();
+    darshan_core = NULL;
+
+    sleep(1);
+
+    /***********************************************************/
+    /* restart darshan */
+    darshan_core_initialize(argc, argv);
+
+    darshan_posix_shutdown_bench_setup(4);
+    darshan_mpiio_shutdown_bench_setup(4);
+
+    if(my_rank == 0)
+        printf("# 1024 shared files per proc\n");
+    DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
+    darshan_core_shutdown();
+    darshan_core = NULL;
+
+    sleep(1);
+
+    /***********************************************************/
+
+    return;
+}
+
 /* ********************************************************* */
 
 void darshan_core_register_module(
@@ -1753,10 +1838,11 @@ void darshan_core_register_module(
 void darshan_core_unregister_module(
     darshan_module_id mod_id)
 {
+    DARSHAN_CORE_LOCK();
+
     if(!darshan_core)
         return;
 
-    DARSHAN_CORE_LOCK();
     /* update darshan internal structures and header */
     free(darshan_core->mod_array[mod_id]);
     darshan_core->mod_array[mod_id] = NULL;
@@ -1765,8 +1851,8 @@ void darshan_core_unregister_module(
     darshan_core->log_hdr_p->mod_map[mod_id].off =
         darshan_core->log_hdr_p->mod_map[mod_id].len = 0;
 #endif
-    DARSHAN_CORE_UNLOCK();
 
+    DARSHAN_CORE_UNLOCK();
     return;
 }
 

darshan-runtime/lib/darshan-hdf5.c

@@ -321,10 +321,11 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
         /* min non-zero (if available) value */
         for(j=HDF5_F_OPEN_TIMESTAMP; j<=HDF5_F_OPEN_TIMESTAMP; j++)
         {
-            if(infile->fcounters[j] > inoutfile->fcounters[j] && inoutfile->fcounters[j] > 0)
-                tmp_file.fcounters[j] = inoutfile->fcounters[j];
-            else
+            if((infile->fcounters[j] < inoutfile->fcounters[j] &&
+               infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0) 
                 tmp_file.fcounters[j] = infile->fcounters[j];
+            else
+                tmp_file.fcounters[j] = inoutfile->fcounters[j];
         }
 
         /* max */

darshan-runtime/lib/darshan-mpiio.c

@@ -110,6 +110,42 @@ static int my_rank = -1;
     MPIIO_UNLOCK(); \
 } while(0)
 
+#define MPIIO_RECORD_OPEN(__ret, __path, __fh, __comm, __mode, __info, __tm1, __tm2) do { \
+    darshan_record_id rec_id; \
+    struct mpiio_file_record_ref *rec_ref; \
+    char *newpath; \
+    int comm_size; \
+    if(__ret != MPI_SUCCESS) break; \
+    newpath = darshan_clean_file_path(__path); \
+    if(!newpath) newpath = (char *)__path; \
+    if(darshan_core_excluded_path(newpath)) { \
+        if(newpath != __path) free(newpath); \
+        break; \
+    } \
+    rec_id = darshan_core_gen_record_id(newpath); \
+    rec_ref = darshan_lookup_record_ref(mpiio_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
+    if(!rec_ref) rec_ref = mpiio_track_new_file_record(rec_id, newpath); \
+    if(!rec_ref) { \
+        if(newpath != __path) free(newpath); \
+        break; \
+    } \
+    rec_ref->file_rec->counters[MPIIO_MODE] = __mode; \
+    DARSHAN_MPI_CALL(PMPI_Comm_size)(__comm, &comm_size); \
+    if(comm_size == 1) \
+        rec_ref->file_rec->counters[MPIIO_INDEP_OPENS] += 1; \
+    else \