...
 
Commits (29)
stages:
- build
- test
build_darshan:
tags:
- shell
- ecp-theta
stage: build
script:
- source darshan-test/automated/common/setup.sh
- source darshan-test/automated/common/configure.sh
- source darshan-test/automated/common/build.sh
test_darshan:
variables:
# XXX this info is hard-coded in our regression scripts
#SCHEDULER_PARAMETERS: "-A radix-io -n 1 -t 10 -q default"
tags:
- batch
- ecp-theta
stage: test
script:
- darshan-test/regression/run-all.sh install scratch cray-module-alcf
......@@ -2,8 +2,16 @@
Darshan Release Change Log
--------------------------
Darshan-3.1.8
=============
* added dynamic trace triggering functionality, allowing Darshan to
decide at runtime whether or not to trace specific files with DXT,
with this decision based on triggers in a user-provided config file
Darshan-3.1.7
=============
* bug fix for handling DXT data in darshan-convert utility. Bug reported
by Mahzad Khoshlessan.
* bug fix for darshan-parser backwards compatibility: Darshan logs generated
by Darshan versions prior to 3.1.0 may have included stdio counters that
were not properly up-converted. Reported by Teng Wang.
......@@ -11,6 +19,9 @@ Darshan-3.1.7
darshan-job-summary.pl when both posix and stdio access is present
* added wrapper for __open_2(), bug reported by Cormac Garvey in which open
calls are not intercepted with some versions of glibc/gcc
* added an instrumentation module for the MDHIM key/val storage system
* added support for properly instrumenting dup(), fileno(), fdopen(), and
rename() calls
Darshan-3.1.6
=============
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_HDF5_LOG_FORMAT_H
/* current HDF5 log format version */
#define DARSHAN_HDF5_VER 1
#define DARSHAN_HDF5_VER 2
#define HDF5_COUNTERS \
/* count of HDF5 opens */\
......@@ -18,9 +18,13 @@
#define HDF5_F_COUNTERS \
/* timestamp of first open */\
X(HDF5_F_OPEN_TIMESTAMP) \
X(HDF5_F_OPEN_START_TIMESTAMP) \
/* timestamp of first close */\
X(HDF5_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(HDF5_F_OPEN_END_TIMESTAMP) \
/* timestamp of last close */\
X(HDF5_F_CLOSE_TIMESTAMP) \
X(HDF5_F_CLOSE_END_TIMESTAMP) \
/* end of counters*/\
X(HDF5_F_NUM_INDICES)
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_MPIIO_LOG_FORMAT_H
/* current MPI-IO log format version */
#define DARSHAN_MPIIO_VER 2
#define DARSHAN_MPIIO_VER 3
/* TODO: maybe use a counter to track cases in which a derived datatype is used? */
......@@ -92,17 +92,21 @@
#define MPIIO_F_COUNTERS \
/* timestamp of first open */\
X(MPIIO_F_OPEN_TIMESTAMP) \
X(MPIIO_F_OPEN_START_TIMESTAMP) \
/* timestamp of first read */\
X(MPIIO_F_READ_START_TIMESTAMP) \
/* timestamp of first write */\
X(MPIIO_F_WRITE_START_TIMESTAMP) \
/* timestamp of first close */\
X(MPIIO_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(MPIIO_F_OPEN_END_TIMESTAMP) \
/* timestamp of last read */\
X(MPIIO_F_READ_END_TIMESTAMP) \
/* timestamp of last write */\
X(MPIIO_F_WRITE_END_TIMESTAMP) \
/* timestamp of last close */\
X(MPIIO_F_CLOSE_TIMESTAMP) \
X(MPIIO_F_CLOSE_END_TIMESTAMP) \
/* cumulative MPI-IO read time */\
X(MPIIO_F_READ_TIME) \
/* cumulative MPI-IO write time */\
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_PNETCDF_LOG_FORMAT_H
/* current PNETCDF log format version */
#define DARSHAN_PNETCDF_VER 1
#define DARSHAN_PNETCDF_VER 2
#define PNETCDF_COUNTERS \
/* count of PNETCDF independent opens */\
......@@ -20,9 +20,13 @@
#define PNETCDF_F_COUNTERS \
/* timestamp of first open */\
X(PNETCDF_F_OPEN_TIMESTAMP) \
X(PNETCDF_F_OPEN_START_TIMESTAMP) \
/* timestamp of first close */\
X(PNETCDF_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(PNETCDF_F_OPEN_END_TIMESTAMP) \
/* timestamp of last close */\
X(PNETCDF_F_CLOSE_TIMESTAMP) \
X(PNETCDF_F_CLOSE_END_TIMESTAMP) \
/* end of counters*/\
X(PNETCDF_F_NUM_INDICES)
......
......@@ -10,7 +10,7 @@
#define DARSHAN_POSIX_VER 4
#define POSIX_COUNTERS \
/* count of posix opens */\
/* count of posix opens (INCLUDING fileno and dup operations) */\
X(POSIX_OPENS) \
/* count of number of filenos */\
X(POSIX_FILENOS) \
......
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for darshan-runtime 3.1.6.
# Generated by GNU Autoconf 2.69 for darshan-runtime 3.1.8.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
......@@ -577,8 +577,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='darshan-runtime'
PACKAGE_TARNAME='darshan-runtime'
PACKAGE_VERSION='3.1.6'
PACKAGE_STRING='darshan-runtime 3.1.6'
PACKAGE_VERSION='3.1.8'
PACKAGE_STRING='darshan-runtime 3.1.8'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
......@@ -667,6 +667,7 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
......@@ -753,6 +754,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
......@@ -1005,6 +1007,15 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
......@@ -1142,7 +1153,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir
libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
......@@ -1255,7 +1266,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures darshan-runtime 3.1.6 to adapt to many kinds of systems.
\`configure' configures darshan-runtime 3.1.8 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
......@@ -1295,6 +1306,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
......@@ -1316,7 +1328,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of darshan-runtime 3.1.6:";;
short | recursive ) echo "Configuration of darshan-runtime 3.1.8:";;
esac
cat <<\_ACEOF
......@@ -1431,7 +1443,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
darshan-runtime configure 3.1.6
darshan-runtime configure 3.1.8
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -1783,7 +1795,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by darshan-runtime $as_me 3.1.6, which was
It was created by darshan-runtime $as_me 3.1.8, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
......@@ -3496,7 +3508,7 @@ ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_inclu
if test "x$ac_cv_header_zlib_h" = xyes; then :
else
as_fn_error $? "z.h not found" "$LINENO" 5
as_fn_error $? "zlib.h not found" "$LINENO" 5
fi
......@@ -4420,7 +4432,7 @@ else
MPICH_LIB_OLD=0
fi
DARSHAN_VERSION="3.1.6"
DARSHAN_VERSION="3.1.8"
......@@ -4944,7 +4956,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by darshan-runtime $as_me 3.1.6, which was
This file was extended by darshan-runtime $as_me 3.1.8, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
......@@ -5006,7 +5018,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
darshan-runtime config.status 3.1.6
darshan-runtime config.status 3.1.8
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
......
......@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl You may need to use autoheader as well if changing any DEFINEs
dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-runtime], [3.1.6])
AC_INIT([darshan-runtime], [3.1.8])
AC_CONFIG_SRCDIR([darshan.h])
AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-runtime-config.h)
......
......@@ -210,12 +210,16 @@ void darshan_clear_record_refs(
*
* Iterate each record reference stored in the hash table pointed
* to by 'hash_head' and perform the given action 'iter_action'.
* The action function takes a single pointer which points to the
* corresponding record reference pointer.
* The action function takes two pointers as input: the first
* points to the corresponding record reference pointer and the
* second is a user-supplied pointer provided for the action.
* 'user_ptr' is the user-supplied pointer that will be passed
* as the 2nd argument to 'iter_action'.
*/
void darshan_iter_record_refs(
void *hash_head,
void (*iter_action)(void *));
void (*iter_action)(void *, void *),
void *user_ptr);
/* darshan_clean_file_path()
*
......
/*
* Copyright (C) 2019 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_DXT_H
#define __DARSHAN_DXT_H
/* dxt_load_trigger_conf()
*
* DXT function exposed to Darshan core to read in any trace triggers
* from the file path in 'trigger_conf_path' before module
* initialization occurs.
*/
void dxt_load_trigger_conf(
char *trigger_conf_path);
/* dxt_posix_runtime_initialize()
*
* DXT function exposed to POSIX module for initializing DXT-POSIX runtime.
*/
void dxt_posix_runtime_initialize(void);
/* dxt_mpiio_runtime_initialize()
*
* DXT function exposed to MPIIO module for initializing DXT-MPIIO runtime.
*/
void dxt_mpiio_runtime_initialize(void);
/* dxt_posix_write(), dxt_posix_read()
*
* DXT function to trace a POSIX write/read call to file record 'rec_id',
* at offset 'offset' and with 'length' size. 'start_time' and 'end_time'
* are starting and ending timestamps for the operation, respectively.
*/
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
/* dxt_mpiio_write(), dxt_mpiio_read()
*
* DXT function to trace a MPIIO write/read call to file record 'rec_id',
* with 'length' size. 'start_time' and 'end_time' are starting and ending
* timestamps for the operation, respectively.
*/
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
void dxt_posix_filter_dynamic_traces(
struct darshan_posix_file *(*rec_id_to_psx_file)(darshan_record_id));
#endif /* __DARSHAN_DXT_H */
......@@ -67,7 +67,7 @@ scheduler does not advertise the job ID) then you can specify `NONE` here.
Darshan will fall back to using the pid of the rank 0 process if the
specified environment variable is not set.
* `--with-log-path=` (this, or `--with-log-path-by-env`, is mandatory): This
specifies the parent directory for the directory tree where darshan logs
specifies the parent directory for the directory tree where Darshan logs
will be placed.
* `--with-log-path-by-env=`: specifies an environment variable to use to
determine the log path at run time.
......@@ -83,7 +83,7 @@ header and library.
* `--disable-ld-preload`: disables building of the Darshan LD_PRELOAD library
* `--disable-bgq-mod`: disables building of the BG/Q module (default checks
and only builds if BG/Q environment detected).
* `--enable-group-readable-logs`: sets darshan log file permissions to allow
* `--enable-group-readable-logs`: sets Darshan log file permissions to allow
group read access.
* `--enable-HDF5-pre-1.10`: enables the Darshan HDF5 instrumentation module,
with support for HDF5 versions prior to 1.10
......@@ -128,7 +128,7 @@ darshan-mk-log-dirs.pl
.A note about log directory permissions
[NOTE]
====
All log files written by darshan have permissions set to only allow
All log files written by Darshan have permissions set to only allow
read access by the owner of the file. You can modify this behavior,
however, by specifying the --enable-group-readable-logs option at
configure time. One notable deployment scenario would be to configure
......@@ -164,7 +164,7 @@ MPI compiler script (e.g. `mpicc`) that includes the link options and
libraries needed by Darshan, or to use existing profiling configuration
hooks for existing MPI compiler scripts. Once this is done, Darshan
instrumentation is transparent; you simply compile applications using
the darshan-enabled MPI compiler scripts.
the Darshan-enabled MPI compiler scripts.
=== Using a profile configuration
......@@ -226,7 +226,7 @@ list for help.
== Instrumenting dynamically-linked applications
For dynamically-linked executables, darshan relies on the `LD_PRELOAD`
For dynamically-linked executables, Darshan relies on the `LD_PRELOAD`
environment variable to insert instrumentation at run time. The executables
should be compiled using the normal, unmodified MPI compiler.
......@@ -283,11 +283,61 @@ The full path to the libfmpich.so library can be omitted if the rpath
variable points to the correct path. Be careful to check the rpath of the
darshan library and the executable before using this configuration, however.
They may provide conflicting paths. Ideally the rpath to the MPI library
would *not* be set by the darshan library, but would instead be specified
would *not* be set by the Darshan library, but would instead be specified
exclusively by the executable itself. You can check the rpath of the
darshan library by running `objdump -x
/home/carns/darshan-install/lib/libdarshan.so |grep RPATH`.
== Using the Darshan eXtended Tracing (DXT) module
DXT support is disabled by default in Darshan, requiring the user to either explicitly
enable tracing for all files or to provide a trace trigger configuration file describing
which files should be traced at runtime.
To enable tracing globally for all files, Darshan users simply need to set the
DXT_ENABLE_IO_TRACE environment variable as follows:
----
export DXT_ENABLE_IO_TRACE=1
----
To enable tracing for particular files, DXT additionally offers a trace
triggering mechansim, with users specifying triggers used to decide whether or
not to trace a particular file at runtime. Files that do not match any trace
trigger will not store trace data in the Darshan log. Currently, DXT supports
the following types of trace triggers:
* file triggers: trace files based on regex matching of file paths
* rank triggers: trace files based on regex matching of ranks
* dynamic triggers: trace files based on runtime analysis of I/O characteristics (e.g., frequent small or unaligned I/O accesses)
Users simply need to specify one or more of these triggers in a text file that is passed
to DXT at runtime -- when multiple triggers are specified, DXT will keep any file traces
that match at least one trigger (i.e., the trace decision is a logical OR accross given triggers).
An example configuration file is given below, illustrating the syntax to use for currently
supported triggers:
----
FILE .h5$ # trace all files with a '.h5' extension
FILE ^/tmp # trace all files with a path prefix of '/tmp'
RANK [1-2] # trace all files accessed by ranks 1-2
SMALL_IO .5 # trace all files with greater than 50% small (less than 10 KB) accesses
UNALIGNED_IO .5 # trace all files with greater than 50% unaligned accesses
----
FILE and RANK triggers take a single parameter representing the regex that will be compared
to the file name and the rank accessing it, respectively. Regex support is provided by the
POSIX `regex.h` interface -- refer to the https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/regex.h.html[manpage] for more details on regex syntax.
SMALL_IO and UNALIGNED_IO triggers take a single parameter representing the lower
threshold percentage of accesses of the given type.
Set the DXT_TRIGGER_CONF_PATH environment variable to notify DXT of the path of the
configuration file:
----
export DXT_TRIGGER_CONF_PATH=/path/to/dxt/config/file
----
== Darshan installation recipes
The following recipes provide examples for prominent HPC systems.
......@@ -556,7 +606,9 @@ behavior at runtime:
* DARSHAN_MODMEM: specifies the maximum amount of memory (in MiB) Darshan instrumentation modules can collectively consume at runtime (if not specified, Darshan uses a default quota of 2 MiB).
* DARSHAN_MMAP_LOGPATH: if Darshan's mmap log file mechanism is enabled, this variable specifies what path the mmap log files should be stored in (if not specified, log files will be stored in `/tmp`).
* DARSHAN_EXCLUDE_DIRS: specifies a list of comma-separated paths that Darshan will not instrument at runtime (in addition to Darshan's default blacklist)
* DXT_ENABLE_IO_TRACE: setting this environment variable enables the DXT (Darshan eXtended Tracing) modules at runtime. Users can specify a numeric value for this variable to set the number of MiB to use for tracing per process; if no value is specified, Darshan will use a default value of 4 MiB.
* DXT_ENABLE_IO_TRACE: setting this environment variable enables the DXT (Darshan eXtended Tracing) modules at runtime for all files instrumented by Darshan. Currently, DXT is hard-coded to use a maximum of 4 MiB of trace memory per process (in addition to memory used by other modules).
* DXT_DISABLE_IO_TRACE: setting this environment variable disables the DXT module at runtime for all files instrumented by Darshan.
* DXT_TRIGGER_CONF_PATH: File path to a DXT trace trigger configuration file, which specifies triggers used by DXT to decide which files to trace at runtime. Note that the trace triggering mechanism is overridden by the DXT_ENABLE_IO_TRACE and DXT_DISABLE_IO_TRACE environment variables.
== Debugging
......
......@@ -103,7 +103,8 @@ void darshan_clear_record_refs(void **hash_head_p, int free_flag)
return;
}
void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *))
void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *, void *),
void *user_ptr)
{
struct darshan_record_ref_tracker *ref_tracker, *tmp;
struct darshan_record_ref_tracker *ref_tracker_head =
......@@ -114,7 +115,7 @@ void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *))
*/
HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp)
{
iter_action(ref_tracker->rec_ref_p);
iter_action(ref_tracker->rec_ref_p, user_ptr);
}
return;
......
......@@ -34,6 +34,7 @@
#include "darshan.h"
#include "darshan-core.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
#ifdef DARSHAN_LUSTRE
#include <lustre/lustre_user.h>
......@@ -308,6 +309,13 @@ void darshan_core_initialize(int argc, char **argv)
/* collect information about command line and mounted file systems */
darshan_get_exe_and_mounts(init_core, argc, argv);
/* determine if/when DXT should be enabled by looking for triggers */
char *trigger_conf = getenv("DXT_TRIGGER_CONF_PATH");
if(trigger_conf)
{
dxt_load_trigger_conf(trigger_conf);
}
/* if darshan was successfully initialized, set the global pointer
* and bootstrap any modules with static initialization routines
*/
......
This diff is collapsed.
......@@ -90,7 +90,7 @@ static int my_rank = -1;
HDF5_UNLOCK(); \
} while(0)
#define HDF5_RECORD_OPEN(__ret, __path, __tm1) do { \
#define HDF5_RECORD_OPEN(__ret, __path, __tm1, __tm2) do { \
darshan_record_id rec_id; \
struct hdf5_file_record_ref *rec_ref; \
char *newpath; \
......@@ -107,8 +107,10 @@ static int my_rank = -1;
if(newpath != __path) free(newpath); \
break; \
} \
if(rec_ref->file_rec->fcounters[HDF5_F_OPEN_TIMESTAMP] == 0) \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_END_TIMESTAMP] = __tm2; \
rec_ref->file_rec->counters[HDF5_OPENS] += 1; \
darshan_add_record_ref(&(hdf5_runtime->hid_hash), &__ret, sizeof(hid_t), rec_ref); \
if(newpath != __path) free(newpath); \
......@@ -123,7 +125,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
{
hid_t ret;
char* tmp;
double tm1;
double tm1, tm2;
unsigned majnum, minnum, relnum;
H5get_libversion(&majnum, &minnum, &relnum);
......@@ -146,6 +148,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
tm1 = darshan_core_wtime();
ret = __real_H5Fcreate(filename, flags, create_plist, access_plist);
tm2 = darshan_core_wtime();
if(ret >= 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -159,7 +162,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
}
HDF5_PRE_RECORD();
HDF5_RECORD_OPEN(ret, filename, tm1);
HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
HDF5_POST_RECORD();
}
......@@ -171,7 +174,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
{
hid_t ret;
char* tmp;
double tm1;
double tm1, tm2;
unsigned majnum, minnum, relnum;
H5get_libversion(&majnum, &minnum, &relnum);
......@@ -194,6 +197,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
tm1 = darshan_core_wtime();
ret = __real_H5Fopen(filename, flags, access_plist);
tm2 = darshan_core_wtime();
if(ret >= 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -207,7 +211,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
}
HDF5_PRE_RECORD();
HDF5_RECORD_OPEN(ret, filename, tm1);
HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
HDF5_POST_RECORD();
}
......@@ -218,19 +222,24 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
{
struct hdf5_file_record_ref *rec_ref;
double tm1, tm2;
herr_t ret;
MAP_OR_FAIL(H5Fclose);
tm1 = darshan_core_wtime();
ret = __real_H5Fclose(file_id);
tm2 = darshan_core_wtime();
HDF5_PRE_RECORD();
rec_ref = darshan_lookup_record_ref(hdf5_runtime->hid_hash,
&file_id, sizeof(hid_t));
if(rec_ref)
{
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_END_TIMESTAMP] = tm2;
darshan_delete_record_ref(&(hdf5_runtime->hid_hash),
&file_id, sizeof(hid_t));
}
......@@ -349,7 +358,7 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* min non-zero (if available) value */
for(j=HDF5_F_OPEN_TIMESTAMP; j<=HDF5_F_OPEN_TIMESTAMP; j++)
for(j=HDF5_F_OPEN_START_TIMESTAMP; j<=HDF5_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -359,7 +368,7 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* max */
for(j=HDF5_F_CLOSE_TIMESTAMP; j<=HDF5_F_CLOSE_TIMESTAMP; j++)
for(j=HDF5_F_OPEN_END_TIMESTAMP; j<=HDF5_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......
......@@ -27,9 +27,9 @@
static void lustre_runtime_initialize(
void);
static void lustre_subtract_shared_rec_size(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void lustre_set_rec_ref_pointers(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static int lustre_record_compare(
const void* a_p, const void* b_p);
int sort_lustre_records(
......@@ -266,7 +266,7 @@ static void lustre_shutdown(
if (my_rank != 0)
{
darshan_iter_record_refs(lustre_runtime->record_id_hash,
&lustre_subtract_shared_rec_size);
&lustre_subtract_shared_rec_size, NULL);
}
}
......@@ -282,7 +282,7 @@ static void lustre_shutdown(
return;
}
static void lustre_subtract_shared_rec_size(void *rec_ref_p)
static void lustre_subtract_shared_rec_size(void *rec_ref_p, void *user_ptr)
{
struct lustre_record_ref *l_rec_ref = (struct lustre_record_ref *)rec_ref_p;
......@@ -291,7 +291,7 @@ static void lustre_subtract_shared_rec_size(void *rec_ref_p)
LUSTRE_RECORD_SIZE( l_rec_ref->record->counters[LUSTRE_STRIPE_WIDTH] );
}
static void lustre_set_rec_ref_pointers(void *rec_ref_p)
static void lustre_set_rec_ref_pointers(void *rec_ref_p, void *user_ptr)
{
lustre_runtime->record_ref_array[lustre_runtime->record_ref_array_ndx] = rec_ref_p;
lustre_runtime->record_ref_array_ndx++;
......@@ -356,7 +356,7 @@ int sort_lustre_records()
/* build the array of record reference pointers we want to sort */
darshan_iter_record_refs(lustre_runtime->record_id_hash,
&lustre_set_rec_ref_pointers);
&lustre_set_rec_ref_pointers, NULL);
/* qsort breaks the hash table, so delete it now to free its memory buffers
* and prevent later confusion */
......
......@@ -578,7 +578,7 @@ static void mdhim_shutdown(
if (my_rank != 0)
{
darshan_iter_record_refs(mdhim_runtime->rec_id_hash,
&mdhim_subtract_shared_rec_size);
&mdhim_subtract_shared_rec_size, NULL);
}
}
*mdhim_buf_sz = mdhim_rec_count * mdhim_runtime->record_buffer_size;
......
......@@ -24,6 +24,7 @@
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
DARSHAN_FORWARD_DECL(PMPI_File_close, int, (MPI_File *fh));
DARSHAN_FORWARD_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
......@@ -159,7 +160,7 @@ static void mpiio_runtime_initialize(
static struct mpiio_file_record_ref *mpiio_track_new_file_record(
darshan_record_id rec_id, const char *path);
static void mpiio_finalize_file_records(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void mpiio_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void mpiio_shared_record_variance(
......@@ -172,16 +173,9 @@ static void mpiio_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **mpiio_buf, int *mpiio_buf_sz);
/* extern DXT function defs */
extern void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
extern void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
static struct mpiio_runtime *mpiio_runtime = NULL;
static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;
static int enable_dxt_io_trace = 0;
#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
......@@ -229,9 +223,10 @@ static int enable_dxt_io_trace = 0;
rec_ref->file_rec->counters[MPIIO_COLL_OPENS] += 1; \
if(__info != MPI_INFO_NULL) \
rec_ref->file_rec->counters[MPIIO_HINTS] += 1; \
if(rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_END_TIMESTAMP] = __tm2; \
DARSHAN_TIMER_INC_NO_OVERLAP(rec_ref->file_rec->fcounters[MPIIO_F_META_TIME], \
__tm1, __tm2, rec_ref->last_meta_end); \
darshan_add_record_ref(&(mpiio_runtime->fh_hash), &__fh, sizeof(MPI_File), rec_ref); \
......@@ -248,9 +243,7 @@ static int enable_dxt_io_trace = 0;
PMPI_Type_size(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_read(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
dxt_mpiio_read(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -280,10 +273,8 @@ static int enable_dxt_io_trace = 0;
if(!rec_ref) break; \
PMPI_Type_size(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_write(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
/* DXT to record detailed write tracing information */ \
dxt_mpiio_write(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -1090,8 +1081,10 @@ int DARSHAN_DECL(MPI_File_close)(MPI_File *fh)
&tmp_fh, sizeof(MPI_File));
if(rec_ref)
{
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_END_TIMESTAMP] = tm2;
DARSHAN_TIMER_INC_NO_OVERLAP(
rec_ref->file_rec->fcounters[MPIIO_F_META_TIME],
tm1, tm2, rec_ref->last_meta_end);
......@@ -1139,10 +1132,8 @@ static void mpiio_runtime_initialize()
}
memset(mpiio_runtime, 0, sizeof(*mpiio_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("DXT_ENABLE_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
/* allow DXT module to initialize if needed */
dxt_mpiio_runtime_initialize();
return;
}
......@@ -1195,7 +1186,7 @@ static struct mpiio_file_record_ref *mpiio_track_new_file_record(
return(rec_ref);
}
static void mpiio_finalize_file_records(void *rec_ref_p)
static void mpiio_finalize_file_records(void *rec_ref_p, void *user_ptr)
{
struct mpiio_file_record_ref *rec_ref =
(struct mpiio_file_record_ref *)rec_ref_p;
......@@ -1273,7 +1264,7 @@ static void mpiio_record_reduction_op(
}
/* min non-zero (if available) value */
for(j=MPIIO_F_OPEN_TIMESTAMP; j<=MPIIO_F_WRITE_START_TIMESTAMP; j++)
for(j=MPIIO_F_OPEN_START_TIMESTAMP; j<=MPIIO_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -1283,7 +1274,7 @@ static void mpiio_record_reduction_op(
}
/* max */
for(j=MPIIO_F_READ_END_TIMESTAMP; j<= MPIIO_F_CLOSE_TIMESTAMP; j++)
for(j=MPIIO_F_OPEN_END_TIMESTAMP; j<= MPIIO_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......@@ -1583,7 +1574,8 @@ static void mpiio_shutdown(
/* perform any final transformations on MPIIO file records before
* writing them out to log file
*/
darshan_iter_record_refs(mpiio_runtime->rec_id_hash, &mpiio_finalize_file_records);
darshan_iter_record_refs(mpiio_runtime->rec_id_hash,
&mpiio_finalize_file_records, NULL);
/* if there are globally shared files, do a shared file reduction */
/* NOTE: the shared file reduction is also skipped if the
......
......@@ -76,7 +76,7 @@ static int my_rank = -1;
PNETCDF_UNLOCK(); \
} while(0)
#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1) do { \
#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1, __tm2) do { \
darshan_record_id rec_id; \
struct pnetcdf_file_record_ref *rec_ref; \
char *newpath; \
......@@ -95,8 +95,10 @@ static int my_rank = -1;
break; \
} \
PMPI_Comm_size(__comm, &comm_size); \
if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] == 0) \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_END_TIMESTAMP] = __tm2; \
if(comm_size == 1) rec_ref->file_rec->counters[PNETCDF_INDEP_OPENS] += 1; \
else rec_ref->file_rec->counters[PNETCDF_COLL_OPENS] += 1; \
darshan_add_record_ref(&(pnetcdf_runtime->ncid_hash), __ncidp, sizeof(int), rec_ref); \
......@@ -112,12 +114,13 @@ int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
{
int ret;
char* tmp;
double tm1;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_create);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_create(comm, path, cmode, info, ncidp);
tm2 = darshan_core_wtime();
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -131,7 +134,7 @@ int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
}
PNETCDF_PRE_RECORD();
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
PNETCDF_POST_RECORD();
}
......@@ -143,12 +146,13 @@ int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
{
int ret;
char* tmp;
double tm1;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_open);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_open(comm, path, omode, info, ncidp);
tm2 = darshan_core_wtime();
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -162,7 +166,7 @@ int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
}
PNETCDF_PRE_RECORD();
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
PNETCDF_POST_RECORD();
}
......@@ -173,18 +177,23 @@ int DARSHAN_DECL(ncmpi_close)(int ncid)
{
struct pnetcdf_file_record_ref *rec_ref;
int ret;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_close);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_close(ncid);
tm2 = darshan_core_wtime();
PNETCDF_PRE_RECORD();
rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->ncid_hash,
&ncid, sizeof(int));
if(rec_ref)
{
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_END_TIMESTAMP] = tm2;
darshan_delete_record_ref(&(pnetcdf_runtime->ncid_hash),
&ncid, sizeof(int));
}
......@@ -302,7 +311,7 @@ static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* min non-zero (if available) value */
for(j=PNETCDF_F_OPEN_TIMESTAMP; j<=PNETCDF_F_OPEN_TIMESTAMP; j++)
for(j=PNETCDF_F_OPEN_START_TIMESTAMP; j<=PNETCDF_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -312,7 +321,7 @@ static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* max */
for(j=PNETCDF_F_CLOSE_TIMESTAMP; j<=PNETCDF_F_CLOSE_TIMESTAMP; j++)
for(j=PNETCDF_F_OPEN_END_TIMESTAMP; j<=PNETCDF_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......
......@@ -29,6 +29,7 @@
#include "utlist.h"
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
......@@ -152,7 +153,7 @@ static void posix_aio_tracker_add(
static struct posix_aio_tracker* posix_aio_tracker_del(
int fd, void *aiocbp);
static void posix_finalize_file_records(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void posix_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void posix_shared_record_variance(
......@@ -165,12 +166,6 @@ static void posix_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **posix_buf, int *posix_buf_sz);
/* extern DXT function defs */
extern void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
extern void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
/* extern function def for querying record name from a STDIO stream */
extern char *darshan_stdio_lookup_record_name(FILE *stream);
......@@ -178,7 +173,6 @@ static struct posix_runtime *posix_runtime = NULL;
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;
static int darshan_mem_alignment = 1;
static int enable_dxt_io_trace = 0;
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
......@@ -259,9 +253,7 @@ static int enable_dxt_io_trace = 0;
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_read(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
dxt_posix_read(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
if(this_offset > rec_ref->last_byte_read) \
rec_ref->file_rec->counters[POSIX_SEQ_READS] += 1; \
if(this_offset == (rec_ref->last_byte_read + 1)) \
......@@ -317,9 +309,7 @@ static int enable_dxt_io_trace = 0;
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_write(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
dxt_posix_write(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
if(this_offset > rec_ref->last_byte_written) \
rec_ref->file_rec->counters[POSIX_SEQ_WRITES] += 1; \
if(this_offset == (rec_ref->last_byte_written + 1)) \
......@@ -1506,10 +1496,8 @@ static void posix_runtime_initialize()
}
memset(posix_runtime, 0, sizeof(*posix_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("DXT_ENABLE_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
/* allow DXT module to initialize if needed */
dxt_posix_runtime_initialize();
return;
}
......@@ -1618,7 +1606,7 @@ static void posix_aio_tracker_add(int fd, void *aiocbp)
return;
}
static void posix_finalize_file_records(void *rec_ref_p)
static void posix_finalize_file_records(void *rec_ref_p, void *user_ptr)
{
struct posix_file_record_ref *rec_ref =
(struct posix_file_record_ref *)rec_ref_p;
......@@ -1971,6 +1959,18 @@ char *darshan_posix_lookup_record_name(int fd)
return(rec_name);
}
static struct darshan_posix_file *darshan_posix_rec_id_to_file(darshan_record_id rec_id)
{
struct posix_file_record_ref *rec_ref;
rec_ref = darshan_lookup_record_ref(posix_runtime->rec_id_hash,
&rec_id, sizeof(darshan_record_id));
if(rec_ref)
return(rec_ref->file_rec);
else
return(NULL);
}
/* posix module shutdown benchmark routine */
void darshan_posix_shutdown_bench_setup(int test_case)
{
......@@ -2067,12 +2067,16 @@ static void posix_shutdown(
POSIX_LOCK();
assert(posix_runtime);
/* allow DXT a chance to filter traces based on dynamic triggers */
dxt_posix_filter_dynamic_traces(darshan_posix_rec_id_to_file);
posix_rec_count = posix_runtime->file_rec_count;
/* perform any final transformations on POSIX file records before
* writing them out to log file
*/
darshan_iter_record_refs(posix_runtime->rec_id_hash, &posix_finalize_file_records);
darshan_iter_record_refs(posix_runtime->rec_id_hash,
&posix_finalize_file_records, NULL);
/* if there are globally shared files, do a shared file reduction */
/* NOTE: the shared file reduction is also skipped if the
......
......@@ -8,10 +8,10 @@
#define __DARSHAN_STDIO_LOG_FORMAT_H
/* current log format version, to support backwards compatibility */
#define DARSHAN_STDIO_VER 1
#define DARSHAN_STDIO_VER 2
#define STDIO_COUNTERS \
/* count of fopens */\
/* count of fopens (INCLUDING fdopen operations) */\
X(STDIO_OPENS) \
/* count of fdopens */\
X(STDIO_FDOPENS) \
......
......@@ -31,6 +31,8 @@ fi
cd ../../;
module list
echo "
<testsuites>
<testsuite name='configure' tests='2' failures='$fcount' time='$thedate'>
......
......@@ -29,7 +29,7 @@ fi
# check results
# in this case we want to confirm that the open counts are zero; cxxpi does not do any IO
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ "$POSIX_OPENS"x != ""x ]; then
echo "Error: Found unexpected POSIX open count of $POSIX_OPENS" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -29,21 +29,21 @@ fi
# check results
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_OPENS" -eq 4 ]; then
echo "Error: STDIO open count of $STDIO_OPENS is incorrect" 1>&2
exit 1
fi
# this will check fprintf counting
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_WRITTEN" -eq 15 ]; then
echo "Error: STDIO bytes written count of $STDIO_BYTES_WRITTEN is incorrect" 1>&2
exit 1
fi
# this will check fscanf counting
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_READ" -eq 15 ]; then
echo "Error: STDIO bytes read count of $STDIO_BYTES_READ is incorrect" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep INDEP_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep INDEP_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -31,22 +31,22 @@ fi
# check at least one counter from each general open/read/write/seek category
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_OPENS" -gt 0 ]; then
echo "Error: STDIO open count of $STDIO_OPENS is incorrect" 1>&2
exit 1
fi
STDIO_SEEKS=`grep STDIO_SEEKS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_SEEKS=`grep STDIO_SEEKS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_SEEKS" -gt 0 ]; then
echo "Error: STDIO open count of $STDIO_SEEKS is incorrect" 1>&2
exit 1
fi
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_WRITTEN" -eq 6 ]; then
echo "Error: STDIO open count of $STDIO_BYTES_WRITTEN is incorrect" 1>&2
exit 1
fi
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_READ" -eq 6 ]; then
echo "Error: STDIO open count of $STDIO_BYTES_READ is incorrect" 1>&2
exit 1
......@@ -55,32 +55,32 @@ fi
# make sure that some of the floating point counters are valid
# use bc for floating point comparison
STDIO_F_OPEN_START_TIMESTAMP=`grep STDIO_F_OPEN_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_OPEN_START_TIMESTAMP=`grep STDIO_F_OPEN_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_OPEN_START_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_OPEN_END_TIMESTAMP=`grep STDIO_F_OPEN_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_OPEN_END_TIMESTAMP=`grep STDIO_F_OPEN_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_OPEN_END_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_META_TIME=`grep STDIO_F_META_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_META_TIME=`grep STDIO_F_META_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_META_TIME > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_WRITE_TIME=`grep STDIO_F_WRITE_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_WRITE_TIME=`grep STDIO_F_WRITE_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_WRITE_TIME > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_CLOSE_START_TIMESTAMP=`grep STDIO_F_CLOSE_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_CLOSE_START_TIMESTAMP=`grep STDIO_F_CLOSE_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_CLOSE_START_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_CLOSE_END_TIMESTAMP=`grep STDIO_F_CLOSE_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_CLOSE_END_TIMESTAMP=`grep STDIO_F_CLOSE_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_CLOSE_END_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
......
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for darshan-util 3.1.6.
# Generated by GNU Autoconf 2.69 for darshan-util 3.1.8.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
......@@ -577,8 +577,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='darshan-util'
PACKAGE_TARNAME='darshan-util'
PACKAGE_VERSION='3.1.6'
PACKAGE_STRING='darshan-util 3.1.6'
PACKAGE_VERSION='3.1.8'
PACKAGE_STRING='darshan-util 3.1.8'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
......@@ -1236,7 +1236,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures darshan-util 3.1.6 to adapt to many kinds of systems.
\`configure' configures darshan-util 3.1.8 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
......@@ -1297,7 +1297,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of darshan-util 3.1.6:";;
short | recursive ) echo "Configuration of darshan-util 3.1.8:";;
esac
cat <<\_ACEOF
......@@ -1393,7 +1393,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
darshan-util configure 3.1.6
darshan-util configure 3.1.8
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -1758,7 +1758,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by darshan-util $as_me 3.1.6, which was
It was created by darshan-util $as_me 3.1.8, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
......@@ -3447,7 +3447,7 @@ ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_inclu
if test "x$ac_cv_header_zlib_h" = xyes; then :
else
as_fn_error $? "z.h not found" "$LINENO" 5
as_fn_error $? "zlib.h not found" "$LINENO" 5
fi
......@@ -4105,7 +4105,7 @@ fi
done
DARSHAN_UTIL_VERSION="3.1.6"
DARSHAN_UTIL_VERSION="3.1.8"
......@@ -4621,7 +4621,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by darshan-util $as_me 3.1.6, which was
This file was extended by darshan-util $as_me 3.1.8, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
......@@ -4683,7 +4683,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
darshan-util config.status 3.1.6
darshan-util config.status 3.1.8
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
......
......@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl You may need to use autoheader as well if changing any DEFINEs
dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-util], [3.1.6])
AC_INIT([darshan-util], [3.1.8])
AC_CONFIG_SRCDIR([darshan-logutils.h])
AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-util-config.h)
......
......@@ -258,7 +258,7 @@ int main(int argc, char **argv)
struct darshan_mnt_info *mnt_data_array;
struct darshan_name_record_ref *name_hash = NULL;
struct darshan_name_record_ref *ref, *tmp;
char *mod_buf;
char *mod_buf, *tmp_mod_buf;
enum darshan_comp_type comp_type;
int bzip2;
int obfuscate;
......@@ -388,26 +388,46 @@ int main(int argc, char **argv)
continue;
}
/* we have module data to convert */
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
/* for dxt, don't use static record buffer and instead have
* darshan-logutils malloc us memory for the trace data
*/
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
{
tmp_mod_buf = NULL;
}
else
{
tmp_mod_buf = mod_buf;
memset(tmp_mod_buf, 0, DEF_MOD_BUF_SIZE);
}
/* loop over each of the module's records and convert */
while((ret = mod_logutils[i]->log_get_record(infile, (void **)&mod_buf)) == 1)
while((ret = mod_logutils[i]->log_get_record(infile, (void **)&tmp_mod_buf)) == 1)
{
base_rec = (struct darshan_base_record *)mod_buf;
base_rec = (struct darshan_base_record *)tmp_mod_buf;
if(!hash || hash == base_rec->id)
{
ret = mod_logutils[i]->log_put_record(outfile, mod_buf);
ret = mod_logutils[i]->log_put_record(outfile, tmp_mod_buf);
if(ret < 0)
{
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
free(tmp_mod_buf);
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
{
free(tmp_mod_buf);
tmp_mod_buf = NULL;
}
else
{
memset(tmp_mod_buf, 0, DEF_MOD_BUF_SIZE);
}
}
if(ret < 0)
......
......@@ -30,6 +30,8 @@ char *hdf5_f_counter_names[] = {
};
#undef X
#define DARSHAN_HDF5_FILE_SIZE_1 40
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
static void darshan_log_print_hdf5_file(void *file_rec,
......@@ -52,6 +54,7 @@ struct darshan_mod_logutil_funcs hdf5_logutils =
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
{
struct darshan_hdf5_file *file = *((struct darshan_hdf5_file **)hdf5_buf_p);
int rec_len;
int i;
int ret;
......@@ -65,12 +68,42 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
return(-1);
}
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, file,
sizeof(struct darshan_hdf5_file));
if(fd->mod_ver[DARSHAN_HDF5_MOD] == DARSHAN_HDF5_VER)
{
/* log format is in current version, so we don't need to do any
* translation of counters while reading
*/
rec_len = sizeof(struct darshan_hdf5_file);
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, file, rec_len);
}
else
{
char scratch[1024] = {0};
char *src_p, *dest_p;
int len;
rec_len = DARSHAN_HDF5_FILE_SIZE_1;
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, scratch, rec_len);
if(ret != rec_len)
goto exit;
/* upconvert version 1 to version 2 in-place */
dest_p = scratch + (sizeof(struct darshan_base_record) +
(1 * sizeof(int64_t)) + (3 * sizeof(double)));
src_p = dest_p - (2 * sizeof(double));
len = sizeof(double);
memmove(dest_p, src_p, len);
/* set F_CLOSE_START and F_OPEN_END to -1 */
*((double *)src_p) = -1;
*((double *)(src_p + sizeof(double))) = -1;
memcpy(file, scratch, sizeof(struct darshan_hdf5_file));
}
exit:
if(*hdf5_buf_p == NULL)
{
</