...
 
Commits (20)
stages:
- build
- test
build_darshan:
tags:
- shell
- ecp-theta
stage: build
script:
- source darshan-test/automated/common/setup.sh
- source darshan-test/automated/common/configure.sh
- source darshan-test/automated/common/build.sh
test_darshan:
variables:
# XXX this info is hard-coded in our regression scripts
#SCHEDULER_PARAMETERS: "-A radix-io -n 1 -t 10 -q default"
tags:
- batch
- ecp-theta
stage: test
script:
- darshan-test/regression/run-all.sh install scratch cray-module-alcf
......@@ -4,6 +4,8 @@ Darshan Release Change Log
Darshan-3.1.7
=============
* bug fix for handling DXT data in darshan-convert utility. Bug reported
by Mahzad Khoshlessan.
* bug fix for darshan-parser backwards compatibility: Darshan logs generated
by Darshan versions prior to 3.1.0 may have included stdio counters that
were not properly up-converted. Reported by Teng Wang.
......@@ -11,6 +13,9 @@ Darshan-3.1.7
darshan-job-summary.pl when both posix and stdio access is present
* added wrapper for __open_2(), bug reported by Cormac Garvey in which open
calls are not intercepted with some versions of glibc/gcc
* added an instrumentation module for the MDHIM key/val storage system
* added support for properly instrumenting dup(), fileno(), fdopen(), and
rename() calls
Darshan-3.1.6
=============
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_HDF5_LOG_FORMAT_H
/* current HDF5 log format version */
#define DARSHAN_HDF5_VER 1
#define DARSHAN_HDF5_VER 2
#define HDF5_COUNTERS \
/* count of HDF5 opens */\
......@@ -18,9 +18,13 @@
#define HDF5_F_COUNTERS \
/* timestamp of first open */\
X(HDF5_F_OPEN_TIMESTAMP) \
X(HDF5_F_OPEN_START_TIMESTAMP) \
/* timestamp of first close */\
X(HDF5_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(HDF5_F_OPEN_END_TIMESTAMP) \
/* timestamp of last close */\
X(HDF5_F_CLOSE_TIMESTAMP) \
X(HDF5_F_CLOSE_END_TIMESTAMP) \
/* end of counters*/\
X(HDF5_F_NUM_INDICES)
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_MPIIO_LOG_FORMAT_H
/* current MPI-IO log format version */
#define DARSHAN_MPIIO_VER 2
#define DARSHAN_MPIIO_VER 3
/* TODO: maybe use a counter to track cases in which a derived datatype is used? */
......@@ -92,17 +92,21 @@
#define MPIIO_F_COUNTERS \
/* timestamp of first open */\
X(MPIIO_F_OPEN_TIMESTAMP) \
X(MPIIO_F_OPEN_START_TIMESTAMP) \
/* timestamp of first read */\
X(MPIIO_F_READ_START_TIMESTAMP) \
/* timestamp of first write */\
X(MPIIO_F_WRITE_START_TIMESTAMP) \
/* timestamp of first close */\
X(MPIIO_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(MPIIO_F_OPEN_END_TIMESTAMP) \
/* timestamp of last read */\
X(MPIIO_F_READ_END_TIMESTAMP) \
/* timestamp of last write */\
X(MPIIO_F_WRITE_END_TIMESTAMP) \
/* timestamp of last close */\
X(MPIIO_F_CLOSE_TIMESTAMP) \
X(MPIIO_F_CLOSE_END_TIMESTAMP) \
/* cumulative MPI-IO read time */\
X(MPIIO_F_READ_TIME) \
/* cumulative MPI-IO write time */\
......
......@@ -8,7 +8,7 @@
#define __DARSHAN_PNETCDF_LOG_FORMAT_H
/* current PNETCDF log format version */
#define DARSHAN_PNETCDF_VER 1
#define DARSHAN_PNETCDF_VER 2
#define PNETCDF_COUNTERS \
/* count of PNETCDF independent opens */\
......@@ -20,9 +20,13 @@
#define PNETCDF_F_COUNTERS \
/* timestamp of first open */\
X(PNETCDF_F_OPEN_TIMESTAMP) \
X(PNETCDF_F_OPEN_START_TIMESTAMP) \
/* timestamp of first close */\
X(PNETCDF_F_CLOSE_START_TIMESTAMP) \
/* timestamp of last open */\
X(PNETCDF_F_OPEN_END_TIMESTAMP) \
/* timestamp of last close */\
X(PNETCDF_F_CLOSE_TIMESTAMP) \
X(PNETCDF_F_CLOSE_END_TIMESTAMP) \
/* end of counters*/\
X(PNETCDF_F_NUM_INDICES)
......
......@@ -10,7 +10,7 @@
#define DARSHAN_POSIX_VER 4
#define POSIX_COUNTERS \
/* count of posix opens */\
/* count of posix opens (INCLUDING fileno and dup operations) */\
X(POSIX_OPENS) \
/* count of number of filenos */\
X(POSIX_FILENOS) \
......
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for darshan-runtime 3.1.6.
# Generated by GNU Autoconf 2.69 for darshan-runtime 3.1.7.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
......@@ -577,8 +577,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='darshan-runtime'
PACKAGE_TARNAME='darshan-runtime'
PACKAGE_VERSION='3.1.6'
PACKAGE_STRING='darshan-runtime 3.1.6'
PACKAGE_VERSION='3.1.7'
PACKAGE_STRING='darshan-runtime 3.1.7'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
......@@ -667,6 +667,7 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
......@@ -753,6 +754,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
......@@ -1005,6 +1007,15 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
......@@ -1142,7 +1153,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir
libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
......@@ -1255,7 +1266,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures darshan-runtime 3.1.6 to adapt to many kinds of systems.
\`configure' configures darshan-runtime 3.1.7 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
......@@ -1295,6 +1306,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
......@@ -1316,7 +1328,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of darshan-runtime 3.1.6:";;
short | recursive ) echo "Configuration of darshan-runtime 3.1.7:";;
esac
cat <<\_ACEOF
......@@ -1431,7 +1443,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
darshan-runtime configure 3.1.6
darshan-runtime configure 3.1.7
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -1783,7 +1795,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by darshan-runtime $as_me 3.1.6, which was
It was created by darshan-runtime $as_me 3.1.7, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
......@@ -3496,7 +3508,7 @@ ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_inclu
if test "x$ac_cv_header_zlib_h" = xyes; then :
else
as_fn_error $? "z.h not found" "$LINENO" 5
as_fn_error $? "zlib.h not found" "$LINENO" 5
fi
......@@ -4420,7 +4432,7 @@ else
MPICH_LIB_OLD=0
fi
DARSHAN_VERSION="3.1.6"
DARSHAN_VERSION="3.1.7"
......@@ -4944,7 +4956,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by darshan-runtime $as_me 3.1.6, which was
This file was extended by darshan-runtime $as_me 3.1.7, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
......@@ -5006,7 +5018,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
darshan-runtime config.status 3.1.6
darshan-runtime config.status 3.1.7
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
......
......@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl You may need to use autoheader as well if changing any DEFINEs
dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-runtime], [3.1.6])
AC_INIT([darshan-runtime], [3.1.7])
AC_CONFIG_SRCDIR([darshan.h])
AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-runtime-config.h)
......
......@@ -210,12 +210,16 @@ void darshan_clear_record_refs(
*
* Iterate each record reference stored in the hash table pointed
* to by 'hash_head' and perform the given action 'iter_action'.
* The action function takes a single pointer which points to the
* corresponding record reference pointer.
* The action function takes two pointers as input: the first
* points to the corresponding record reference pointer and the
* second is a user-supplied pointer provided for the action.
* 'user_ptr' is the user-supplied pointer that will be passed
* as the 2nd argument to 'iter_action'.
*/
void darshan_iter_record_refs(
void *hash_head,
void (*iter_action)(void *));
void (*iter_action)(void *, void *),
void *user_ptr);
/* darshan_clean_file_path()
*
......
/*
* Copyright (C) 2019 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_DXT_H
#define __DARSHAN_DXT_H
/* dxt_load_trigger_conf()
*
* DXT function exposed to Darshan core to read in any trace triggers
* from the file path in 'trigger_conf_path' before module
* initialization occurs.
*/
void dxt_load_trigger_conf(
char *trigger_conf_path);
/* dxt_posix_runtime_initialize()
*
* DXT function exposed to POSIX module for initializing DXT-POSIX runtime.
*/
void dxt_posix_runtime_initialize(void);
/* dxt_mpiio_runtime_initialize()
*
* DXT function exposed to MPIIO module for initializing DXT-MPIIO runtime.
*/
void dxt_mpiio_runtime_initialize(void);
/* dxt_posix_write(), dxt_posix_read()
*
* DXT function to trace a POSIX write/read call to file record 'rec_id',
* at offset 'offset' and with 'length' size. 'start_time' and 'end_time'
* are starting and ending timestamps for the operation, respectively.
*/
void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
/* dxt_mpiio_write(), dxt_mpiio_read()
*
* DXT function to trace a MPIIO write/read call to file record 'rec_id',
* with 'length' size. 'start_time' and 'end_time' are starting and ending
* timestamps for the operation, respectively.
*/
void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
void dxt_posix_filter_dynamic_traces(
struct darshan_posix_file *(*rec_id_to_psx_file)(darshan_record_id));
#endif /* __DARSHAN_DXT_H */
......@@ -103,7 +103,8 @@ void darshan_clear_record_refs(void **hash_head_p, int free_flag)
return;
}
void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *))
void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *, void *),
void *user_ptr)
{
struct darshan_record_ref_tracker *ref_tracker, *tmp;
struct darshan_record_ref_tracker *ref_tracker_head =
......@@ -114,7 +115,7 @@ void darshan_iter_record_refs(void *hash_head, void (*iter_action)(void *))
*/
HASH_ITER(hlink, ref_tracker_head, ref_tracker, tmp)
{
iter_action(ref_tracker->rec_ref_p);
iter_action(ref_tracker->rec_ref_p, user_ptr);
}
return;
......
......@@ -34,6 +34,7 @@
#include "darshan.h"
#include "darshan-core.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
#ifdef DARSHAN_LUSTRE
#include <lustre/lustre_user.h>
......@@ -308,6 +309,13 @@ void darshan_core_initialize(int argc, char **argv)
/* collect information about command line and mounted file systems */
darshan_get_exe_and_mounts(init_core, argc, argv);
/* determine if/when DXT should be enabled by looking for triggers */
char *trigger_conf = getenv("DARSHAN_DXT_TRIGGER_CONF_PATH");
if(trigger_conf)
{
dxt_load_trigger_conf(trigger_conf);
}
/* if darshan was successfully initialized, set the global pointer
* and bootstrap any modules with static initialization routines
*/
......
This diff is collapsed.
......@@ -90,7 +90,7 @@ static int my_rank = -1;
HDF5_UNLOCK(); \
} while(0)
#define HDF5_RECORD_OPEN(__ret, __path, __tm1) do { \
#define HDF5_RECORD_OPEN(__ret, __path, __tm1, __tm2) do { \
darshan_record_id rec_id; \
struct hdf5_file_record_ref *rec_ref; \
char *newpath; \
......@@ -107,8 +107,10 @@ static int my_rank = -1;
if(newpath != __path) free(newpath); \
break; \
} \
if(rec_ref->file_rec->fcounters[HDF5_F_OPEN_TIMESTAMP] == 0) \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[HDF5_F_OPEN_END_TIMESTAMP] = __tm2; \
rec_ref->file_rec->counters[HDF5_OPENS] += 1; \
darshan_add_record_ref(&(hdf5_runtime->hid_hash), &__ret, sizeof(hid_t), rec_ref); \
if(newpath != __path) free(newpath); \
......@@ -123,7 +125,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
{
hid_t ret;
char* tmp;
double tm1;
double tm1, tm2;
unsigned majnum, minnum, relnum;
H5get_libversion(&majnum, &minnum, &relnum);
......@@ -146,6 +148,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
tm1 = darshan_core_wtime();
ret = __real_H5Fcreate(filename, flags, create_plist, access_plist);
tm2 = darshan_core_wtime();
if(ret >= 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -159,7 +162,7 @@ hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
}
HDF5_PRE_RECORD();
HDF5_RECORD_OPEN(ret, filename, tm1);
HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
HDF5_POST_RECORD();
}
......@@ -171,7 +174,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
{
hid_t ret;
char* tmp;
double tm1;
double tm1, tm2;
unsigned majnum, minnum, relnum;
H5get_libversion(&majnum, &minnum, &relnum);
......@@ -194,6 +197,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
tm1 = darshan_core_wtime();
ret = __real_H5Fopen(filename, flags, access_plist);
tm2 = darshan_core_wtime();
if(ret >= 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -207,7 +211,7 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
}
HDF5_PRE_RECORD();
HDF5_RECORD_OPEN(ret, filename, tm1);
HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
HDF5_POST_RECORD();
}
......@@ -218,19 +222,24 @@ hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
{
struct hdf5_file_record_ref *rec_ref;
double tm1, tm2;
herr_t ret;
MAP_OR_FAIL(H5Fclose);
tm1 = darshan_core_wtime();
ret = __real_H5Fclose(file_id);
tm2 = darshan_core_wtime();
HDF5_PRE_RECORD();
rec_ref = darshan_lookup_record_ref(hdf5_runtime->hid_hash,
&file_id, sizeof(hid_t));
if(rec_ref)
{
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[HDF5_F_CLOSE_END_TIMESTAMP] = tm2;
darshan_delete_record_ref(&(hdf5_runtime->hid_hash),
&file_id, sizeof(hid_t));
}
......@@ -349,7 +358,7 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* min non-zero (if available) value */
for(j=HDF5_F_OPEN_TIMESTAMP; j<=HDF5_F_OPEN_TIMESTAMP; j++)
for(j=HDF5_F_OPEN_START_TIMESTAMP; j<=HDF5_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -359,7 +368,7 @@ static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* max */
for(j=HDF5_F_CLOSE_TIMESTAMP; j<=HDF5_F_CLOSE_TIMESTAMP; j++)
for(j=HDF5_F_OPEN_END_TIMESTAMP; j<=HDF5_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......
......@@ -27,9 +27,9 @@
static void lustre_runtime_initialize(
void);
static void lustre_subtract_shared_rec_size(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void lustre_set_rec_ref_pointers(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static int lustre_record_compare(
const void* a_p, const void* b_p);
int sort_lustre_records(
......@@ -266,7 +266,7 @@ static void lustre_shutdown(
if (my_rank != 0)
{
darshan_iter_record_refs(lustre_runtime->record_id_hash,
&lustre_subtract_shared_rec_size);
&lustre_subtract_shared_rec_size, NULL);
}
}
......@@ -282,7 +282,7 @@ static void lustre_shutdown(
return;
}
static void lustre_subtract_shared_rec_size(void *rec_ref_p)
static void lustre_subtract_shared_rec_size(void *rec_ref_p, void *user_ptr)
{
struct lustre_record_ref *l_rec_ref = (struct lustre_record_ref *)rec_ref_p;
......@@ -291,7 +291,7 @@ static void lustre_subtract_shared_rec_size(void *rec_ref_p)
LUSTRE_RECORD_SIZE( l_rec_ref->record->counters[LUSTRE_STRIPE_WIDTH] );
}
static void lustre_set_rec_ref_pointers(void *rec_ref_p)
static void lustre_set_rec_ref_pointers(void *rec_ref_p, void *user_ptr)
{
lustre_runtime->record_ref_array[lustre_runtime->record_ref_array_ndx] = rec_ref_p;
lustre_runtime->record_ref_array_ndx++;
......@@ -356,7 +356,7 @@ int sort_lustre_records()
/* build the array of record reference pointers we want to sort */
darshan_iter_record_refs(lustre_runtime->record_id_hash,
&lustre_set_rec_ref_pointers);
&lustre_set_rec_ref_pointers, NULL);
/* qsort breaks the hash table, so delete it now to free its memory buffers
* and prevent later confusion */
......
......@@ -578,7 +578,7 @@ static void mdhim_shutdown(
if (my_rank != 0)
{
darshan_iter_record_refs(mdhim_runtime->rec_id_hash,
&mdhim_subtract_shared_rec_size);
&mdhim_subtract_shared_rec_size, NULL);
}
}
*mdhim_buf_sz = mdhim_rec_count * mdhim_runtime->record_buffer_size;
......
......@@ -24,6 +24,7 @@
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
DARSHAN_FORWARD_DECL(PMPI_File_close, int, (MPI_File *fh));
DARSHAN_FORWARD_DECL(PMPI_File_iread_at, int, (MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST *request));
......@@ -159,7 +160,7 @@ static void mpiio_runtime_initialize(
static struct mpiio_file_record_ref *mpiio_track_new_file_record(
darshan_record_id rec_id, const char *path);
static void mpiio_finalize_file_records(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void mpiio_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void mpiio_shared_record_variance(
......@@ -172,16 +173,9 @@ static void mpiio_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **mpiio_buf, int *mpiio_buf_sz);
/* extern DXT function defs */
extern void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
extern void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
static struct mpiio_runtime *mpiio_runtime = NULL;
static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;
static int enable_dxt_io_trace = 0;
#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
......@@ -229,9 +223,10 @@ static int enable_dxt_io_trace = 0;
rec_ref->file_rec->counters[MPIIO_COLL_OPENS] += 1; \
if(__info != MPI_INFO_NULL) \
rec_ref->file_rec->counters[MPIIO_HINTS] += 1; \
if(rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[MPIIO_F_OPEN_END_TIMESTAMP] = __tm2; \
DARSHAN_TIMER_INC_NO_OVERLAP(rec_ref->file_rec->fcounters[MPIIO_F_META_TIME], \
__tm1, __tm2, rec_ref->last_meta_end); \
darshan_add_record_ref(&(mpiio_runtime->fh_hash), &__fh, sizeof(MPI_File), rec_ref); \
......@@ -248,9 +243,7 @@ static int enable_dxt_io_trace = 0;
PMPI_Type_size(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_read(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
dxt_mpiio_read(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -280,10 +273,8 @@ static int enable_dxt_io_trace = 0;
if(!rec_ref) break; \
PMPI_Type_size(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_write(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
/* DXT to record detailed write tracing information */ \
dxt_mpiio_write(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -1090,8 +1081,10 @@ int DARSHAN_DECL(MPI_File_close)(MPI_File *fh)
&tmp_fh, sizeof(MPI_File));
if(rec_ref)
{
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[MPIIO_F_CLOSE_END_TIMESTAMP] = tm2;
DARSHAN_TIMER_INC_NO_OVERLAP(
rec_ref->file_rec->fcounters[MPIIO_F_META_TIME],
tm1, tm2, rec_ref->last_meta_end);
......@@ -1139,10 +1132,8 @@ static void mpiio_runtime_initialize()
}
memset(mpiio_runtime, 0, sizeof(*mpiio_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("DXT_ENABLE_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
/* allow DXT module to initialize if needed */
dxt_mpiio_runtime_initialize();
return;
}
......@@ -1195,7 +1186,7 @@ static struct mpiio_file_record_ref *mpiio_track_new_file_record(
return(rec_ref);
}
static void mpiio_finalize_file_records(void *rec_ref_p)
static void mpiio_finalize_file_records(void *rec_ref_p, void *user_ptr)
{
struct mpiio_file_record_ref *rec_ref =
(struct mpiio_file_record_ref *)rec_ref_p;
......@@ -1273,7 +1264,7 @@ static void mpiio_record_reduction_op(
}
/* min non-zero (if available) value */
for(j=MPIIO_F_OPEN_TIMESTAMP; j<=MPIIO_F_WRITE_START_TIMESTAMP; j++)
for(j=MPIIO_F_OPEN_START_TIMESTAMP; j<=MPIIO_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -1283,7 +1274,7 @@ static void mpiio_record_reduction_op(
}
/* max */
for(j=MPIIO_F_READ_END_TIMESTAMP; j<= MPIIO_F_CLOSE_TIMESTAMP; j++)
for(j=MPIIO_F_OPEN_END_TIMESTAMP; j<= MPIIO_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......@@ -1583,7 +1574,8 @@ static void mpiio_shutdown(
/* perform any final transformations on MPIIO file records before
* writing them out to log file
*/
darshan_iter_record_refs(mpiio_runtime->rec_id_hash, &mpiio_finalize_file_records);
darshan_iter_record_refs(mpiio_runtime->rec_id_hash,
&mpiio_finalize_file_records, NULL);
/* if there are globally shared files, do a shared file reduction */
/* NOTE: the shared file reduction is also skipped if the
......
......@@ -76,7 +76,7 @@ static int my_rank = -1;
PNETCDF_UNLOCK(); \
} while(0)
#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1) do { \
#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1, __tm2) do { \
darshan_record_id rec_id; \
struct pnetcdf_file_record_ref *rec_ref; \
char *newpath; \
......@@ -95,8 +95,10 @@ static int my_rank = -1;
break; \
} \
PMPI_Comm_size(__comm, &comm_size); \
if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] == 0) \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] = __tm1; \
if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] == 0 || \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] > __tm1) \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] = __tm1; \
rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_END_TIMESTAMP] = __tm2; \
if(comm_size == 1) rec_ref->file_rec->counters[PNETCDF_INDEP_OPENS] += 1; \
else rec_ref->file_rec->counters[PNETCDF_COLL_OPENS] += 1; \
darshan_add_record_ref(&(pnetcdf_runtime->ncid_hash), __ncidp, sizeof(int), rec_ref); \
......@@ -112,12 +114,13 @@ int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
{
int ret;
char* tmp;
double tm1;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_create);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_create(comm, path, cmode, info, ncidp);
tm2 = darshan_core_wtime();
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -131,7 +134,7 @@ int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
}
PNETCDF_PRE_RECORD();
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
PNETCDF_POST_RECORD();
}
......@@ -143,12 +146,13 @@ int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
{
int ret;
char* tmp;
double tm1;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_open);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_open(comm, path, omode, info, ncidp);
tm2 = darshan_core_wtime();
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
......@@ -162,7 +166,7 @@ int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
}
PNETCDF_PRE_RECORD();
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
PNETCDF_POST_RECORD();
}
......@@ -173,18 +177,23 @@ int DARSHAN_DECL(ncmpi_close)(int ncid)
{
struct pnetcdf_file_record_ref *rec_ref;
int ret;
double tm1, tm2;
MAP_OR_FAIL(ncmpi_close);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_close(ncid);
tm2 = darshan_core_wtime();
PNETCDF_PRE_RECORD();
rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->ncid_hash,
&ncid, sizeof(int));
if(rec_ref)
{
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
if(rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] == 0 ||
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] > tm1)
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] = tm1;
rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_END_TIMESTAMP] = tm2;
darshan_delete_record_ref(&(pnetcdf_runtime->ncid_hash),
&ncid, sizeof(int));
}
......@@ -302,7 +311,7 @@ static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* min non-zero (if available) value */
for(j=PNETCDF_F_OPEN_TIMESTAMP; j<=PNETCDF_F_OPEN_TIMESTAMP; j++)
for(j=PNETCDF_F_OPEN_START_TIMESTAMP; j<=PNETCDF_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
......@@ -312,7 +321,7 @@ static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* max */
for(j=PNETCDF_F_CLOSE_TIMESTAMP; j<=PNETCDF_F_CLOSE_TIMESTAMP; j++)
for(j=PNETCDF_F_OPEN_END_TIMESTAMP; j<=PNETCDF_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
......
......@@ -29,6 +29,7 @@
#include "utlist.h"
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-dxt.h"
#ifndef HAVE_OFF64_T
typedef int64_t off64_t;
......@@ -152,7 +153,7 @@ static void posix_aio_tracker_add(
static struct posix_aio_tracker* posix_aio_tracker_del(
int fd, void *aiocbp);
static void posix_finalize_file_records(
void *rec_ref_p);
void *rec_ref_p, void *user_ptr);
static void posix_record_reduction_op(
void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void posix_shared_record_variance(
......@@ -165,12 +166,6 @@ static void posix_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **posix_buf, int *posix_buf_sz);
/* extern DXT function defs */
extern void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
extern void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
/* extern function def for querying record name from a STDIO stream */
extern char *darshan_stdio_lookup_record_name(FILE *stream);
......@@ -178,7 +173,6 @@ static struct posix_runtime *posix_runtime = NULL;
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;
static int darshan_mem_alignment = 1;
static int enable_dxt_io_trace = 0;
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
......@@ -259,9 +253,7 @@ static int enable_dxt_io_trace = 0;
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_read(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
dxt_posix_read(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
if(this_offset > rec_ref->last_byte_read) \
rec_ref->file_rec->counters[POSIX_SEQ_READS] += 1; \
if(this_offset == (rec_ref->last_byte_read + 1)) \
......@@ -317,9 +309,7 @@ static int enable_dxt_io_trace = 0;
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_write(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
dxt_posix_write(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
if(this_offset > rec_ref->last_byte_written) \
rec_ref->file_rec->counters[POSIX_SEQ_WRITES] += 1; \
if(this_offset == (rec_ref->last_byte_written + 1)) \
......@@ -1506,10 +1496,8 @@ static void posix_runtime_initialize()
}
memset(posix_runtime, 0, sizeof(*posix_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("DXT_ENABLE_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
/* allow DXT module to initialize if needed */
dxt_posix_runtime_initialize();
return;
}
......@@ -1618,7 +1606,7 @@ static void posix_aio_tracker_add(int fd, void *aiocbp)
return;
}
static void posix_finalize_file_records(void *rec_ref_p)
static void posix_finalize_file_records(void *rec_ref_p, void *user_ptr)
{
struct posix_file_record_ref *rec_ref =
(struct posix_file_record_ref *)rec_ref_p;
......@@ -1971,6 +1959,18 @@ char *darshan_posix_lookup_record_name(int fd)
return(rec_name);
}
static struct darshan_posix_file *darshan_posix_rec_id_to_file(darshan_record_id rec_id)
{
struct posix_file_record_ref *rec_ref;
rec_ref = darshan_lookup_record_ref(posix_runtime->rec_id_hash,
&rec_id, sizeof(darshan_record_id));
if(rec_ref)
return(rec_ref->file_rec);
else
return(NULL);
}
/* posix module shutdown benchmark routine */
void darshan_posix_shutdown_bench_setup(int test_case)
{
......@@ -2067,12 +2067,16 @@ static void posix_shutdown(
POSIX_LOCK();
assert(posix_runtime);
/* allow DXT a chance to filter traces based on dynamic triggers */
dxt_posix_filter_dynamic_traces(darshan_posix_rec_id_to_file);
posix_rec_count = posix_runtime->file_rec_count;
/* perform any final transformations on POSIX file records before
* writing them out to log file
*/
darshan_iter_record_refs(posix_runtime->rec_id_hash, &posix_finalize_file_records);
darshan_iter_record_refs(posix_runtime->rec_id_hash,
&posix_finalize_file_records, NULL);
/* if there are globally shared files, do a shared file reduction */
/* NOTE: the shared file reduction is also skipped if the
......
......@@ -8,10 +8,10 @@
#define __DARSHAN_STDIO_LOG_FORMAT_H
/* current log format version, to support backwards compatibility */
#define DARSHAN_STDIO_VER 1
#define DARSHAN_STDIO_VER 2
#define STDIO_COUNTERS \
/* count of fopens */\
/* count of fopens (INCLUDING fdopen operations) */\
X(STDIO_OPENS) \
/* count of fdopens */\
X(STDIO_FDOPENS) \
......
......@@ -29,7 +29,7 @@ fi
# check results
# in this case we want to confirm that the open counts are zero; cxxpi does not do any IO
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ "$POSIX_OPENS"x != ""x ]; then
echo "Error: Found unexpected POSIX open count of $POSIX_OPENS" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep COLL_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -29,21 +29,21 @@ fi
# check results
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_OPENS" -eq 4 ]; then
echo "Error: STDIO open count of $STDIO_OPENS is incorrect" 1>&2
exit 1
fi
# this will check fprintf counting
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_WRITTEN" -eq 15 ]; then
echo "Error: STDIO bytes written count of $STDIO_BYTES_WRITTEN is incorrect" 1>&2
exit 1
fi
# this will check fscanf counting
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_READ" -eq 15 ]; then
echo "Error: STDIO bytes read count of $STDIO_BYTES_READ is incorrect" 1>&2
exit 1
......
......@@ -29,12 +29,12 @@ fi
# check results
# in this case we want to confirm that both the MPI and POSIX open counters were triggered
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
POSIX_OPENS=`grep POSIX_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$POSIX_OPENS" -gt 0 ]; then
echo "Error: POSIX open count of $POSIX_OPENS is incorrect" 1>&2
exit 1
fi
MPI_OPENS=`grep INDEP_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |cut -f 5`
MPI_OPENS=`grep INDEP_OPENS $DARSHAN_TMP/${PROG}.darshan.txt |grep -vE "^#" |cut -f 5`
if [ ! "$MPI_OPENS" -gt 0 ]; then
echo "Error: MPI open count of $MPI_OPENS is incorrect" 1>&2
exit 1
......
......@@ -31,22 +31,22 @@ fi
# check at least one counter from each general open/read/write/seek category
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_OPENS=`grep STDIO_OPENS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_OPENS" -gt 0 ]; then
echo "Error: STDIO open count of $STDIO_OPENS is incorrect" 1>&2
exit 1
fi
STDIO_SEEKS=`grep STDIO_SEEKS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_SEEKS=`grep STDIO_SEEKS $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_SEEKS" -gt 0 ]; then
echo "Error: STDIO open count of $STDIO_SEEKS is incorrect" 1>&2
exit 1
fi
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_WRITTEN=`grep STDIO_BYTES_WRITTEN $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_WRITTEN" -eq 6 ]; then
echo "Error: STDIO open count of $STDIO_BYTES_WRITTEN is incorrect" 1>&2
exit 1
fi
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_BYTES_READ=`grep STDIO_BYTES_READ $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! "$STDIO_BYTES_READ" -eq 6 ]; then
echo "Error: STDIO open count of $STDIO_BYTES_READ is incorrect" 1>&2
exit 1
......@@ -55,32 +55,32 @@ fi
# make sure that some of the floating point counters are valid
# use bc for floating point comparison
STDIO_F_OPEN_START_TIMESTAMP=`grep STDIO_F_OPEN_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_OPEN_START_TIMESTAMP=`grep STDIO_F_OPEN_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_OPEN_START_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_OPEN_END_TIMESTAMP=`grep STDIO_F_OPEN_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_OPEN_END_TIMESTAMP=`grep STDIO_F_OPEN_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_OPEN_END_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_META_TIME=`grep STDIO_F_META_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_META_TIME=`grep STDIO_F_META_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_META_TIME > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_WRITE_TIME=`grep STDIO_F_WRITE_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_WRITE_TIME=`grep STDIO_F_WRITE_TIME $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_WRITE_TIME > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_CLOSE_START_TIMESTAMP=`grep STDIO_F_CLOSE_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_CLOSE_START_TIMESTAMP=`grep STDIO_F_CLOSE_START_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_CLOSE_START_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
fi
STDIO_F_CLOSE_END_TIMESTAMP=`grep STDIO_F_CLOSE_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
STDIO_F_CLOSE_END_TIMESTAMP=`grep STDIO_F_CLOSE_END_TIMESTAMP $DARSHAN_TMP/${PROG}.darshan.txt | grep -vE "^#" | grep -vE "STDIN|STDOUT|STDERR" | cut -f 5`
if [ ! $(echo "$STDIO_F_CLOSE_END_TIMESTAMP > 0" | bc -l) ]; then
echo "Error: counter is incorrect" 1>&2
exit 1
......
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for darshan-util 3.1.6.
# Generated by GNU Autoconf 2.69 for darshan-util 3.1.7.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
......@@ -577,8 +577,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='darshan-util'
PACKAGE_TARNAME='darshan-util'
PACKAGE_VERSION='3.1.6'
PACKAGE_STRING='darshan-util 3.1.6'
PACKAGE_VERSION='3.1.7'
PACKAGE_STRING='darshan-util 3.1.7'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
......@@ -661,6 +661,7 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
......@@ -734,6 +735,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
......@@ -986,6 +988,15 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
......@@ -1123,7 +1134,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir
libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
......@@ -1236,7 +1247,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures darshan-util 3.1.6 to adapt to many kinds of systems.
\`configure' configures darshan-util 3.1.7 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
......@@ -1276,6 +1287,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
......@@ -1297,7 +1309,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of darshan-util 3.1.6:";;
short | recursive ) echo "Configuration of darshan-util 3.1.7:";;
esac
cat <<\_ACEOF
......@@ -1393,7 +1405,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
darshan-util configure 3.1.6
darshan-util configure 3.1.7
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -1758,7 +1770,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by darshan-util $as_me 3.1.6, which was
It was created by darshan-util $as_me 3.1.7, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
......@@ -3447,7 +3459,7 @@ ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_inclu
if test "x$ac_cv_header_zlib_h" = xyes; then :
else
as_fn_error $? "z.h not found" "$LINENO" 5
as_fn_error $? "zlib.h not found" "$LINENO" 5
fi
......@@ -4105,7 +4117,7 @@ fi
done
DARSHAN_UTIL_VERSION="3.1.6"
DARSHAN_UTIL_VERSION="3.1.7"
......@@ -4621,7 +4633,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by darshan-util $as_me 3.1.6, which was
This file was extended by darshan-util $as_me 3.1.7, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
......@@ -4683,7 +4695,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
darshan-util config.status 3.1.6
darshan-util config.status 3.1.7
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
......
......@@ -5,7 +5,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl You may need to use autoheader as well if changing any DEFINEs
dnl sanity checks, output header, location of scripts used here
AC_INIT([darshan-util], [3.1.6])
AC_INIT([darshan-util], [3.1.7])
AC_CONFIG_SRCDIR([darshan-logutils.h])
AC_CONFIG_AUX_DIR(../maint/config)
AC_CONFIG_HEADER(darshan-util-config.h)
......
......@@ -258,7 +258,7 @@ int main(int argc, char **argv)
struct darshan_mnt_info *mnt_data_array;
struct darshan_name_record_ref *name_hash = NULL;
struct darshan_name_record_ref *ref, *tmp;
char *mod_buf;
char *mod_buf, *tmp_mod_buf;
enum darshan_comp_type comp_type;
int bzip2;
int obfuscate;
......@@ -388,26 +388,46 @@ int main(int argc, char **argv)
continue;
}
/* we have module data to convert */
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
/* for dxt, don't use static record buffer and instead have
* darshan-logutils malloc us memory for the trace data
*/
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
{
tmp_mod_buf = NULL;
}
else
{
tmp_mod_buf = mod_buf;
memset(tmp_mod_buf, 0, DEF_MOD_BUF_SIZE);
}
/* loop over each of the module's records and convert */
while((ret = mod_logutils[i]->log_get_record(infile, (void **)&mod_buf)) == 1)
while((ret = mod_logutils[i]->log_get_record(infile, (void **)&tmp_mod_buf)) == 1)
{
base_rec = (struct darshan_base_record *)mod_buf;
base_rec = (struct darshan_base_record *)tmp_mod_buf;
if(!hash || hash == base_rec->id)
{
ret = mod_logutils[i]->log_put_record(outfile, mod_buf);
ret = mod_logutils[i]->log_put_record(outfile, tmp_mod_buf);
if(ret < 0)
{
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
free(tmp_mod_buf);
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
{
free(tmp_mod_buf);
tmp_mod_buf = NULL;
}
else
{
memset(tmp_mod_buf, 0, DEF_MOD_BUF_SIZE);
}
}
if(ret < 0)
......
......@@ -30,6 +30,8 @@ char *hdf5_f_counter_names[] = {
};
#undef X
#define DARSHAN_HDF5_FILE_SIZE_1 40
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
static void darshan_log_print_hdf5_file(void *file_rec,
......@@ -52,6 +54,7 @@ struct darshan_mod_logutil_funcs hdf5_logutils =
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
{
struct darshan_hdf5_file *file = *((struct darshan_hdf5_file **)hdf5_buf_p);
int rec_len;
int i;
int ret;
......@@ -65,12 +68,42 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
return(-1);
}
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, file,
sizeof(struct darshan_hdf5_file));
if(fd->mod_ver[DARSHAN_HDF5_MOD] == DARSHAN_HDF5_VER)
{
/* log format is in current version, so we don't need to do any
* translation of counters while reading
*/
rec_len = sizeof(struct darshan_hdf5_file);
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, file, rec_len);
}
else
{
char scratch[1024] = {0};
char *src_p, *dest_p;
int len;
rec_len = DARSHAN_HDF5_FILE_SIZE_1;
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, scratch, rec_len);
if(ret != rec_len)
goto exit;
/* upconvert version 1 to version 2 in-place */
dest_p = scratch + (sizeof(struct darshan_base_record) +
(1 * sizeof(int64_t)) + (3 * sizeof(double)));
src_p = dest_p - (2 * sizeof(double));
len = sizeof(double);
memmove(dest_p, src_p, len);
/* set F_CLOSE_START and F_OPEN_END to -1 */
*((double *)src_p) = -1;
*((double *)(src_p + sizeof(double))) = -1;
memcpy(file, scratch, sizeof(struct darshan_hdf5_file));
}
exit:
if(*hdf5_buf_p == NULL)
{
if(ret == sizeof(struct darshan_hdf5_file))
if(ret == rec_len)
*hdf5_buf_p = file;
else
free(file);
......@@ -78,7 +111,7 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
if(ret < 0)
return(-1);
else if(ret < sizeof(struct darshan_hdf5_file))
else if(ret < rec_len)
return(0);
else
{
......@@ -90,7 +123,16 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
{
/* skip counters we explicitly set to -1 since they don't
* need to be byte swapped
*/
if((fd->mod_ver[DARSHAN_HDF5_MOD] == 1) &&
((i == HDF5_F_CLOSE_START_TIMESTAMP) ||
(i == HDF5_F_OPEN_END_TIMESTAMP)))
continue;
DARSHAN_BSWAP64(&file->fcounters[i]);
}
}
return(1);
......@@ -140,8 +182,15 @@ static void darshan_log_print_hdf5_description(int ver)
{
printf("\n# description of HDF5 counters:\n");
printf("# HDF5_OPENS: HDF5 file open operation counts.\n");
printf("# HDF5_F_OPEN_TIMESTAMP: timestamp of first HDF5 file open.\n");
printf("# HDF5_F_CLOSE_TIMESTAMP: timestamp of last HDF5 file close.\n");
printf("# HDF5_F_*_START_TIMESTAMP: timestamp of first HDF5 file open/close.\n");
printf("# HDF5_F_*_END_TIMESTAMP: timestamp of last HDF5 file open/close.\n");
if(ver == 1)
{
printf("\n# WARNING: HDF5 module log format version 1 does not support the following counters:\n");
printf("# - HDF5_F_CLOSE_START_TIMESTAMP\n");
printf("# - HDF5_F_OPEN_END_TIMESTAMP\n");
}
return;
}
......@@ -242,7 +291,8 @@ static void darshan_log_agg_hdf5_files(void *rec, void *agg_rec, int init_flag)
{
switch(i)
{
case HDF5_F_OPEN_TIMESTAMP:
case HDF5_F_OPEN_START_TIMESTAMP:
case HDF5_F_CLOSE_START_TIMESTAMP:
/* minimum non-zero */
if((hdf5_rec->fcounters[i] > 0) &&
((agg_hdf5_rec->fcounters[i] == 0) ||
......@@ -251,7 +301,8 @@ static void darshan_log_agg_hdf5_files(void *rec, void *agg_rec, int init_flag)
agg_hdf5_rec->fcounters[i] = hdf5_rec->fcounters[i];
}
break;
case HDF5_F_CLOSE_TIMESTAMP:
case HDF5_F_OPEN_END_TIMESTAMP:
case HDF5_F_CLOSE_END_TIMESTAMP:
/* maximum */
if(hdf5_rec->fcounters[i] > agg_hdf5_rec->fcounters[i])
{
......
......@@ -30,6 +30,8 @@ char *mpiio_f_counter_names[] = {
};
#undef X
#define DARSHAN_MPIIO_FILE_SIZE_1 544
static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p);
static int darshan_log_put_mpiio_file(darshan_fd fd, void* mpiio_buf);
static void darshan_log_print_mpiio_file(void *file_rec,
......@@ -52,6 +54,7 @@ struct darshan_mod_logutil_funcs mpiio_logutils =
static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p)
{
struct darshan_mpiio_file *file = *((struct darshan_mpiio_file **)mpiio_buf_p);
int rec_len;
int i;
int ret;
......@@ -64,13 +67,43 @@ static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p)
if(!file)
return(-1);
}
ret = darshan_log_get_mod(fd, DARSHAN_MPIIO_MOD, file,
sizeof(struct darshan_mpiio_file));
if(fd->mod_ver[DARSHAN_MPIIO_MOD] == DARSHAN_MPIIO_VER)
{
/* log format is in current version, so we don't need to do any
* translation of counters while reading
*/
rec_len = sizeof(struct darshan_mpiio_file);
ret = darshan_log_get_mod(fd, DARSHAN_MPIIO_MOD, file, rec_len);
}
else
{
char scratch[1024] = {0};
char *src_p, *dest_p;
int len;
rec_len = DARSHAN_MPIIO_FILE_SIZE_1;
ret = darshan_log_get_mod(fd, DARSHAN_MPIIO_MOD, scratch, rec_len);
if(ret != rec_len)
goto exit;
/* upconvert versions 1/2 to version 3 in-place */
dest_p = scratch + (sizeof(struct darshan_base_record) +
(51 * sizeof(int64_t)) + (5 * sizeof(double)));