Commit 4fd1d6d3 authored by Shane Snyder's avatar Shane Snyder
Browse files

port lustre mod to recent darshan changes

parent 2d3f980d
...@@ -35,9 +35,13 @@ static int lustre_record_compare( ...@@ -35,9 +35,13 @@ static int lustre_record_compare(
int sort_lustre_records( int sort_lustre_records(
void); void);
#ifdef HAVE_MPI
static void lustre_mpi_redux(
void *lustre_buf, MPI_Comm mod_comm,
darshan_record_id *shared_recs, int shared_rec_count);
#endif
static void lustre_shutdown( static void lustre_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs, void **lustre_buf, int *lustre_buf_sz);
int shared_rec_count, void **lustre_buf, int *lustre_buf_sz);
struct lustre_runtime *lustre_runtime = NULL; struct lustre_runtime *lustre_runtime = NULL;
static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
...@@ -49,6 +53,7 @@ static int my_rank = -1; ...@@ -49,6 +53,7 @@ static int my_rank = -1;
#ifndef LOV_MAX_STRIPE_COUNT /* for Lustre < 2.4 */ #ifndef LOV_MAX_STRIPE_COUNT /* for Lustre < 2.4 */
#define LOV_MAX_STRIPE_COUNT 2000 #define LOV_MAX_STRIPE_COUNT 2000
#endif #endif
void darshan_instrument_lustre_file(const char* filepath, int fd) void darshan_instrument_lustre_file(const char* filepath, int fd)
{ {
struct lustre_record_ref *rec_ref; struct lustre_record_ref *rec_ref;
...@@ -177,6 +182,13 @@ void darshan_instrument_lustre_file(const char* filepath, int fd) ...@@ -177,6 +182,13 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
static void lustre_runtime_initialize() static void lustre_runtime_initialize()
{ {
int lustre_buf_size; int lustre_buf_size;
darshan_module_funcs mod_funcs = {
#ifdef HAVE_MPI
.mod_redux_func = &lustre_mpi_redux,
#endif
.mod_shutdown_func = &lustre_shutdown
};
/* try and store a default number of records for this module, assuming /* try and store a default number of records for this module, assuming
* each file uses 64 OSTs * each file uses 64 OSTs
...@@ -186,7 +198,7 @@ static void lustre_runtime_initialize() ...@@ -186,7 +198,7 @@ static void lustre_runtime_initialize()
/* register the lustre module with darshan-core */ /* register the lustre module with darshan-core */
darshan_core_register_module( darshan_core_register_module(
DARSHAN_LUSTRE_MOD, DARSHAN_LUSTRE_MOD,
&lustre_shutdown, mod_funcs,
&lustre_buf_size, &lustre_buf_size,
&my_rank, &my_rank,
NULL); NULL);
...@@ -215,12 +227,12 @@ static void lustre_runtime_initialize() ...@@ -215,12 +227,12 @@ static void lustre_runtime_initialize()
* Functions exported by Lustre module for coordinating with darshan-core * * Functions exported by Lustre module for coordinating with darshan-core *
**************************************************************************/ **************************************************************************/
static void lustre_shutdown( #ifdef HAVE_MPI
static void lustre_mpi_redux(
void *posix_buf,
MPI_Comm mod_comm, MPI_Comm mod_comm,
darshan_record_id *shared_recs, darshan_record_id *shared_recs,
int shared_rec_count, int shared_rec_count)
void **lustre_buf,
int *lustre_buf_sz)
{ {
struct lustre_record_ref *rec_ref; struct lustre_record_ref *rec_ref;
int i; int i;
...@@ -228,9 +240,6 @@ static void lustre_shutdown( ...@@ -228,9 +240,6 @@ static void lustre_shutdown(
LUSTRE_LOCK(); LUSTRE_LOCK();
assert(lustre_runtime); assert(lustre_runtime);
lustre_runtime->record_buffer = *lustre_buf;
lustre_runtime->record_buffer_size = *lustre_buf_sz;
/* if there are globally shared files, do a shared file reduction */ /* if there are globally shared files, do a shared file reduction */
/* NOTE: the shared file reduction is also skipped if the /* NOTE: the shared file reduction is also skipped if the
* DARSHAN_DISABLE_SHARED_REDUCTION environment variable is set. * DARSHAN_DISABLE_SHARED_REDUCTION environment variable is set.
...@@ -253,21 +262,36 @@ static void lustre_shutdown( ...@@ -253,21 +262,36 @@ static void lustre_shutdown(
else else
darshan_core_fprintf(stderr, "WARNING: unexpected condition in Darshan, possibly triggered by memory corruption. Darshan log may be incorrect.\n"); darshan_core_fprintf(stderr, "WARNING: unexpected condition in Darshan, possibly triggered by memory corruption. Darshan log may be incorrect.\n");
} }
}
/* sort the array of files descending by rank so that we get all of the LUSTRE_UNLOCK();
* shared files (marked by rank -1) in a contiguous portion at end return;
* of the array }
*/ #endif
sort_lustre_records();
/* simply drop all shared records from the end of the record array on static void lustre_shutdown(
* non-root ranks simply by recalculating the size of the buffer void **lustre_buf,
*/ int *lustre_buf_sz)
if (my_rank != 0) {
{ LUSTRE_LOCK();
darshan_iter_record_refs(lustre_runtime->record_id_hash, assert(lustre_runtime);
&lustre_subtract_shared_rec_size);
} lustre_runtime->record_buffer = *lustre_buf;
lustre_runtime->record_buffer_size = *lustre_buf_sz;
/* sort the array of files descending by rank so that we get all of the
* shared files (marked by rank -1) in a contiguous portion at end
* of the array
*/
sort_lustre_records();
/* simply drop all shared records from the end of the record array on
* non-root ranks simply by recalculating the size of the buffer
*/
if (my_rank != 0)
{
darshan_iter_record_refs(lustre_runtime->record_id_hash,
&lustre_subtract_shared_rec_size);
} }
/* modify output buffer size to account for any shared records that were removed */ /* modify output buffer size to account for any shared records that were removed */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment