Commit 628724ba authored by Shane Snyder's avatar Shane Snyder

clean up & reorganize mpiio module source file

parent 28f19402
......@@ -11,27 +11,113 @@
enum darshan_mpiio_indices
{
DARSHAN_MPIIO_INDEP_OPENS, /* independent opens */
DARSHAN_MPIIO_COLL_OPENS, /* collective opens */
DARSHAN_MPIIO_HINTS, /* how many times hints were used */
MPIIO_INDEP_OPENS, /* count of MPI independent opens */
MPIIO_COLL_OPENS, /* count of MPI collective opens */
MPIIO_INDEP_READS, /* count of MPI independent reads */
MPIIO_INDEP_WRITES, /* count of MPI independent writes */
MPIIO_COLL_READS, /* count of MPI collective reads */
MPIIO_COLL_WRITES, /* count of MPI collective writes */
MPIIO_SPLIT_READS, /* count of MPI split collective reads */
MPIIO_SPLIT_WRITES, /* count of MPI split collective writes */
MPIIO_NB_READS, /* count of MPI nonblocking reads */
MPIIO_NB_WRITES, /* count of MPI nonblocking writes */
MPIIO_SYNCS, /* count of MPI file syncs */
MPIIO_HINTS, /* count of MPI hints used */
MPIIO_VIEWS, /* count of MPI set view calls */
#if 0
/* type categories */
MPIIO_COMBINER_NAMED, /* count of each MPI datatype category */
MPIIO_COMBINER_DUP,
MPIIO_COMBINER_CONTIGUOUS,
MPIIO_COMBINER_VECTOR,
MPIIO_COMBINER_HVECTOR_INTEGER,
MPIIO_COMBINER_HVECTOR,
MPIIO_COMBINER_INDEXED,
MPIIO_COMBINER_HINDEXED_INTEGER,
MPIIO_COMBINER_HINDEXED,
MPIIO_COMBINER_INDEXED_BLOCK,
MPIIO_COMBINER_STRUCT_INTEGER,
MPIIO_COMBINER_STRUCT,
MPIIO_COMBINER_SUBARRAY,
MPIIO_COMBINER_DARRAY,
MPIIO_COMBINER_F90_REAL,
MPIIO_COMBINER_F90_COMPLEX,
MPIIO_COMBINER_F90_INTEGER,
MPIIO_COMBINER_RESIZED,
/* buckets */
MPIIO_SIZE_READ_AGG_0_100, /* count of MPI read size ranges */
MPIIO_SIZE_READ_AGG_100_1K,
MPIIO_SIZE_READ_AGG_1K_10K,
MPIIO_SIZE_READ_AGG_10K_100K,
MPIIO_SIZE_READ_AGG_100K_1M,
MPIIO_SIZE_READ_AGG_1M_4M,
MPIIO_SIZE_READ_AGG_4M_10M,
MPIIO_SIZE_READ_AGG_10M_100M,
MPIIO_SIZE_READ_AGG_100M_1G,
MPIIO_SIZE_READ_AGG_1G_PLUS,
/* buckets */
MPIIO_SIZE_WRITE_AGG_0_100, /* count of MPI write size ranges */
MPIIO_SIZE_WRITE_AGG_100_1K,
MPIIO_SIZE_WRITE_AGG_1K_10K,
MPIIO_SIZE_WRITE_AGG_10K_100K,
MPIIO_SIZE_WRITE_AGG_100K_1M,
MPIIO_SIZE_WRITE_AGG_1M_4M,
MPIIO_SIZE_WRITE_AGG_4M_10M,
MPIIO_SIZE_WRITE_AGG_10M_100M,
MPIIO_SIZE_WRITE_AGG_100M_1G,
MPIIO_SIZE_WRITE_AGG_1G_PLUS,
/* buckets */
MPIIO_EXTENT_READ_0_100, /* count of MPI read extent ranges */
MPIIO_EXTENT_READ_100_1K,
MPIIO_EXTENT_READ_1K_10K,
MPIIO_EXTENT_READ_10K_100K,
MPIIO_EXTENT_READ_100K_1M,
MPIIO_EXTENT_READ_1M_4M,
MPIIO_EXTENT_READ_4M_10M,
MPIIO_EXTENT_READ_10M_100M,
MPIIO_EXTENT_READ_100M_1G,
MPIIO_EXTENT_READ_1G_PLUS,
/* buckets */
MPIIO_EXTENT_WRITE_0_100, /* count of MPI write extent ranges */
MPIIO_EXTENT_WRITE_100_1K,
MPIIO_EXTENT_WRITE_1K_10K,
MPIIO_EXTENT_WRITE_10K_100K,
MPIIO_EXTENT_WRITE_100K_1M,
MPIIO_EXTENT_WRITE_1M_4M,
MPIIO_EXTENT_WRITE_4M_10M,
MPIIO_EXTENT_WRITE_10M_100M,
MPIIO_EXTENT_WRITE_100M_1G,
MPIIO_EXTENT_WRITE_1G_PLUS,
#endif
DARSHAN_MPIIO_NUM_INDICES,
MPIIO_NUM_INDICES,
};
enum darshan_mpiio_f_indices
{
DARSHAN_MPIIO_F_META_TIME, /* cumulative metadata time */
DARSHAN_MPIIO_F_OPEN_TIMESTAMP, /* first open timestamp */
MPIIO_F_OPEN_TIMESTAMP,
#if 0
MPIIO_F_READ_START_TIMESTAMP,
MPIIO_F_WRITE_START_TIMESTAMP,
MPIIO_F_READ_END_TIMESTAMP,
MPIIO_F_WRITE_END_TIMESTAMP,
#endif
MPIIO_F_CLOSE_TIMESTAMP,
#if 0
MPIIO_F_READ_TIME,
MPIIO_F_WRITE_TIME,
#endif
MPIIO_F_META_TIME,
DARSHAN_MPIIO_F_NUM_INDICES,
MPIIO_F_NUM_INDICES,
};
struct darshan_mpiio_file
{
darshan_record_id f_id;
int64_t rank;
int64_t counters[DARSHAN_MPIIO_NUM_INDICES];
double fcounters[DARSHAN_MPIIO_F_NUM_INDICES];
int64_t counters[MPIIO_NUM_INDICES];
double fcounters[MPIIO_F_NUM_INDICES];
};
#endif /* __DARSHAN_MPIIO_LOG_FORMAT_H */
......@@ -25,6 +25,7 @@
#include <pthread.h>
#include "uthash.h"
#include "darshan.h"
#include "darshan-mpiio-log-format.h"
#include "darshan-dynamic.h"
......@@ -55,11 +56,7 @@
struct mpiio_file_runtime
{
struct darshan_mpiio_file* file_record;
double last_mpi_meta_end; /* end time of last MPI meta op (so far) */
/* TODO: any stateful (but not intended for persistent storage in the log)
* information about MPI-IO access. If we don't have any then this struct
* could be eliminated.
*/
double last_mpi_meta_end;
UT_hash_handle hlink;
};
......@@ -89,10 +86,14 @@ struct mpiio_file_runtime
struct mpiio_file_runtime_ref
{
struct mpiio_file_runtime* file;
MPI_File *fh;
MPI_File fh;
UT_hash_handle hlink;
};
/* The mpiio_runtime structure maintains necessary state for storing
* MPI-IO file records and for coordinating with darshan-core at
* shutdown time.
*/
struct mpiio_runtime
{
struct mpiio_file_runtime* file_runtime_array;
......@@ -110,22 +111,26 @@ static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER
static int instrumentation_disabled = 0;
static int my_rank = -1;
#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
static void mpiio_runtime_initialize(void);
static struct mpiio_file_runtime* mpiio_file_by_name_setfh(const char* name, MPI_File fh);
static struct mpiio_file_runtime* mpiio_file_by_fh(MPI_File fh);
static void mpiio_file_close_fh(MPI_File fh);
static int mpiio_file_compare(const void* a, const void* b);
static void mpiio_begin_shutdown(void);
static void mpiio_shutdown(void);
static void mpiio_get_output_data(
void **buffer,
int *size);
static struct mpiio_file_runtime* mpiio_file_by_name_setfh(const char* name, MPI_File *fh);
static struct mpiio_file_runtime* mpiio_file_by_name(const char *name);
static void mpiio_setup_reduction(darshan_record_id *shared_recs, int *shared_rec_count,
void **send_buf, void **recv_buf, int *rec_size);
static void mpiio_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype);
static void mpiio_setup_reduction(darshan_record_id *shared_recs,
int *shared_rec_count, void **send_buf, void **recv_buf, int *rec_size);
static int mpiio_file_compare(const void* a, const void* b);
static void mpiio_get_output_data(void **buffer, int *size);
static void mpiio_shutdown(void);
#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
/**********************************************************
* Wrappers for MPI-IO functions of interest *
**********************************************************/
#ifdef HAVE_MPIIO_CONST
int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh)
......@@ -158,27 +163,27 @@ int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_F
filename = tmp + 1;
}
file = mpiio_file_by_name_setfh(filename, fh);
file = mpiio_file_by_name_setfh(filename, (*fh));
if(file)
{
file->file_record->rank = my_rank;
DARSHAN_COUNTER_F_INC_NO_OVERLAP(file->file_record, tm1, tm2, file->last_mpi_meta_end, DARSHAN_MPIIO_F_META_TIME);
if(DARSHAN_COUNTER_F_VALUE(file->file_record, DARSHAN_MPIIO_F_OPEN_TIMESTAMP) == 0)
DARSHAN_COUNTER_F_SET(file->file_record, DARSHAN_MPIIO_F_OPEN_TIMESTAMP,
tm1);
DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
if(comm_size == 1)
{
DARSHAN_COUNTER_INC(file->file_record, DARSHAN_MPIIO_INDEP_OPENS, 1);
DARSHAN_COUNTER_INC(file->file_record, MPIIO_INDEP_OPENS, 1);
}
else
{
DARSHAN_COUNTER_INC(file->file_record, DARSHAN_MPIIO_COLL_OPENS, 1);
DARSHAN_COUNTER_INC(file->file_record, MPIIO_COLL_OPENS, 1);
}
if(info != MPI_INFO_NULL)
{
DARSHAN_COUNTER_INC(file->file_record, DARSHAN_MPIIO_HINTS, 1);
DARSHAN_COUNTER_INC(file->file_record, MPIIO_HINTS, 1);
}
if(DARSHAN_COUNTER_F_VALUE(file->file_record, MPIIO_F_OPEN_TIMESTAMP) == 0)
DARSHAN_COUNTER_F_SET(file->file_record, MPIIO_F_OPEN_TIMESTAMP, tm1);
DARSHAN_COUNTER_F_INC_NO_OVERLAP(file->file_record, tm1, tm2,
file->last_mpi_meta_end, MPIIO_F_META_TIME);
}
MPIIO_UNLOCK();
......@@ -187,6 +192,37 @@ int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_F
return(ret);
}
int MPI_File_close(MPI_File *fh)
{
int ret;
struct mpiio_file_runtime* file;
double tm1, tm2;
tm1 = darshan_core_wtime();
ret = DARSHAN_MPI_CALL(PMPI_File_close)(fh);
tm2 = darshan_core_wtime();
MPIIO_LOCK();
mpiio_runtime_initialize();
file = mpiio_file_by_fh(*fh);
if(file)
{
DARSHAN_COUNTER_F_SET(file->file_record, MPIIO_F_CLOSE_TIMESTAMP,
darshan_core_wtime());
DARSHAN_COUNTER_F_INC_NO_OVERLAP(file->file_record, tm1, tm2,
file->last_mpi_meta_end, MPIIO_F_META_TIME);
mpiio_file_close_fh(*fh);
}
MPIIO_UNLOCK();
return(ret);
}
/***********************************************************
* Internal functions for manipulating MPI-IO module state *
***********************************************************/
/* initialize data structures and register with darshan-core component */
static void mpiio_runtime_initialize()
{
int mem_limit;
......@@ -245,62 +281,7 @@ static void mpiio_runtime_initialize()
return;
}
static void mpiio_begin_shutdown()
{
assert(mpiio_runtime);
MPIIO_LOCK();
instrumentation_disabled = 1;
MPIIO_UNLOCK();
return;
}
static void mpiio_get_output_data(
void **buffer,
int *size)
{
assert(mpiio_runtime);
/* TODO: clean up reduction stuff */
if(my_rank == 0)
{
int tmp_ndx = mpiio_runtime->file_array_ndx - mpiio_runtime->shared_rec_count;
memcpy(&(mpiio_runtime->file_record_array[tmp_ndx]), mpiio_runtime->red_buf,
mpiio_runtime->shared_rec_count * sizeof(struct darshan_mpiio_file));
free(mpiio_runtime->red_buf);
}
else
{
mpiio_runtime->file_array_ndx -= mpiio_runtime->shared_rec_count;
}
*buffer = (void *)(mpiio_runtime->file_record_array);
*size = mpiio_runtime->file_array_ndx * sizeof(struct darshan_mpiio_file);
return;
}
static void mpiio_shutdown()
{
struct mpiio_file_runtime_ref *ref, *tmp;
HASH_ITER(hlink, mpiio_runtime->fh_hash, ref, tmp)
{
HASH_DELETE(hlink, mpiio_runtime->fh_hash, ref);
free(ref);
}
HASH_CLEAR(hlink, mpiio_runtime->file_hash); /* these entries are freed all at once below */
free(mpiio_runtime->file_runtime_array);
free(mpiio_runtime->file_record_array);
free(mpiio_runtime);
mpiio_runtime = NULL;
return;
}
#if 0
/* get a MPIIO file record for the given file path */
static struct mpiio_file_runtime* mpiio_file_by_name(const char *name)
{
......@@ -350,12 +331,14 @@ static struct mpiio_file_runtime* mpiio_file_by_name(const char *name)
free(newname);
return(file);
}
#endif
/* get an MPIIO file record for the given file path, and also create a
* reference structure using the corresponding file handle
*/
static struct mpiio_file_runtime* mpiio_file_by_name_setfh(const char* name, MPI_File *fh)
static struct mpiio_file_runtime* mpiio_file_by_name_setfh(const char* name, MPI_File fh)
{
#if 0
struct mpiio_file_runtime* file;
struct mpiio_file_runtime_ref* ref;
......@@ -392,54 +375,34 @@ static struct mpiio_file_runtime* mpiio_file_by_name_setfh(const char* name, MPI
HASH_ADD(hlink, mpiio_runtime->fh_hash, fh, sizeof(fh), ref);
return(file);
#endif
}
static void mpiio_record_reduction_op(
void* infile_v,
void* inoutfile_v,
int *len,
MPI_Datatype *datatype)
/* compare function for sorting file records by descending rank */
static int mpiio_file_compare(const void* a_p, const void* b_p)
{
struct darshan_mpiio_file tmp_file;
struct darshan_mpiio_file *infile = infile_v;
struct darshan_mpiio_file *inoutfile = inoutfile_v;
int i, j;
assert(mpiio_runtime);
for(i=0; i<*len; i++)
{
memset(&tmp_file, 0, sizeof(struct darshan_mpiio_file));
const struct darshan_mpiio_file* a = a_p;
const struct darshan_mpiio_file* b = b_p;
tmp_file.f_id = infile->f_id;
tmp_file.rank = -1;
if(a->rank < b->rank)
return 1;
if(a->rank > b->rank)
return -1;
/* sum */
for(j=DARSHAN_MPIIO_INDEP_OPENS; j<=DARSHAN_MPIIO_HINTS; j++)
{
tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
}
return 0;
}
/* sum (floating point) */
for(j=DARSHAN_MPIIO_F_META_TIME; j<=DARSHAN_MPIIO_F_META_TIME; j++)
{
tmp_file.fcounters[j] = infile->fcounters[j] + inoutfile->fcounters[j];
}
/**************************************************************************
* Functions exported by MPI-IO module for coordinating with darshan-core *
**************************************************************************/
/* min non-zero (if available) value */
for(j=DARSHAN_MPIIO_F_OPEN_TIMESTAMP; j<=DARSHAN_MPIIO_F_OPEN_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j] && inoutfile->fcounters[j] > 0)
tmp_file.fcounters[j] = inoutfile->fcounters[j];
else
tmp_file.fcounters[j] = infile->fcounters[j];
}
static void mpiio_begin_shutdown()
{
assert(mpiio_runtime);
/* update pointers */
*inoutfile = tmp_file;
inoutfile++;
infile++;
}
MPIIO_LOCK();
instrumentation_disabled = 1;
MPIIO_UNLOCK();
return;
}
......@@ -495,18 +458,99 @@ static void mpiio_setup_reduction(
return;
}
/* compare function for sorting file records by descending rank */
static int mpiio_file_compare(const void* a_p, const void* b_p)
static void mpiio_record_reduction_op(
void* infile_v,
void* inoutfile_v,
int *len,
MPI_Datatype *datatype)
{
const struct darshan_mpiio_file* a = a_p;
const struct darshan_mpiio_file* b = b_p;
struct darshan_mpiio_file tmp_file;
struct darshan_mpiio_file *infile = infile_v;
struct darshan_mpiio_file *inoutfile = inoutfile_v;
int i, j;
if(a->rank < b->rank)
return 1;
if(a->rank > b->rank)
return -1;
assert(mpiio_runtime);
return 0;
for(i=0; i<*len; i++)
{
memset(&tmp_file, 0, sizeof(struct darshan_mpiio_file));
tmp_file.f_id = infile->f_id;
tmp_file.rank = -1;
/* sum */
for(j=MPIIO_INDEP_OPENS; j<=MPIIO_HINTS; j++)
{
tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
}
/* sum (floating point) */
for(j=MPIIO_F_META_TIME; j<=MPIIO_F_META_TIME; j++)
{
tmp_file.fcounters[j] = infile->fcounters[j] + inoutfile->fcounters[j];
}
/* min non-zero (if available) value */
for(j=MPIIO_F_OPEN_TIMESTAMP; j<=MPIIO_F_OPEN_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j] && inoutfile->fcounters[j] > 0)
tmp_file.fcounters[j] = inoutfile->fcounters[j];
else
tmp_file.fcounters[j] = infile->fcounters[j];
}
/* update pointers */
*inoutfile = tmp_file;
inoutfile++;
infile++;
}
return;
}
static void mpiio_get_output_data(
void **buffer,
int *size)
{
assert(mpiio_runtime);
/* TODO: clean up reduction stuff */
if(my_rank == 0)
{
int tmp_ndx = mpiio_runtime->file_array_ndx - mpiio_runtime->shared_rec_count;
memcpy(&(mpiio_runtime->file_record_array[tmp_ndx]), mpiio_runtime->red_buf,
mpiio_runtime->shared_rec_count * sizeof(struct darshan_mpiio_file));
free(mpiio_runtime->red_buf);
}
else
{
mpiio_runtime->file_array_ndx -= mpiio_runtime->shared_rec_count;
}
*buffer = (void *)(mpiio_runtime->file_record_array);
*size = mpiio_runtime->file_array_ndx * sizeof(struct darshan_mpiio_file);
return;
}
static void mpiio_shutdown()
{
struct mpiio_file_runtime_ref *ref, *tmp;
HASH_ITER(hlink, mpiio_runtime->fh_hash, ref, tmp)
{
HASH_DELETE(hlink, mpiio_runtime->fh_hash, ref);
free(ref);
}
HASH_CLEAR(hlink, mpiio_runtime->file_hash); /* these entries are freed all at once below */
free(mpiio_runtime->file_runtime_array);
free(mpiio_runtime->file_record_array);
free(mpiio_runtime);
mpiio_runtime = NULL;
return;
}
/*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment