Commit 94efcc17 authored by Kevin Harms's avatar Kevin Harms
Browse files

Updates to bgq module core

parent 7dc249cb
......@@ -41,7 +41,6 @@ struct bgq_runtime
struct darshan_bgq_record record;
};
/* null_runtime is the global data structure encapsulating "NULL" module state */
static struct bgq_runtime *bgq_runtime = NULL;
static pthread_mutex_t bgq_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
......@@ -52,12 +51,14 @@ static int instrumentation_disabled = 0;
static int my_rank = -1;
/* internal helper functions for the "NULL" module */
static void bgq_runtime_initialize(void);
void bgq_runtime_initialize(void);
/* forward declaration for module functions needed to interface with darshan-core */
static void bgq_begin_shutdown(void);
static void bgq_get_output_data(void **buffer, int *size);
static void bgq_shutdown(void);
static void bgq_setup_reduction(darshan_record_id *shared_recs,int *shared_rec_count,void **send_buf,void **recv_buf,int *rec_size);
static void bgq_record_reduction_op(void* infile_v,void* inoutfile_v,int *len,MPI_Datatype *datatype);
/* macros for obtaining/releasing the "NULL" module lock */
#define BGQ_LOCK() pthread_mutex_lock(&bgq_runtime_mutex)
......@@ -70,10 +71,13 @@ static void capture(struct darshan_bgq_record *rec)
{
#ifdef __bgq__
Personality_t person;
int r;
rec->counters[BGQ_CSJOBID] = Kernel_GetJobID();
rec->counters[BGQ_RANKSPERNODE] = Kernel_ProcessCount();
rec->counters[BGQ_INODES] = MPIX_IO_node();
r = Kernel_GetPersonality(&person, sizeof(person));
if (r == 0)
{
......@@ -104,14 +108,14 @@ static void capture(struct darshan_bgq_record *rec)
* Internal functions for manipulating BGQ module state *
**********************************************************/
static void bgq_runtime_initialize()
void bgq_runtime_initialize()
{
/* struct of function pointers for interfacing with darshan-core */
struct darshan_module_funcs bgq_mod_fns =
{
.begin_shutdown = bgq_begin_shutdown,
.setup_reduction = NULL,
.record_reduction_op = NULL,
.setup_reduction = bgq_setup_reduction,
.record_reduction_op = bgq_record_reduction_op,
.get_output_data = bgq_get_output_data,
.shutdown = bgq_shutdown
};
......@@ -198,9 +202,9 @@ static void bgq_get_output_data(
* I/O records, and set the output size according to the number of records
* currently being tracked.
*/
if (bgq_runtime)
if ((bgq_runtime) && (my_rank == 0))
{
*buffer = (void *)&bgq_runtime->record;
*buffer = &bgq_runtime->record;
*size = sizeof(struct darshan_bgq_record);
}
else
......@@ -224,6 +228,70 @@ static void bgq_shutdown()
return;
}
static void bgq_setup_reduction(
darshan_record_id *shared_recs,
int *shared_rec_count,
void **send_buf,
void **recv_buf,
int *rec_size)
{
int i;
int found;
for (i = 0; i < *shared_rec_count; i++)
{
if (shared_recs[i] == bgq_runtime->record.f_id)
{
found = 1;
break;
}
}
if (found)
{
printf("found bgq shared record\n");
*rec_size = sizeof(struct darshan_bgq_record);
*shared_rec_count = 1;
*send_buf = &bgq_runtime->record;
*recv_buf = &bgq_runtime->record;
}
return;
}
static void bgq_record_reduction_op(
void* infile_v,
void* inoutfile_v,
int* len,
MPI_Datatype *datatype)
{
int i;
int j;
struct darshan_bgq_record *infile = infile_v;
struct darshan_bgq_record *inoutfile = inoutfile_v;
for (i = 0; i<*len; i++)
{
for (j = 0; j < BGQ_NUM_INDICES; j++)
{
if (infile->counters[j] != inoutfile->counters[j])
{
// unexpected
fprintf(stderr,
"%lu counter mismatch: %d [%lu] [%lu]\n",
infile->f_id,
j,
infile->counters[j],
inoutfile->counters[j]);
}
}
infile++;
inoutfile++;
}
return;
}
/*
* Local variables:
* c-indent-level: 4
......
......@@ -208,6 +208,11 @@ void darshan_core_initialize(int argc, char **argv)
}
}
#ifdef __bgq__
extern void bgq_runtime_initialize();
bgq_runtime_initialize();
#endif
return;
}
......
......@@ -453,16 +453,19 @@ static void mpiio_setup_reduction(
{
struct mpiio_file_runtime *file;
int i;
int count;
assert(mpiio_runtime);
/* necessary initialization of shared records (e.g., change rank to -1) */
for(i = 0; i < *shared_rec_count; i++)
for(i = 0,count = 0; i < *shared_rec_count; i++)
{
HASH_FIND(hlink, mpiio_runtime->file_hash, &shared_recs[i],
sizeof(darshan_record_id), file);
assert(file);
if (!file) {
continue;
}
count++;
file->file_record->rank = -1;
}
......@@ -475,22 +478,18 @@ static void mpiio_setup_reduction(
/* make *send_buf point to the shared files at the end of sorted array */
*send_buf =
&(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-(*shared_rec_count)]);
&(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-(count)]);
/* allocate memory for the reduction output on rank 0 */
if(my_rank == 0)
if((my_rank == 0) && (count > 0))
{
*recv_buf = malloc(*shared_rec_count * sizeof(struct darshan_mpiio_file));
if(!(*recv_buf))
return;
*recv_buf = malloc(count * sizeof(struct darshan_mpiio_file));
mpiio_runtime->red_buf = *recv_buf;
}
*rec_size = sizeof(struct darshan_mpiio_file);
/* TODO: cleaner way to do this? */
if(my_rank == 0)
mpiio_runtime->red_buf = *recv_buf;
mpiio_runtime->shared_rec_count = *shared_rec_count;
mpiio_runtime->shared_rec_count = count;
return;
}
......
......@@ -1655,15 +1655,19 @@ static void posix_setup_reduction(
struct posix_file_runtime *file;
int i;
double posix_time;
int count;
assert(posix_runtime);
/* necessary initialization of shared records (e.g., change rank to -1) */
for(i = 0; i < *shared_rec_count; i++)
for(i = 0,count = 0; i < *shared_rec_count; i++)
{
HASH_FIND(hlink, posix_runtime->file_hash, &shared_recs[i],
sizeof(darshan_record_id), file);
assert(file);
if (!file) {
continue;
}
count++;
posix_time =
file->file_record->fcounters[POSIX_F_READ_TIME] +
......@@ -1702,22 +1706,22 @@ static void posix_setup_reduction(
/* make *send_buf point to the shared files at the end of sorted array */
*send_buf =
&(posix_runtime->file_record_array[posix_runtime->file_array_ndx-(*shared_rec_count)]);
&(posix_runtime->file_record_array[posix_runtime->file_array_ndx-(count)]);
/* allocate memory for the reduction output on rank 0 */
if(my_rank == 0)
if((my_rank == 0) && (count > 0))
{
*recv_buf = malloc(*shared_rec_count * sizeof(struct darshan_posix_file));
if(!(*recv_buf))
return;
printf("shared count = %d\n", count);
*recv_buf = malloc(count * sizeof(struct darshan_posix_file));
posix_runtime->red_buf = *recv_buf;
printf("recv_buf = %p\n", *recv_buf);
// if(!(*recv_buf))
// return;
}
*rec_size = sizeof(struct darshan_posix_file);
/* TODO: cleaner way to do this? */
if(my_rank == 0)
posix_runtime->red_buf = *recv_buf;
posix_runtime->shared_rec_count = *shared_rec_count;
posix_runtime->shared_rec_count = count;
return;
}
......@@ -1962,7 +1966,7 @@ static void posix_get_output_data(
/* TODO: cleaner way to do this? */
/* clean up reduction state */
if(my_rank == 0)
if((my_rank == 0) && (posix_runtime->red_buf))
{
int tmp_ndx = posix_runtime->file_array_ndx - posix_runtime->shared_rec_count;
memcpy(&(posix_runtime->file_record_array[tmp_ndx]), posix_runtime->red_buf,
......
all: libdarshan-util.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser
all: libdarshan-util.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser darshan-bgq-parser
DESTDIR =
srcdir = @srcdir@
......@@ -74,6 +74,9 @@ darshan-posix-parser: darshan-posix-parser.c darshan-logutils.h darshan-posix-lo
darshan-mpiio-parser: darshan-mpiio-parser.c darshan-logutils.h darshan-mpiio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
darshan-bgq-parser: darshan-bgq-parser.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
#darshan-parser: darshan-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o | uthash-1.9.2
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
......@@ -109,6 +112,7 @@ install:: all
install -m 755 darshan-base-parser $(bindir)
install -m 755 darshan-posix-parser $(bindir)
install -m 755 darshan-mpiio-parser $(bindir)
install -m 755 darshan-bgq-parser $(bindir)
# install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir)
# install -m 755 darshan-analyzer $(bindir)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment