Commit fba89f42 authored by Shane Snyder's avatar Shane Snyder
Browse files

port bgq module over to new interface changes

parent c407379a
...@@ -68,7 +68,6 @@ enum darshan_bgq_f_indices ...@@ -68,7 +68,6 @@ enum darshan_bgq_f_indices
struct darshan_bgq_record struct darshan_bgq_record
{ {
struct darshan_base_record base_rec; struct darshan_base_record base_rec;
int alignment;
int64_t counters[BGQ_NUM_INDICES]; int64_t counters[BGQ_NUM_INDICES];
double fcounters[BGQ_F_NUM_INDICES]; double fcounters[BGQ_F_NUM_INDICES];
}; };
......
...@@ -37,29 +37,20 @@ ...@@ -37,29 +37,20 @@
*/ */
struct bgq_runtime struct bgq_runtime
{ {
struct darshan_bgq_record record; struct darshan_bgq_record *record;
/* TODO: we don't need the mmap and regular buffer, both */
struct darshan_bgq_record *mmap_buf;
}; };
static struct bgq_runtime *bgq_runtime = NULL; static struct bgq_runtime *bgq_runtime = NULL;
static pthread_mutex_t bgq_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; static pthread_mutex_t bgq_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
/* the instrumentation_disabled flag is used to toggle functions on/off */
static int instrumentation_disabled = 0;
/* my_rank indicates the MPI rank of this process */ /* my_rank indicates the MPI rank of this process */
static int my_rank = -1; static int my_rank = -1;
static int darshan_mem_alignment = 1;
/* internal helper functions for the BGQ module */ /* internal helper functions for the BGQ module */
void bgq_runtime_initialize(void); void bgq_runtime_initialize(void);
/* forward declaration for module functions needed to interface with darshan-core */ /* forward declaration for shutdown function needed to interface with darshan-core */
static void bgq_begin_shutdown(void); static void bgq_shutdown(MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count, void **buffer, int *size);
static void bgq_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count, void **buffer, int *size);
static void bgq_shutdown(void);
/* macros for obtaining/releasing the BGQ module lock */ /* macros for obtaining/releasing the BGQ module lock */
#define BGQ_LOCK() pthread_mutex_lock(&bgq_runtime_mutex) #define BGQ_LOCK() pthread_mutex_lock(&bgq_runtime_mutex)
...@@ -68,7 +59,7 @@ static void bgq_shutdown(void); ...@@ -68,7 +59,7 @@ static void bgq_shutdown(void);
/* /*
* Function which updates all the counter data * Function which updates all the counter data
*/ */
static void capture(struct darshan_bgq_record *rec) static void capture(struct darshan_bgq_record *rec, darshan_record_id rec_id)
{ {
Personality_t person; Personality_t person;
int r; int r;
...@@ -96,7 +87,8 @@ static void capture(struct darshan_bgq_record *rec) ...@@ -96,7 +87,8 @@ static void capture(struct darshan_bgq_record *rec)
rec->counters[BGQ_DDRPERNODE] = person.DDR_Config.DDRSizeMB; rec->counters[BGQ_DDRPERNODE] = person.DDR_Config.DDRSizeMB;
} }
rec->rank = my_rank; rec->base_rec.id = rec_id;
rec->base_rec.rank = my_rank;
rec->fcounters[BGQ_F_TIMESTAMP] = darshan_core_wtime(); rec->fcounters[BGQ_F_TIMESTAMP] = darshan_core_wtime();
return; return;
...@@ -108,46 +100,33 @@ static void capture(struct darshan_bgq_record *rec) ...@@ -108,46 +100,33 @@ static void capture(struct darshan_bgq_record *rec)
void bgq_runtime_initialize() void bgq_runtime_initialize()
{ {
/* struct of function pointers for interfacing with darshan-core */ int bgq_buf_size;
struct darshan_module_funcs bgq_mod_fns = darshan_record_id rec_id;
{
.begin_shutdown = bgq_begin_shutdown,
.get_output_data = bgq_get_output_data,
.shutdown = bgq_shutdown
};
int mem_limit;
void *mmap_buf;
int mmap_buf_size;
char *recname = "darshan-internal-bgq";
BGQ_LOCK(); BGQ_LOCK();
/* don't do anything if already initialized or instrumenation is disabled */ /* don't do anything if already initialized */
if(bgq_runtime || instrumentation_disabled) if(bgq_runtime)
{
BGQ_UNLOCK();
return; return;
}
/* we just need to store one single record */
bgq_buf_size = sizeof(struct darshan_bgq_record);
/* register the BG/Q module with the darshan-core component */ /* register the BG/Q module with the darshan-core component */
darshan_core_register_module( darshan_core_register_module(
DARSHAN_BGQ_MOD, DARSHAN_BGQ_MOD,
&bgq_mod_fns, &bgq_shutdown,
&bgq_buf_size,
&my_rank, &my_rank,
&mem_limit, NULL);
&mmap_buf,
&mmap_buf_size,
&darshan_mem_alignment);
/* return if no memory assigned by darshan-core */
if(mem_limit == 0)
{
instrumentation_disabled = 1;
BGQ_UNLOCK();
return;
}
/* not enough memory to fit bgq module */ /* not enough memory to fit bgq module record */
if (mem_limit < sizeof(*bgq_runtime)) if(bgq_buf_size < sizeof(struct darshan_bgq_record))
{ {
instrumentation_disabled = 1; darshan_core_unregister_module(DARSHAN_BGQ_MOD);
BGQ_UNLOCK(); BGQ_UNLOCK();
return; return;
} }
...@@ -156,51 +135,31 @@ void bgq_runtime_initialize() ...@@ -156,51 +135,31 @@ void bgq_runtime_initialize()
bgq_runtime = malloc(sizeof(*bgq_runtime)); bgq_runtime = malloc(sizeof(*bgq_runtime));
if(!bgq_runtime) if(!bgq_runtime)
{ {
instrumentation_disabled = 1; darshan_core_unregister_module(DARSHAN_BGQ_MOD);
BGQ_UNLOCK(); BGQ_UNLOCK();
return; return;
} }
memset(bgq_runtime, 0, sizeof(*bgq_runtime)); memset(bgq_runtime, 0, sizeof(*bgq_runtime));
darshan_core_register_record( rec_id = darshan_core_gen_record_id("darshan-bgq-record");
recname,
strlen(recname), /* register the bgq file record with darshan-core */
bgq_runtime->record = darshan_core_register_record(
rec_id,
NULL,
DARSHAN_BGQ_MOD, DARSHAN_BGQ_MOD,
1, sizeof(struct darshan_bgq_record),
0, NULL);
&bgq_runtime->record.f_id, if(!(bgq_runtime->record))
&bgq_runtime->record.alignment);
/* if record is set to 0, darshan-core is out of space and will not
* track this record, so we should avoid tracking it, too
*/
if(bgq_runtime->record.f_id == 0)
{ {
instrumentation_disabled = 1; darshan_core_unregister_module(DARSHAN_BGQ_MOD);
free(bgq_runtime); free(bgq_runtime);
bgq_runtime = NULL; bgq_runtime = NULL;
BGQ_UNLOCK(); BGQ_UNLOCK();
return; return;
} }
capture(&bgq_runtime->record); capture(bgq_runtime->record, rec_id);
BGQ_UNLOCK();
return;
}
/* Perform any necessary steps prior to shutting down for the BGQ module. */
static void bgq_begin_shutdown()
{
BGQ_LOCK();
/* In general, we want to disable all wrappers while Darshan shuts down.
* This is to avoid race conditions and ensure data consistency, as
* executing wrappers could potentially modify module state while Darshan
* is in the process of shutting down.
*/
instrumentation_disabled = 1;
BGQ_UNLOCK(); BGQ_UNLOCK();
...@@ -214,22 +173,25 @@ static int cmpr(const void *p1, const void *p2) ...@@ -214,22 +173,25 @@ static int cmpr(const void *p1, const void *p2)
return ((*a == *b) ? 0 : ((*a < *b) ? -1 : 1)); return ((*a == *b) ? 0 : ((*a < *b) ? -1 : 1));
} }
/********************************************************************************
* shutdown function exported by this module for coordinating with darshan-core *
********************************************************************************/
/* Pass output data for the BGQ module back to darshan-core to log to file. */ /* Pass output data for the BGQ module back to darshan-core to log to file. */
static void bgq_get_output_data( static void bgq_shutdown(
MPI_Comm mod_comm, MPI_Comm mod_comm,
darshan_record_id *shared_recs, darshan_record_id *shared_recs,
int shared_rec_count, int shared_rec_count,
void **buffer, void **buffer,
int *size) int *size)
{ {
/* Just set the output buffer to point at the array of the BGQ module's
* I/O records, and set the output size according to the number of records
* currently being tracked.
*/
int nprocs; int nprocs;
int result; int result;
uint64_t *ion_ids; uint64_t *ion_ids;
BGQ_LOCK();
assert(bgq_runtime);
if (my_rank == 0) if (my_rank == 0)
{ {
DARSHAN_MPI_CALL(PMPI_Comm_size)(mod_comm, &nprocs); DARSHAN_MPI_CALL(PMPI_Comm_size)(mod_comm, &nprocs);
...@@ -238,12 +200,13 @@ static void bgq_get_output_data( ...@@ -238,12 +200,13 @@ static void bgq_get_output_data(
} }
DARSHAN_MPI_CALL(PMPI_Bcast)(&result, 1, MPI_INT, 0, mod_comm); DARSHAN_MPI_CALL(PMPI_Bcast)(&result, 1, MPI_INT, 0, mod_comm);
if (bgq_runtime && result) /* caclulate the number of I/O nodes */
if (result)
{ {
int i, found; int i, found;
uint64_t val; uint64_t val;
DARSHAN_MPI_CALL(PMPI_Gather)(&bgq_runtime->record.counters[BGQ_INODES], DARSHAN_MPI_CALL(PMPI_Gather)(&bgq_runtime->record->counters[BGQ_INODES],
1, 1,
MPI_LONG_LONG_INT, MPI_LONG_LONG_INT,
ion_ids, ion_ids,
...@@ -262,32 +225,21 @@ static void bgq_get_output_data( ...@@ -262,32 +225,21 @@ static void bgq_get_output_data(
found += 1; found += 1;
} }
} }
bgq_runtime->record.counters[BGQ_INODES] = found; bgq_runtime->record->counters[BGQ_INODES] = found;
} }
} }
if ((bgq_runtime) && (my_rank == 0)) /* non-zero ranks throw out their BGQ record */
{ if (my_rank != 0)
*buffer = &bgq_runtime->record;
*size = sizeof(struct darshan_bgq_record);
}
else
{ {
*buffer = NULL; *buffer = NULL;
*size = 0; *size = 0;
} }
return; free(bgq_runtime);
} bgq_runtime = NULL;
/* Shutdown the BGQ module by freeing up all data structures. */ BGQ_UNLOCK();
static void bgq_shutdown()
{
if (bgq_runtime)
{
free(bgq_runtime);
bgq_runtime = NULL;
}
return; return;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment