Commit 4e6d0c89 authored by Shane Snyder's avatar Shane Snyder

move shared file redux code to get_output func

parent f4df41a3
......@@ -56,38 +56,28 @@
/* module developers provide the following functions to darshan-core */
struct darshan_module_funcs
{
/* perform any necessary pre-shutdown steps */
/* perform any necessary pre-shutdown steps
*
* NOTE: this typically includes disabling wrapper functions so
* darshan-core can shutdown in a consistent state.
*/
void (*begin_shutdown)(void);
/* retrieve module data to write to log file */
/* retrieve module data to write to log file
*
* NOTE: module developers can use this function to run collective
* MPI operations at shutdown time. Typically this functionality
* has been used to reduce records shared globablly (given in the
* 'shared_recs' array) into a single data record.
*/
void (*get_output_data)(
void** buf, /* output parameter to save module buffer address */
int* size /* output parameter to save module buffer size */
MPI_Comm mod_comm, /* MPI communicator to run collectives with */
darshan_record_id *shared_recs, /* list of shared data record ids */
int shared_rec_count, /* count of shared data records */
void** mod_buf, /* output parameter to save module buffer address */
int* mod_buf_sz /* output parameter to save module buffer size */
);
/* shutdown module data structures */
void (*shutdown)(void);
/* (OPTIONAL) perform any necessary steps prior to performing a reduction
* of shared Darshan I/O records. To bypass shared file reduction mechanism,
* set this pointer to NULL.
*/
void (*setup_reduction)(
darshan_record_id *shared_recs, /* input list of shared records */
int *shared_rec_count, /* in/out shared record count */
void **send_buf, /* send buffer for shared file reduction */
void **recv_buf, /* recv buffer for shared file reduction (root only) */
int *rec_size /* size of records being stored for this module */
);
/* (OPTIONAL) perform the actual shared file reduction operation. This
* operation follows the prototype of MPI_Op_create, which allows the
* specification of user-defined combination functions which may be used
* directly by MPI. To bypass shared file reduction mechanism, set this
* pointer to NULL.
*/
void (*record_reduction_op)(
void* infile_v,
void* inoutfile_v,
int *len,
MPI_Datatype *datatype
);
};
/* paths that darshan will not trace */
......
......@@ -452,8 +452,9 @@ void darshan_core_shutdown()
for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
struct darshan_core_module* this_mod = final_core->mod_array[i];
darshan_record_id mod_shared_recs[DARSHAN_CORE_MAX_RECORDS];
struct darshan_core_record_ref *ref = NULL;
darshan_record_id mod_shared_recs[DARSHAN_CORE_MAX_RECORDS];
int mod_shared_rec_cnt = 0;
void* mod_buf = NULL;
int mod_buf_sz = 0;
int j;
......@@ -469,63 +470,30 @@ void darshan_core_shutdown()
}
if(internal_timing_flag)
mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* if all processes used this module, prepare to do a shared file reduction */
if(global_mod_use_count[i] == nprocs)
{
int shared_rec_count = 0;
int rec_sz = 0;
void *red_send_buf = NULL, *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
/* set the shared file list for this module */
memset(mod_shared_recs, 0, DARSHAN_CORE_MAX_RECORDS * sizeof(darshan_record_id));
for(j = 0; j < DARSHAN_CORE_MAX_RECORDS && shared_recs[j] != 0; j++)
{
HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
sizeof(darshan_record_id), ref);
assert(ref);
if(DARSHAN_CORE_MOD_ISSET(ref->global_mod_flags, i))
{
mod_shared_recs[shared_rec_count++] = shared_recs[j];
}
}
mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* if there are globally shared files, do a shared file reduction */
if(shared_rec_count && this_mod->mod_funcs.setup_reduction &&
this_mod->mod_funcs.record_reduction_op)
/* set the shared file list for this module */
memset(mod_shared_recs, 0, DARSHAN_CORE_MAX_RECORDS * sizeof(darshan_record_id));
for(j = 0; j < DARSHAN_CORE_MAX_RECORDS && shared_recs[j] != 0; j++)
{
HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
sizeof(darshan_record_id), ref);
assert(ref);
if(DARSHAN_CORE_MOD_ISSET(ref->global_mod_flags, i))
{
this_mod->mod_funcs.setup_reduction(mod_shared_recs, &shared_rec_count,
&red_send_buf, &red_recv_buf, &rec_sz);
if(shared_rec_count)
{
/* construct a datatype for a file record. This is serving no purpose
* except to make sure we can do a reduction on proper boundaries
*/
DARSHAN_MPI_CALL(PMPI_Type_contiguous)(rec_sz, MPI_BYTE, &red_type);
DARSHAN_MPI_CALL(PMPI_Type_commit)(&red_type);
/* register a reduction operator for this module */
DARSHAN_MPI_CALL(PMPI_Op_create)(this_mod->mod_funcs.record_reduction_op,
1, &red_op);
/* reduce shared file records for this module */
DARSHAN_MPI_CALL(PMPI_Reduce)(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Type_free)(&red_type);
DARSHAN_MPI_CALL(PMPI_Op_free)(&red_op);
}
mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
}
}
/* if module is registered locally, get the corresponding output buffer */
/* if module is registered locally, get the corresponding output buffer
*
* NOTE: this function can be used to run collective operations across
* modules, if there are file records shared globally.
*/
if(this_mod)
{
/* get output buffer from module */
this_mod->mod_funcs.get_output_data(&mod_buf, &mod_buf_sz);
this_mod->mod_funcs.get_output_data(MPI_COMM_WORLD, mod_shared_recs,
mod_shared_rec_cnt, &mod_buf, &mod_buf_sz);
}
final_core->log_header.mod_map[i].off = tmp_off;
......
This diff is collapsed.
......@@ -119,7 +119,8 @@ static struct null_record_runtime* null_record_by_name(const char *name);
/* forward declaration for module functions needed to interface with darshan-core */
static void null_begin_shutdown(void);
static void null_get_output_data(void **buffer, int *size);
static void null_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **null_buf, int *null_buf_sz);
static void null_shutdown(void);
/* macros for obtaining/releasing the "NULL" module lock */
......@@ -203,8 +204,6 @@ static void null_runtime_initialize()
struct darshan_module_funcs null_mod_fns =
{
.begin_shutdown = &null_begin_shutdown,
.setup_reduction = NULL, /* "NULL" module does not do reductions */
.record_reduction_op = NULL, /* "NULL" module does not do reductions */
.get_output_data = &null_get_output_data,
.shutdown = &null_shutdown
};
......@@ -334,17 +333,30 @@ static void null_begin_shutdown()
/* Pass output data for the "NULL" module back to darshan-core to log to file. */
static void null_get_output_data(
void **buffer,
int *size)
MPI_Comm mod_comm,
darshan_record_id *shared_recs,
int shared_rec_count,
void **null_buf,
int *null_buf_sz)
{
assert(null_runtime);
/* NOTE: this function can be used to run collective operations prior to
* shutting down the module, as implied by the MPI communicator passed in
* as the first agrument. Typically, module developers will want to run a
* reduction on shared data records (passed in in the 'shared_recs' array),
* but other collective routines can be run here as well. For a detailed
* example illustrating how to run shared file reductions, consider the
* POSIX or MPIIO instrumentation modules, as they both implement this
* functionality.
*/
/* Just set the output buffer to point at the array of the "NULL" module's
* I/O records, and set the output size according to the number of records
* currently being tracked.
*/
*buffer = (void *)(null_runtime->record_array);
*size = null_runtime->rec_array_ndx * sizeof(struct darshan_null_record);
*null_buf = (void *)(null_runtime->record_array);
*null_buf_sz = null_runtime->rec_array_ndx * sizeof(struct darshan_null_record);
return;
}
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment