Commit 72380aad authored by Shane Snyder's avatar Shane Snyder
Browse files

more docs and added unregister functions

parent acc05e4a
......@@ -63,12 +63,15 @@ void darshan_core_register_module(
struct darshan_module_funcs *funcs,
int *runtime_mem_limit);
void darshan_core_unregister_module(
darshan_module_id mod_id);
void darshan_core_register_record(
void *name,
int len,
int printable_flag,
darshan_module_id mod_id,
darshan_record_id *id);
darshan_record_id *rec_id);
void darshan_core_unregister_record(
darshan_record_id rec_id,
......@@ -32,7 +32,7 @@ extern char* __progname;
/* internal variable delcarations */
static struct darshan_core_runtime *darshan_core = NULL;
static pthread_mutex_t darshan_core_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t darshan_core_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;
static int nprocs = -1;
......@@ -1484,19 +1484,19 @@ static int darshan_log_coll_write(MPI_File log_fh, void *buf, int count,
/* ********************************************************* */
void darshan_core_register_module(
darshan_module_id id,
darshan_module_id mod_id,
struct darshan_module_funcs *funcs,
int *runtime_mem_limit)
struct darshan_core_module* mod;
*runtime_mem_limit = 0;
if(!darshan_core || (id >= DARSHAN_MAX_MODS))
if(!darshan_core || (mod_id >= DARSHAN_MAX_MODS))
/* see if this module is already registered */
/* if module is already registered just return */
/* NOTE: we do not recalculate memory limit here, just set to 0 */
......@@ -1513,11 +1513,11 @@ void darshan_core_register_module(
memset(mod, 0, sizeof(*mod));
mod->id = id;
mod->id = mod_id;
mod->mod_funcs = *funcs;
/* register module with darshan */
darshan_core->mod_array[id] = mod;
darshan_core->mod_array[mod_id] = mod;
/* TODO: something smarter than just 2 MiB per module */
*runtime_mem_limit = 2 * 1024 * 1024;
......@@ -1527,6 +1527,31 @@ void darshan_core_register_module(
/* TODO: implement & test*/
void darshan_core_unregister_module(
darshan_module_id mod_id)
struct darshan_core_record_ref *ref, *tmp;
/* iterate all records and disassociate this module from them */
HASH_ITER(hlink, darshan_core->rec_hash, ref, tmp)
darshan_core_unregister_record(ref->, mod_id);
darshan_core->mod_array[mod_id] = NULL;
void darshan_core_register_record(
void *name,
int len,
......@@ -1579,6 +1604,7 @@ void darshan_core_register_record(
/* TODO: test */
void darshan_core_unregister_record(
darshan_record_id rec_id,
darshan_module_id mod_id)
......@@ -103,6 +103,7 @@ static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER
static int instrumentation_disabled = 0;
static int my_rank = -1;
/* TODO: I'm sure these should be applied on all modules */
/* these are paths that we will not trace */
static char* exclusions[] = {
......@@ -362,7 +362,6 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
/* TODO: check for duplicate entries? */
buf_ptr = hash_buf;
while(buf_ptr < (hash_buf + hash_buf_sz))
......@@ -377,6 +376,16 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
path_ptr = (char *)buf_ptr;
buf_ptr += *path_len_ptr;
/* we need to sort out endianness issues before deserializing */
HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
ref = malloc(sizeof(*ref));
......@@ -389,13 +398,6 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
/* we need to sort out endianness issues before deserializing */
/* set the fields for this record */
ref-> = *rec_id_ptr;
memcpy(ref->, path_ptr, *path_len_ptr);
......@@ -404,6 +406,7 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
/* add this record to the hash */
HASH_ADD(hlink, *hash,, sizeof(darshan_record_id), ref);
......@@ -476,15 +479,13 @@ int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
*mod_buf = tmp_buf;
*mod_buf_sz = tmp_buf_sz;
/* TODO: bswaps */
#if 0
/* TODO: hardcoded for posix -- what can we do generally?
* different function for each module and a way to map to this function?
/* TODO: we need bswaps here, too */
int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
char *comp_buf;
......@@ -544,7 +545,6 @@ int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
fprintf(stderr, "Error: %s\n", err_string);
/* darshan_log_close()
......@@ -563,7 +563,7 @@ void darshan_log_close(darshan_fd fd)
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* ******************************************* */
/* return 0 on successful seek to offset, -1 on failure.
......@@ -168,11 +168,18 @@ int main(int argc, char **argv)
/* TODO: move this functionality somewhere else so it can be reused */
/* loop over the POSIX file records and print out counters */
for(i = 0; i < (posix_mod_buf_sz / sizeof(struct darshan_posix_file)); i++)
struct darshan_posix_file next_rec = posix_mod_buf[i];
/* get the pathname for this record */
HASH_FIND(hlink, rec_hash, &next_rec.f_id, sizeof(darshan_record_id), ref);
......@@ -42,9 +42,9 @@[darshan-
necessary steps for building these repositories should not have changed in the new version of
== Architectural overview
== Darshan dev-modular overview
The Darshan source tree is composed of two primary components:
The Darshan source tree is organized into two primary components:
* *darshan-runtime*: Darshan runtime environment necessary for instrumenting MPI
applications and generating I/O characterization logs.
......@@ -55,22 +55,303 @@ I/O characterization log.
The following subsections provide an overview of each of these components with specific
attention to how new instrumentation modules may be integrated into Darshan.
=== darshan-runtime
=== Darshan-runtime
At a high level, the darshan-runtime library is responsible for instrumenting MPI applications
and generating a log file containing the resulting I/O characterization.
The primary responsibilities of the darshan-runtime component are:
The I/O behavior an application is primarily instrumented by intercepting function calls of
interest and recording relevant data.
* intercepting I/O functions of interest from a target application;
// TODO: how does dynamic vs static executable affect a module developer?
* extracting statistics, timing information, and other data characterizing the application's I/O workload;
* compressing I/O characterization data and corresponding metadata;
* logging the compressed I/O characterization to file for future evaluation
=== darshan-util
The first two responsibilities are the burden of the instrumentation module developer, while the last
two are handled automatically by Darshan.
==== Instrumentation modules
The wrapper functions used to intercept I/O function calls of interest are central to the design of
any Darshan instrumentation module. These wrappers are used to extract pertinent I/O data from
the function call and persist this data in some state structure maintained by the module. The wrappers
are inserted at compile time for statically linked executables (e.g., using the linkers `--wrap`
mechanism) and at runtime for dynamically linked executables (using LD_PRELOAD).
*NOTE*: Modules should not perform any I/O or communication within wrapper functions. Darshan records
I/O data independently on each application process, then merges the data from all processes when the
job is shutting down. This defers expensive I/O and communication operations to the shutdown process,
limiting Darshan's impact on application I/O performance.
When the instrumented application terminates and Darshan begins its shutdown procedure, it requires
a way to interface with any active modules that have data to contribute to the output I/O characterization.
Darshan requires that module developers implement the following functions to allow the Darshan runtime
environment to coordinate with modules while shutting down:
struct darshan_module_funcs
void (*disable_instrumentation)(void);
void (*prepare_for_reduction)(
darshan_record_id *shared_recs,
int *shared_rec_count,
void **send_buf,
void **recv_buf,
int *rec_size
void (*reduce_records)(
void* a,
void* b,
int *len,
MPI_Datatype *datatype
void (*get_output_data)(
void** buf,
int* size
void (*shutdown)(void);
This function informs the module that Darshan is about to begin shutting down. It should disable
all wrappers and stop updating internal data structures to ensure data consistency and avoid
other race conditions.
Since Darshan aggregates shared data records (i.e., records which all application processes
accessed) into a single record, module developers must provide mechanisms for performing a reduction
on these records.
This function is used to prepare a module for performing a reduction operation. In general, this
just involves providing the input buffers to the reduction, and (on rank 0 only) providing output
buffer space to store the result of the reduction.
* _shared_recs_ is a set of Darshan record identifiers which are associated with this module.
These are the records which need to be reduced into single shared data records.
* _shared_rec_count_ is a pointer to an integer storing the number of shared records will
be reduced by this module. When the function is called this variable points to the number
of shared records detected by Darshan, but the module can decide not to reduce any number
of these records. Upon completion of the function, this variable should point to the number
of shared records to perform reductions on (i.e., the size of the input and output buffers).
* _send_buf_ is a pointer to the address of the send buffer used for performing the reduction
operation. Upon completion, this variable should point to a buffer containing *_shared_rec_count_
records that will be reduced.
* _recv_buf_ is a pointer to the address of the receive bufffer used for performing the reduction
operation. Upon completion, this variable should point to a buffer containing *_shared_rec_count_
records that will be reduced. This variable is only valid on the root process (rank 0). This
buffer address needs to be stored with module state, as it will be needed when retrieiving
the final output buffers from this module.
* _rec_size_ is just the size of the record structure being reduced for this module.
This is the function which performs the actual shared record reduction operation. The prototype
of this function matches that of the user function provided to the MPI_Op_create function. Refer
to the[documentation] for further
This function is responsible for passing back a single buffer storing all data this module is
contributing to the output I/O characterization. On rank 0, this may involve copying the results
of the shared record reduction into the output buffer.
* _buf_ is a pointer to the address of the buffer this module is contributing to the I/O
* _size_ is the size of this module's output buffer.
This function is a signal from Darshan that it is safe to shutdown. It should clean up and free
all internal data structures.
==== darshan-core
Within darshan-runtime, the darshan-core component manages the initialization and shutdown of the
Darshan environment, provides instrumentation module developers an interface for registering modules
with Darshan, and manages the compressing and the writing of the resultant I/O charracterization.
Each of the functions defined by this interface are explained in detail below.
void darshan_core_register_module(
darshan_module_id mod_id,
struct darshan_module_funcs *funcs,
int *runtime_mem_limit);
The `darshan_core_register_module` function registers Darshan instrumentation modules with the
darshan-core runtime environment. This function needs to be called at least once for any module
that will contribute data to Darshan's final I/O characterization.
* _mod_id_ is a unique identifier for the given module, which is defined in the Darshan log
format header file (darshan-log-format.h).
* _funcs_ is the structure of function pointers (as described above) that a module developer must
provide to interface with the darshan-core runtime.
* _runtime_mem_limit_ is a pointer to an integer which will store the amount of memory Darshan
allows this module to use at runtime. Currently, darshan-core will hardcode this value to 2 MiB,
but in the future this may be changed to optimize Darshan's memory footprint. Note that Darshan
does not allocate any memory for modules, it just informs a module how much memory it can use.
void darshan_core_unregister_module(
darshan_module_id mod_id);
The `darshan_core_unregister_module` function disassociates the given module from the
darshan-core runtime. Consequentially, Darshan does not interface with the given module at
shutdown time and will not log any I/O data from the module. This function should only be used
if a module registers itself with darshan-core but later decides it does not want to contribute
any I/O data.
* _mod_id_ is the unique identifer for the module being unregistered.
void darshan_core_register_record(
void *name,
int len,
int printable_flag,
darshan_module_id mod_id,
darshan_record_id *rec_id);
The `darshan_core_register_record` function registers some data record with the darshan-core
runtime. This record could reference a POSIX file or perhaps an object identifier for an
object storage system, for instance. A unique identifier for the given record name is
generated by Darshan, which should then be used by the module for referencing the corresponding
record. This allows multiple modules to refer to a specific data record in a consistent manner
and also provides a mechanism for mapping these records back to important metadata stored by
darshan-core. It is safe (and likely necessary) to call this function many times for the same
record -- darshan-core will just set the corresponding record identifier if the record has
been previously registered.
* _name_ is just the name of the data record, which could be a file path, object ID, etc.
* _len_ is the size of the input record name. For string record names, this would just be the
string length, but for nonprintable record names (e.g., an integer object identifier), this
is the size of the record name type.
== Adding new modules
* _printable_flag_ indicates whether the input record name is a printable ASCII string.
* _mod_id_ is the identifier for the module attempting to register this record.
* _rec_id_ is a pointer to a variable which will store the unique record identifier generated
by Darshan.
void darshan_core_unregister_record(
darshan_record_id rec_id,
darshan_module_id mod_id);
The `darshan_core_unregister_record` functoin disassociates the given module identifier from the
given record identifier. If no other modules are associated with the given record identifier, then
Darshan removes all internal references to the record. This function should only be used if a
module registers a record with darshan-core, but later decides not to store the record internally.
* _rec_id_ is the record identifier we want to unregister.
* _mod_id_ is the module identifier that is unregistering _rec_id_.
double darshan_core_wtime(void);
The `darshan_core_wtime` function simply returns a floating point number of seconds since
Darshan was initialized. This functionality can be used to time the duration of application
I/O calls or to store timestamps of when functions of interest were called.
==== darshan-common
darshan-common is a utility component of darshan-runtime, providing module developers with
general functions that are likely to be reused across multiple modules. These functions are
distinct from darshan-core functions since they do not require access to internal Darshan
char* darshan_clean_file_path(
const char* path);
The `darshan_clean_file_path` function just cleans up the input path string, converting
relative paths to absolute paths and suppressing any potential noise within the string.
* _path_ is the input path string to be cleaned up.
As more modules are contributed, it is likely that more functionality can be refactored out
of module implementations and maintained in darshan-common, facilitating code reuse and
simplifying maintenance.
=== Darshan-util
== Adding new instrumentation modules
In this section we outline each step necessary to adding a module to Darshan.
=== Log format headers
The following modifications to Darshan log format headers are required for defining
the module's record structure:
* Add module identifier to darshan_module_id enum and add module string name to the
darshan_module_name array in `darshan-log-format.h`.
* Add a top-level header that defines a data record structure for the module. An exemplar
log header for the POSIX instrumentation module is given in `darshan-posix-log-format.h`.
=== Darshan-runtime
==== Build modifications
The following modifications to the darshan-runtime build system are necessary to integrate
new instrumentation modules:
* Necessary linker flags for wrapping this module's functions need to be added to the definition
of `CP_WRAPPERS` in ``.
* Targets must be added to `` to build static and shared objects for the module's
source files, which will be stored in the `lib/` directory. The prerequisites to building
static and dynamic versions of `lib-darshan` must be updated to include these objects, as well.
It is necessary to rerun the `prepare` script and reconfigure darshan-runtime for these changes
to take effect.
==== Instrumentation module implementation
An exemplar instrumentation module for POSIX I/O functions is given in `lib/darshan-posix.c` as
reference. In addtion to the development notes from above and the reference POSIX module, we
provide the following notes to assist module developers:
* Modules only need to include the `darshan.h` header to interface with darshan-core.
* Lacking a way to bootstrap themselves, modules will have to include some logic in their
wrappers to initialize necessary module state if initialization has not already occurred.
- Part of this initialization process should be registering the module with darshan-core,
since this informs the module how much memory it may allocate.
* The file record identifier given when registering a record with darshan-core can be used
to store the record structure in a hash table or some other structure.
- The `darshan_core_register_record` function is really more like a lookup function. It
may be called multiple times for the same record -- if the record already exists, the function
simply returns its record ID.
- It may be necessary to maintain a separate hash table for other handles which the module
may use to refer to a given record. For instance, the POSIX module may need to look up a
file record based on a given file descriptor, rather than a path name.
=== Darshan-util
==== Build modifications
== Other resources
*[Darshan website]
*[darshan-runtime documentation]
*[darshan-util documentation]
*[Darshan-users mailing list]
*[Darshan trac page]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment