modular instrumentation for MDHIM keyval

darshan-log-format.h

@@ -119,6 +119,7 @@ struct darshan_base_record
 #include "darshan-stdio-log-format.h"
 /* DXT */
 #include "darshan-dxt-log-format.h"
+#include "darshan-mdhim-log-format.h"
 
 /* X-macro for keeping module ordering consistent */
 /* NOTE: first val used to define module enum values, 
@@ -141,7 +142,9 @@ struct darshan_base_record
     X(DARSHAN_STDIO_MOD,    "STDIO",    DARSHAN_STDIO_VER,      &stdio_logutils) \
     /* DXT */ \
     X(DXT_POSIX_MOD,       "DXT_POSIX",  DXT_POSIX_VER,         &dxt_posix_logutils) \
-    X(DXT_MPIIO_MOD,       "DXT_MPIIO",  DXT_MPIIO_VER,         &dxt_mpiio_logutils)
+    X(DXT_MPIIO_MOD,       "DXT_MPIIO",  DXT_MPIIO_VER,         &dxt_mpiio_logutils) \
+    X(DARSHAN_MDHIM_MOD,   "MDHIM",      DARSHAN_MDHIM_VER,     &mdhim_logutils)
+
 
 /* unique identifiers to distinguish between available darshan modules */
 /* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]

darshan-mdhim-log-format.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_MDHIM_LOG_FORMAT_H
+#define __DARSHAN_MDHIM_LOG_FORMAT_H
+
+/* current log format version, to support backwards compatibility */
+#define DARSHAN_MDHIM_VER 1
+
+#define MDHIM_COUNTERS \
+    /* number of 'put' function calls */\
+    X(MDHIM_PUTS) \
+    /* larget payload for a 'put' */ \
+    X(MDHIM_PUT_MAX_SIZE)\
+    /* number of 'get' function calls */\
+    X(MDHIM_GETS) \
+    /* largest get */ \
+    X(MDHIM_GET_MAX_SIZE) \
+    /* end of counters */ \
+    X(MDHIM_NUM_INDICES)
+
+#define MDHIM_F_COUNTERS \
+    /* timestamp of the first call to a 'put' function */\
+    X(MDHIM_F_PUT_TIMESTAMP) \
+    /* timer indicating duration of call to 'foo' with max MDHIM_PUT_MAX_DAT value */\
+    X(MDHIM_F_PUT_MAX_DURATION) \
+    X(MDHIM_F_GET_TIMESTAMP) \
+    X(MDHIM_F_GET_MAX_DURATION) \
+    /* end of counters */\
+    X(MDHIM_F_NUM_INDICES)
+
+#define X(a) a,
+/* integer counters for the "NULL" example module */
+enum darshan_mdhim_indices
+{
+    MDHIM_COUNTERS
+};
+
+/* floating point counters for the "NULL" example module */
+enum darshan_mdhim_f_indices
+{
+    MDHIM_F_COUNTERS
+};
+#undef X
+
+/* the darshan_mdhim_record structure encompasses the high-level data/counters
+ * which would actually be logged to file by Darshan for the "MDHIM"
+ * module. This implementation logs the following data for each
+ * record:
+ *      - a darshan_base_record structure, which contains the record id & rank
+ *      - integer I/O counters (operation counts, I/O sizes, etc.)
+ *      - floating point I/O counters (timestamps, cumulative timers, etc.)
+ */
+struct darshan_mdhim_record
+{
+    struct darshan_base_record base_rec;
+    int64_t counters[MDHIM_NUM_INDICES];
+    double fcounters[MDHIM_F_NUM_INDICES];
+};
+
+#endif /* __DARSHAN_MDHIM_LOG_FORMAT_H */

darshan-runtime/Makefile.in

@@ -36,6 +36,7 @@ CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir)
 LIBS = -lz @LIBBZ2@
 
 BUILD_HDF5_MODULE = @BUILD_HDF5_MODULE@
+BUILD_MDHIM_MODULE = @BUILD_MDHIM_MODULE@
 
 DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-pnetcdf.o lib/darshan-stdio.o lib/darshan-dxt.o
 DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-pnetcdf.po lib/darshan-stdio.po lib/darshan-dxt.po
@@ -59,6 +60,13 @@ CFLAGS += -DDARSHAN_LUSTRE
 CFLAGS_SHARED += -DDARSHAN_LUSTRE
 endif
 
+ifdef BUILD_MDHIM_MODULE
+DARSHAN_STATIC_MOD_OBJS += lib/darshan-mdhim.o
+DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-mdhim.po
+CFLAGS += -DDARSHAN_MDHIM
+CFLAGS_SHARED += -DDARSHAN_MDHIM
+endif
+
 lib::
 	@mkdir -p $@
 
@@ -140,6 +148,13 @@ lib/darshan-dxt.o: lib/darshan-dxt.c darshan.h darshan-common.h $(DARSHAN_LOG_FO
 lib/darshan-dxt.po: lib/darshan-dxt.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
+lib/darshan-mdhim.o: lib/darshan-mdhim.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-mdhim.po: lib/darshan-mdhim.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+
 lib/lookup3.o: lib/lookup3.c
 	$(CC) $(CFLAGS) -c $< -o $@
 
@@ -196,6 +211,9 @@ endif
 	install -m 644 $(srcdir)/share/ld-opts/darshan-posix-ld-opts $(datarootdir)/ld-opts/darshan-posix-ld-opts
 ifdef BUILD_HDF5_MODULE
 	install -m 644 $(srcdir)/share/ld-opts/darshan-hdf5-ld-opts $(datarootdir)/ld-opts/darshan-hdf5-ld-opts
+endif
+ifdef BUILD_MDHIM_MODULE
+	install -m 644 $(srcdir)/share/ld-opts/darshan-mdhim-ld-opts $(datarootdir)/ld-opts/darshan-mdhim-ld-opts
 endif
 	install -m 644 $(srcdir)/share/ld-opts/darshan-pnetcdf-ld-opts $(datarootdir)/ld-opts/darshan-pnetcdf-ld-opts
 	install -m 644 $(srcdir)/share/ld-opts/darshan-stdio-ld-opts $(datarootdir)/ld-opts/darshan-stdio-ld-opts

darshan-runtime/configure

@@ -621,6 +621,8 @@ ac_includes_default="\
 
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
+DARSHAN_MDHIM_LD_OPTS
+BUILD_MDHIM_MODULE
 DARSHAN_HDF5_LD_OPTS
 BUILD_HDF5_MODULE
 DARSHAN_USE_LUSTRE
@@ -702,6 +704,7 @@ with_jobid_env
 with_mod_mem
 enable_HDF5_post_1_10
 enable_HDF5_pre_1_10
+enable_mdhim
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1332,6 +1335,7 @@ Optional Features:
                           Enable HDF5 module for HDF5 version 1.10 or later
   --enable-HDF5-pre-1.10
                           Enable HDF5 module for HDF5 versions earlier than 1.10
+  --enable-mdhim          Enable mdhim module
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -3761,6 +3765,7 @@ fi
 
 fi
 
+
 if test x$GOT_ALIGNMENT != x1; then
     as_fn_error $? "must provide --with-mem-align=<num> argument to configure." "$LINENO" 5
 fi
@@ -4223,6 +4228,33 @@ fi
 fi
 
 
+BUILD_MDHIM_MODULE=
+DARSHAN_MDHIM_LD_OPTS=
+# Check whether --enable-mdhim was given.
+if test "${enable_mdhim+set}" = set; then :
+  enableval=$enable_mdhim;
+fi
+
+if test "x$enable_mdhim" = "xyes"; then :
+  BUILD_MDHIM_MODULE=1
+      DARSHAN_MDHIM_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-mdhim-ld-opts"
+      for ac_header in mdhim.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "mdhim.h" "ac_cv_header_mdhim_h" "$ac_includes_default"
+if test "x$ac_cv_header_mdhim_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_MDHIM_H 1
+_ACEOF
+
+else
+  as_fn_error $? "mdhim requested but headers cannot be found" "$LINENO" 5
+fi
+
+done
+
+fi
+
+
 # determine if the MPI library includes MPI-IO functions or not
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for MPI-IO support in MPI" >&5
 $as_echo_n "checking for MPI-IO support in MPI... " >&6; }
@@ -4401,6 +4433,8 @@ DARSHAN_VERSION="3.1.6"
 
 
 
+
+
 ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/ld-opts/darshan-base-ld-opts"
 
 cat >confcache <<\_ACEOF

darshan-runtime/configure.in

@@ -162,6 +162,7 @@ if test x$enable_lustre_mod != xno; then
             []))
 fi
 
+
 if test x$GOT_ALIGNMENT != x1; then
     AC_MSG_ERROR(must provide --with-mem-align=<num> argument to configure.)
 fi
@@ -283,6 +284,22 @@ AC_ARG_ENABLE(HDF5-pre-1.10,
 fi]
 ,)
 
+BUILD_MDHIM_MODULE=
+DARSHAN_MDHIM_LD_OPTS=
+AC_ARG_ENABLE([mdhim],
+	      AS_HELP_STRING([--enable-mdhim], [Enable mdhim module]),
+	      [],[] )
+AS_IF([test "x$enable_mdhim" = "xyes"],
+      BUILD_MDHIM_MODULE=1
+      DARSHAN_MDHIM_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-mdhim-ld-opts"
+      AC_CHECK_HEADERS([mdhim.h],
+		       [],
+		       AC_MSG_ERROR([mdhim requested but headers cannot be found]) ),
+      []
+
+      )
+
+
 # determine if the MPI library includes MPI-IO functions or not
 AC_MSG_CHECKING(for MPI-IO support in MPI)
 AC_TRY_LINK([#include <mpi.h>], [
@@ -382,6 +399,8 @@ AC_SUBST(DARSHAN_USE_BGQ)
 AC_SUBST(DARSHAN_USE_LUSTRE)
 AC_SUBST(BUILD_HDF5_MODULE)
 AC_SUBST(DARSHAN_HDF5_LD_OPTS)
+AC_SUBST(BUILD_MDHIM_MODULE)
+AC_SUBST(DARSHAN_MDHIM_LD_OPTS)
 AC_OUTPUT(Makefile
 darshan-mk-log-dirs.pl
 darshan-gen-cc.pl

darshan-runtime/darshan-runtime-config.h.in

@@ -12,6 +12,9 @@
 /* Define to 1 if you have the `z' library (-lz). */
 #undef HAVE_LIBZ
 
+/* Define to 1 if you have the <mdhim.h> header file. */
+#undef HAVE_MDHIM_H
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 

darshan-runtime/lib/darshan-mdhim.c

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
+#include "darshan-runtime-config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <string.h>
+#include <assert.h>
+
+#include <mdhim.h>
+
+#include "darshan.h"
+#include "darshan-dynamic.h"
+
+#define RECORD_STRING "total-mdhim-obj-stats"
+
+/* The DARSHAN_FORWARD_DECL macro (defined in darshan.h) is used to provide forward
+ * declarations for wrapped funcions, regardless of whether Darshan is used with
+ * statically or dynamically linked executables.
+ */
+DARSHAN_FORWARD_DECL(mdhimPut, struct mdhim_brm_t *, (mdhim_t *md,
+            void *key, int key_len, void *value, int value_len,
+            struct secondary_info *secondary_global_info,
+            struct secondary_info *secondary_local_info));
+
+DARSHAN_FORWARD_DECL(mdhimGet, struct mdhim_bgetrm_t *, (mdhim_t *md,
+        struct index_t *index, void *key, int key_len, int op));
+
+/* The mdhim_record_ref structure maintains necessary runtime metadata
+ * for the MDHIM module record (darshan_mdhim_record structure, defined in
+ * darshan-mdhim-log-format.h) pointed to by 'record_p'. This metadata
+ * assists with the instrumenting of specific statistics in the record.
+ *
+ * RATIONALE: the MDHIM module needs to track some stateful, volatile
+ * information about each record it has registered (for instance, most
+ * recent  access time, amount of bytes transferred) to aid in
+ * instrumentation, but this information can't be stored in the
+ * darshan_mdhim_record struct
+ * because we don't want it to appear in the final darshan log file.  We
+ * therefore associate a mdhim_record_ref struct with each
+ * darshan_mdhim_record struct in order to track this information (i.e.,
+ * the mapping between mdhim_record_ref structs to darshan_mdhim_record
+ * structs is one-to-one).
+ *
+ * NOTE: we use the 'darshan_record_ref' interface (in darshan-common)
+ * to associate different types of handles with this mdhim_record_ref
+ * struct.  This allows us to index this struct (and the underlying
+ * record) by using either the corresponding Darshan record identifier
+ * or by any other arbitrary handle.
+ */
+struct mdhim_record_ref
+{
+    /* Darshan record for the MDHIM example module */
+    struct darshan_mdhim_record *record_p;
+
+    /* ... other runtime data ... */
+};
+
+/* The mdhim_runtime structure maintains necessary state for storing
+ * MDHIM records and for coordinating with darshan-core at shutdown time.
+ */
+struct mdhim_runtime
+{
+    /* rec_id_hash is a pointer to a hash table of MDHIM module record
+     * references, indexed by Darshan record id
+     */
+    void *rec_id_hash;
+    /* number of records currently tracked */
+    int rec_count;
+};
+
+/* internal helper functions for the MDHIM module */
+static void mdhim_runtime_initialize(
+    void);
+static struct mdhim_record_ref *mdhim_track_new_record(
+    darshan_record_id rec_id, const char *name);
+static void mdhim_cleanup_runtime(
+    void);
+
+/* forward declaration for MDHIM shutdown function needed to interface
+ * with darshan-core
+ */
+static void mdhim_shutdown(MPI_Comm mod_comm, darshan_record_id *shared_recs,
+    int shared_rec_count, void **mdhim_buf, int *mdhim_buf_sz);
+
+/* mdhim_runtime is the global data structure encapsulating "MDHIM"
+ * module state */
+static struct mdhim_runtime *mdhim_runtime = NULL;
+/* The mdhim_runtime_mutex is a lock used when updating the
+ * mdhim_runtime global structure (or any other global data structures).
+ * This is necessary to avoid race conditions as multiple threads may
+ * execute function wrappers and update module state.
+ * NOTE: Recursive mutexes are used in case functions wrapped by this
+ * module call other wrapped functions that would result in deadlock,
+ * otherwise. This mechanism may not be necessary for all
+ * instrumentation modules.
+ */
+static pthread_mutex_t mdhim_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+/* my_rank indicates the MPI rank of this process */
+static int my_rank = -1;
+
+/* macros for obtaining/releasing the "MDHIM" module lock */
+#define MDHIM_LOCK() pthread_mutex_lock(&mdhim_runtime_mutex)
+#define MDHIM_UNLOCK() pthread_mutex_unlock(&mdhim_runtime_mutex)
+
+/* the MDHIM_PRE_RECORD macro is executed before performing MDHIM
+ * module instrumentation of a call. It obtains a lock for updating
+ * module data strucutres, and ensure the MDHIM module has been properly
+ * initialized before instrumenting.
+ */
+#define MDHIM_PRE_RECORD() do { \
+    MDHIM_LOCK(); \
+    if(!darshan_core_disabled_instrumentation()) { \
+        if(!mdhim_runtime) mdhim_runtime_initialize(); \
+        if(mdhim_runtime) break; \
+    } \
+    MDHIM_UNLOCK(); \
+    return(ret); \
+} while(0)
+
+/* the MDHIM_POST_RECORD macro is executed after performing MDHIM
+ * module instrumentation. It simply releases the module lock.
+ */
+#define MDHIM_POST_RECORD() do { \
+    MDHIM_UNLOCK(); \
+} while(0)
+
+/* macro for instrumenting the "MDHIM" module's put function */
+#define MDHIM_RECORD_PUT(__ret, __md, __vallen, __tm1, __tm2) do{ \
+    darshan_record_id rec_id; \
+    struct mdhim_record_ref *rec_ref; \
+    double __elapsed = __tm2 - __tm1; \
+    /* if put returns error (return code < 0), don't instrument anything */ \
+    if(__ret < 0) break; \
+    /* posix uses '__name' to generate a unique Darshan record id */ \
+    /* but mdhim doesn't use string names for its keyval store  */ \
+    rec_id = darshan_core_gen_record_id(RECORD_STRING); \
+    /* look up a record reference for this record id using darshan rec_ref interface */ \
+    rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
+    /* if no reference was found, track a new one for this record */ \
+    if(!rec_ref) rec_ref = mdhim_track_new_record(rec_id, RECORD_STRING); \
+    /* if we still don't have a valid reference, back out */ \
+    if(!rec_ref) break; \
+    /* increment counter indicating number of calls to 'put' */ \
+    rec_ref->record_p->counters[MDHIM_PUTS] += 1; \
+    /* store max data value for calls to 'put', and corresponding time duration */ \
+    if(rec_ref->record_p->counters[MDHIM_PUT_MAX_SIZE] < __vallen) { \
+        rec_ref->record_p->counters[MDHIM_PUT_MAX_SIZE] = __vallen; \
+        rec_ref->record_p->fcounters[MDHIM_F_PUT_MAX_DURATION] = __elapsed; \
+    } \
+    /* store timestamp of first call to 'put' */ \
+    if(rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] == 0 || \
+     rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] > __tm1) \
+        rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] = __tm1; \
+} while(0)
+
+/* macro for instrumenting the "MDHIM" module's get function */
+#define MDHIM_RECORD_GET(__ret, __md, __keylen, __tm1, __tm2) do{ \
+    darshan_record_id rec_id; \
+    struct mdhim_record_ref *rec_ref; \
+    double __elapsed = __tm2 - __tm1; \
+    /* if get returns error (return code < 0), don't instrument anything */ \
+    if(__ret == NULL) break; \
+    /* posix uses '__name' to generate a unique Darshan record id */ \
+    /* but mdhim doesn't use string names for its keyval store  */ \
+    rec_id = darshan_core_gen_record_id(RECORD_STRING); \
+    /* look up a record reference for this record id using darshan rec_ref interface */ \
+    rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
+    /* if no reference was found, track a new one for this record */ \
+    if(!rec_ref) rec_ref = mdhim_track_new_record(rec_id, RECORD_STRING); \
+    /* if we still don't have a valid reference, back out */ \
+    if(!rec_ref) break; \
+    /* increment counter indicating number of calls to 'get' */ \
+    rec_ref->record_p->counters[MDHIM_GETS] += 1; \
+    /* store max data value for calls to 'get', and corresponding time duration */ \
+    if(rec_ref->record_p->counters[MDHIM_GET_MAX_SIZE] < __keylen) { \
+        rec_ref->record_p->counters[MDHIM_GET_MAX_SIZE] = __keylen; \
+        rec_ref->record_p->fcounters[MDHIM_F_GET_MAX_DURATION] = __elapsed; \
+    } \
+    /* store timestamp of first call to 'get' */ \
+    if(rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] == 0 || \
+     rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] > __tm1) \
+        rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] = __tm1; \
+} while(0)
+
+/**********************************************************
+ *    Wrappers for "MDHIM" module functions of interest    * 
+ **********************************************************/
+
+/* The DARSHAN_DECL macro provides the appropriate wrapper function
+ * names, depending on whether the Darshan library is statically or
+ * dynamically linked.
+ */
+struct mdhim_brm_t *DARSHAN_DECL(mdhimPut)(mdhim_t *md,
+        void *key, int key_len,
+        void *value, int value_len,
+        struct secondary_info *secondary_global_info,
+        struct secondary_info * secondary_local_info)
+{
+    struct mdhim_brm_t *ret;
+    double tm1, tm2;
+
+    /* The MAP_OR_FAIL macro attempts to obtain the address of the actual
+     * underlying put function call (__real_put), in the case of LD_PRELOADing
+     * the Darshan library. For statically linked executables, this macro is
+     * just a NOP. 
+     */
+    MAP_OR_FAIL(mdhimPut);
+
+    /* In general, Darshan wrappers begin by calling the real version of the
+     * given wrapper function. Timers are used to record the duration of this
+     * operation. */
+    tm1 = darshan_core_wtime();
+    ret = __real_mdhimPut(md, key, key_len, value, value_len,
+            secondary_global_info, secondary_global_info);
+    tm2 = darshan_core_wtime();
+
+    MDHIM_PRE_RECORD();
+    /* Call macro for instrumenting data for mdhimPut function calls. */
+    /* TODO: call the mdhim hash routines and instrument which servers
+     * get this request */
+    MDHIM_RECORD_PUT(ret, md, value_len, tm1, tm2);
+    MDHIM_POST_RECORD();
+
+    return(ret);
+}
+
+struct mdhim_bgetrm_t * DARSHAN_DECL(mdhimGet)(mdhim_t *md,
+        struct index_t *index, void *key, int key_len, int op)
+{
+    struct mdhim_bgetrm_t *ret;
+    double tm1, tm2;
+
+    MAP_OR_FAIL(mdhimGet);
+
+    /* In general, Darshan wrappers begin by calling the real version of the
+     * given wrapper function. Timers are used to record the duration of this
+     * operation. */
+    tm1 = darshan_core_wtime();
+    ret = __real_mdhimGet(md, index, key, key_len, op);
+    tm2 = darshan_core_wtime();
+
+    MDHIM_PRE_RECORD();
+    /* Call macro for instrumenting data for get function calls. */
+    MDHIM_RECORD_GET(ret, md, key_len, tm1, tm2);
+    MDHIM_POST_RECORD();
+    return ret;
+}
+
+/**********************************************************
+ * Internal functions for manipulating MDHIM module state *
+ **********************************************************/
+
+/* Initialize internal MDHIM module data structures and register with
+ * darshan-core. */
+static void mdhim_runtime_initialize()
+{
+    int mdhim_buf_size;
+
+    /* try and store a default number of records for this module */
+    mdhim_buf_size = DARSHAN_DEF_MOD_REC_COUNT *
+        sizeof(struct darshan_mdhim_record);
+
+    /* register the MDHIM module with the darshan-core component */
+    darshan_core_register_module(
+        DARSHAN_MDHIM_MOD,   /* Darshan module identifier, defined in darshan-log-format.h */
+        &mdhim_shutdown,
+        &mdhim_buf_size,
+        &my_rank,
+        NULL);
+
+    /* return if darshan-core does not provide enough module memory for at 
+     * least one MDHIM record
+     */
+    if(mdhim_buf_size < sizeof(struct darshan_mdhim_record))
+    {
+        darshan_core_unregister_module(DARSHAN_MDHIM_MOD);
+        return;
+    }
+
+    /* initialize module's global state */
+    mdhim_runtime = calloc(1, sizeof(*mdhim_runtime));
+    if(!mdhim_runtime)
+    {
+        darshan_core_unregister_module(DARSHAN_MDHIM_MOD);
+        return;
+    }
+    return;
+}
+
+/* allocate and track a new MDHIM module record */
+static struct mdhim_record_ref *mdhim_track_new_record(
+    darshan_record_id rec_id, const char *name)
+{
+    struct darshan_mdhim_record *record_p = NULL;
+    struct mdhim_record_ref *rec_ref = NULL;
+    int ret;
+
+    rec_ref = calloc(1, sizeof(*rec_ref));
+    if(!rec_ref)
+        return(NULL);
+
+    /* allocate a new MDHIM record reference and add it to the hash
+     * table, using the Darshan record identifier as the handle
+     */
+    ret = darshan_add_record_ref(&(mdhim_runtime->rec_id_hash), &rec_id,
+        sizeof(darshan_record_id), rec_ref);
+    if(ret == 0)
+    {
+        free(rec_ref);
+        return(NULL);
+    }
+
+    /* register the actual file record with darshan-core so it is persisted
+     * in the log file
+     */
+    record_p = darshan_core_register_record(
+        rec_id,
+        name,
+        DARSHAN_MDHIM_MOD,
+        sizeof(struct darshan_mdhim_record),
+        NULL);
+
+    if(!record_p)
+    {
+        /* if registration fails, delete record reference and return */
+        darshan_delete_record_ref(&(mdhim_runtime->rec_id_hash),
+            &rec_id, sizeof(darshan_record_id));
+        free(rec_ref);
+        return(NULL);
+    }
+
+    /* registering this file record was successful, so initialize some fields */
+    record_p->base_rec.id = rec_id;
+    record_p->base_rec.rank = my_rank;
+    rec_ref->record_p = record_p;
+    mdhim_runtime->rec_count++;
+
+
+    /* return pointer to the record reference */
+    return(rec_ref);
+}
+
+/* cleanup MDHIM module internal data structures */
+static void mdhim_cleanup_runtime()
+{
+    /* iterate the hash of record references and free them */
+    darshan_clear_record_refs(&(mdhim_runtime->rec_id_hash), 1);
+
+    free(mdhim_runtime);
+    mdhim_runtime = NULL;
+
+    return;
+}
+
+/***********************************************************************
+ * shutdown function exported by the MDHIM module for coordinating with
+ * darshan-core *
+ ***********************************************************************/
+
+/* Pass output data for the MDHIM module back to darshan-core to log to
+ * file, and shutdown/free internal data structures.
+ */
+static void mdhim_shutdown(
+    MPI_Comm mod_comm,
+    darshan_record_id *shared_recs,
+    int shared_rec_count,
+    void **mdhim_buf,
+    int *mdhim_buf_sz)