From ffeeed89b904d01c077a8b2bbe9adc99a37b2edb Mon Sep 17 00:00:00 2001 From: Shane Snyder Date: Tue, 23 Feb 2016 16:56:24 -0600 Subject: [PATCH] some initial luste mod logic and makefile changes --- darshan-log-format.h | 4 +- darshan-runtime/Makefile.in | 42 ++++++++++++-------- darshan-runtime/lib/darshan-core.c | 4 ++ darshan-runtime/lib/darshan-lustre.c | 57 +++++++++++++++++++++++----- 4 files changed, 81 insertions(+), 26 deletions(-) diff --git a/darshan-log-format.h b/darshan-log-format.h index b534ee8..1d1ea4e 100644 --- a/darshan-log-format.h +++ b/darshan-log-format.h @@ -103,6 +103,7 @@ struct darshan_record #include "darshan-hdf5-log-format.h" #include "darshan-pnetcdf-log-format.h" #include "darshan-bgq-log-format.h" +#include "darshan-lustre-log-format.h" /* X-macro for keeping module ordering consistent */ /* NOTE: first val used to define module enum values, @@ -120,7 +121,8 @@ struct darshan_record X(DARSHAN_MPIIO_MOD, "MPI-IO", DARSHAN_MPIIO_VER, &mpiio_logutils) \ X(DARSHAN_HDF5_MOD, "HDF5", DARSHAN_HDF5_VER, &hdf5_logutils) \ X(DARSHAN_PNETCDF_MOD, "PNETCDF", DARSHAN_PNETCDF_VER, &pnetcdf_logutils) \ - X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils) + X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils) \ + X(DARSHAN_LUSTRE_MOD, "LUSTRE", DARSHAN_LUSTRE_VER, NULL) /* unique identifiers to distinguish between available darshan modules */ /* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1] diff --git a/darshan-runtime/Makefile.in b/darshan-runtime/Makefile.in index e33fc68..7845b65 100644 --- a/darshan-runtime/Makefile.in +++ b/darshan-runtime/Makefile.in @@ -36,6 +36,10 @@ LIBS = -lz @LIBBZ2@ DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po +# TODO: make the lustre module enabled using config options +DARSHAN_STATIC_MOD_OBJS += lib/darshan-lustre.o +DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-lustre.po + ifdef DARSHAN_USE_BGQ DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-bgq.po @@ -49,63 +53,69 @@ lib:: lib/darshan-core-init-finalize.o: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib +lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ lib/darshan-core.o: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib +lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-common.o: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib +lib/darshan-common.o: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-common.po: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib +lib/darshan-common.po: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-null.o: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib +lib/darshan-null.o: lib/darshan-null.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-null.po: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib +lib/darshan-null.po: lib/darshan-null.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-posix.o: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib +lib/darshan-posix.o: lib/darshan-posix.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-posix.po: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib +lib/darshan-posix.po: lib/darshan-posix.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib +lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib +lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib +lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib +lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ -lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib +lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib +lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ lib/darshan-hdf5-stubs.o: lib/darshan-hdf5-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib +lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib $(CC) $(CFLAGS) -c $< -o $@ -lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib +lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib $(CC) $(CFLAGS_SHARED) -c $< -o $@ lib/darshan-pnetcdf-stubs.o: lib/darshan-pnetcdf-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib $(CC) $(CFLAGS) -c $< -o $@ +lib/darshan-lustre.o: lib/darshan-lustre.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib + $(CC) $(CFLAGS) -c $< -o $@ + +lib/darshan-lustre.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib + $(CC) $(CFLAGS_SHARED) -c $< -o $@ + lib/lookup3.o: lib/lookup3.c $(CC) $(CFLAGS) -c $< -o $@ diff --git a/darshan-runtime/lib/darshan-core.c b/darshan-runtime/lib/darshan-core.c index b73a7ce..e738a42 100644 --- a/darshan-runtime/lib/darshan-core.c +++ b/darshan-runtime/lib/darshan-core.c @@ -1743,6 +1743,10 @@ void darshan_core_register_record( DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id); DARSHAN_CORE_UNLOCK(); + /* TODO: call into lustre if functionality enabled and if lustre data has not + * already been captured for this file + */ + if(file_alignment) darshan_block_size_from_path(name, file_alignment); diff --git a/darshan-runtime/lib/darshan-lustre.c b/darshan-runtime/lib/darshan-lustre.c index 72bdd0a..d773082 100644 --- a/darshan-runtime/lib/darshan-lustre.c +++ b/darshan-runtime/lib/darshan-lustre.c @@ -22,19 +22,15 @@ #include "darshan.h" #include "darshan-dynamic.h" -struct lustre_record_runtime -{ - struct darshan_lustre_record *record; - UT_hash_handle hlink; -}; - +/* Lustre record data is currently immutable, so once it's set it should be + * final. Since we won't likely need to search for records after they've been + * created, a simple array is fine for storing them instead of a hash table + */ struct lustre_runtime { - struct lustre_record_runtime *record_runtime_array; struct darshan_lustre_record *record_array; int record_array_size; int record_array_ndx; - struct lustre_record_runtime *record_hash; }; static struct lustre_runtime *lustre_runtime = NULL; @@ -42,6 +38,8 @@ static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZE static int instrumentation_disabled = 0; static int my_rank = -1; +static void lustre_runtime_initialize(void); + static void lustre_begin_shutdown(void); static void lustre_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count, void **lustre_buf, int *lustre_buf_sz); @@ -52,12 +50,49 @@ static void lustre_shutdown(void); void darshan_instrument_lustre_file(char *filepath) { + /* make sure the lustre module is already initialized */ + lustre_runtime_initialize(); /* TODO: implement gathering of lustre data */ return; } +static void lustre_runtime_initialize() +{ + int mem_limit; + struct darshan_module_funcs lustre_mod_fns = + { + .begin_shutdown = &lustre_begin_shutdown, + .get_output_data = &lustre_get_output_data, + .shutdown = &lustre_shutdown + }; + + /* don't do anything if already initialized or instrumenation is disabled */ + if(lustre_runtime || instrumentation_disabled) + return; + + /* register the lustre module with darshan-core */ + darshan_core_register_module( + DARSHAN_LUSTRE_MOD, + &lustre_mod_fns, + &my_rank, + &mem_limit, + NULL); + + /* return if no memory assigned by darshan core */ + if(mem_limit == 0) + return; + + lustre_runtime = malloc(sizeof(*lustre_runtime)); + if(!lustre_runtime) + return; + memset(lustre_runtime, 0, sizeof(*lustre_runtime)); + + + return; +} + /************************************************************************** * Functions exported by Lustre module for coordinating with darshan-core * **************************************************************************/ @@ -83,7 +118,11 @@ static void lustre_get_output_data( { assert(lustre_runtime); - /* TODO: reduce shared lustre records, and set ouptut buffers */ + /* TODO: determine lustre record shared across all processes, + * and have only rank 0 write these records out. No shared + * reductions should be necessary as the Lustre data for a + * given file should be the same on each process + */ return; } -- 2.26.2