Check in Darshan eXtended Lustre Tracing(DXLT) code

1. Use Lustre Module in Darshan
2. Integrate DXLT IO tracing code into one module
3. Add hooks in POSIX and MPI Modules
4. Add Intel copyright
Signed-off-by: Cong Xu <cong.xu@intel.com>

darshan-dxlt-log-format.h

+/*
+ * Copyright (C) 2016 Intel Corporation.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#ifndef __DARSHAN_DXLT_LOG_FORMAT_H
+#define __DARSHAN_DXLT_LOG_FORMAT_H
+
+/* current DXLT log format version */
+#define DXLT_POSIX_VER 1
+#define DXLT_MPIIO_VER 1
+
+/*
+ * DXLT, the segment_info structure maintains detailed Segment IO tracing
+ * information
+ */
+typedef struct segment_info {
+    int64_t offset;
+    int64_t length;
+    double start_time;
+    double end_time;
+} segment_info;
+
+#define X(a) a,
+#undef X
+
+/* file record structure for DXLT files. a record is created and stored for
+ * every DXLT file opened by the original application. For the DXLT module,
+ * the record includes:
+ *      - a darshan_base_record structure, which contains the record id & rank
+ *      - integer file I/O statistics (open, read/write counts, etc)
+ *      - floating point I/O statistics (timestamps, cumulative timers, etc.)
+ */
+struct dxlt_file_record {
+    struct darshan_base_record base_rec;
+    int64_t shared_record; /* -1 means it is a shared file record */
+
+    int32_t stripe_size;
+    int32_t stripe_count;
+    OST_ID *ost_ids;
+
+    int64_t write_count;
+    int64_t write_available_buf;
+    segment_info *write_traces;
+
+    int64_t read_count;
+    int64_t read_available_buf;
+    segment_info *read_traces;
+};
+
+#endif /* __DARSHAN_DXLT_LOG_FORMAT_H */

darshan-log-format.h

@@ -117,6 +117,8 @@ struct darshan_base_record
 #include "darshan-bgq-log-format.h"
 #include "darshan-lustre-log-format.h"
 #include "darshan-stdio-log-format.h"
+/* DXLT */
+#include "darshan-dxlt-log-format.h"
 
 /* X-macro for keeping module ordering consistent */
 /* NOTE: first val used to define module enum values, 
@@ -136,7 +138,10 @@ struct darshan_base_record
     X(DARSHAN_PNETCDF_MOD,  "PNETCDF",  DARSHAN_PNETCDF_VER,    &pnetcdf_logutils) \
     X(DARSHAN_BGQ_MOD,      "BG/Q",     DARSHAN_BGQ_VER,        &bgq_logutils) \
     X(DARSHAN_LUSTRE_MOD,   "LUSTRE",   DARSHAN_LUSTRE_VER,     &lustre_logutils) \
-    X(DARSHAN_STDIO_MOD,    "STDIO",    DARSHAN_STDIO_VER,      &stdio_logutils)
+    X(DARSHAN_STDIO_MOD,    "STDIO",    DARSHAN_STDIO_VER,      &stdio_logutils) \
+    /* DXLT */ \
+    X(DXLT_POSIX_MOD,       "X_POSIX",  DXLT_POSIX_VER,         &dxlt_posix_logutils) \
+    X(DXLT_MPIIO_MOD,       "X_MPIIO",  DXLT_MPIIO_VER,         &dxlt_mpiio_logutils)
 
 /* unique identifiers to distinguish between available darshan modules */
 /* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]

darshan-runtime/Makefile.in

@@ -35,8 +35,8 @@ CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir)
 
 LIBS = -lz @LIBBZ2@
 
-DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o lib/darshan-stdio.o
-DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po lib/darshan-stdio.po
+DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o lib/darshan-stdio.o lib/darshan-dxlt.o
+DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po lib/darshan-stdio.po lib/darshan-dxlt.po
 
 ifdef DARSHAN_USE_BGQ
 DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o
@@ -127,6 +127,12 @@ lib/darshan-stdio.o: lib/darshan-stdio.c darshan.h darshan-common.h $(DARSHAN_LO
 lib/darshan-stdio.po: lib/darshan-stdio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-stdio-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
+lib/darshan-dxlt.o: lib/darshan-dxlt.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxlt-log-format.h | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-dxlt.po: lib/darshan-dxlt.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxlt-log-format.h | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+
 lib/lookup3.o: lib/lookup3.c
 	$(CC) $(CFLAGS) -c $< -o $@
 

darshan-runtime/darshan-core.h

@@ -38,11 +38,12 @@
 #define DARSHAN_DEF_MMAP_LOG_PATH "/tmp"
 #endif
 
+/* DXLT */
 /* Maximum amount of memory per instrumentation module in MiB */
 #ifdef __DARSHAN_MOD_MEM_MAX
 #define DARSHAN_MOD_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
 #else
-#define DARSHAN_MOD_MEM_MAX (2 * 1024 * 1024) /* 2 MiB default */
+#define DARSHAN_MOD_MEM_MAX (4 * 1024 * 1024) /* 4 MiB default */
 #endif
 
 /* default name record buf can store 2048 records of size 100 bytes */

darshan-runtime/lib/darshan-lustre.c

@@ -177,6 +177,27 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
     return;
 }
 
+/* DXLT */
+void dxlt_get_lustre_stripe_info(
+        darshan_record_id rec_id,
+        struct dxlt_file_record *file_rec)
+{
+    struct lustre_record_ref *rec_ref;
+    int ost_ids_size = 0;
+
+    rec_ref = darshan_lookup_record_ref(lustre_runtime->record_id_hash,
+        &rec_id, sizeof(darshan_record_id));
+    assert(rec_ref);
+
+    file_rec->stripe_count = rec_ref->record->counters[LUSTRE_STRIPE_WIDTH];
+    file_rec->stripe_size = rec_ref->record->counters[LUSTRE_STRIPE_SIZE];
+
+    ost_ids_size = file_rec->stripe_count * sizeof(OST_ID);
+    file_rec->ost_ids = (OST_ID*) malloc(ost_ids_size);
+
+    memcpy(file_rec->ost_ids, rec_ref->record->ost_ids, ost_ids_size);
+}
+
 static void lustre_runtime_initialize()
 {
     int lustre_buf_size;

darshan-runtime/lib/darshan-mpiio.c

@@ -87,6 +87,17 @@ static void mpiio_shutdown(
     MPI_Comm mod_comm, darshan_record_id *shared_recs,
     int shared_rec_count, void **mpiio_buf, int *mpiio_buf_sz);
 
+/* DXLT */
+extern void dxlt_mpiio_runtime_initialize();
+extern void dxlt_mpiio_track_new_file_record(
+    darshan_record_id rec_id, const char *path);
+extern void dxlt_mpiio_add_record_ref(
+    darshan_record_id rec_id, MPI_File fh);
+extern void dxlt_mpiio_write(MPI_File fh, int64_t length,
+    double start_time, double end_time);
+extern void dxlt_mpiio_read(MPI_File fh, int64_t length,
+    double start_time, double end_time);
+
 static struct mpiio_runtime *mpiio_runtime = NULL;
 static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
 static int instrumentation_disabled = 0;
@@ -98,7 +109,13 @@ static int my_rank = -1;
 #define MPIIO_PRE_RECORD() do { \
     MPIIO_LOCK(); \
     if(!instrumentation_disabled) { \
-        if(!mpiio_runtime) mpiio_runtime_initialize(); \
+        if(!mpiio_runtime) { \
+            mpiio_runtime_initialize(); \
+            /* DXLT */ \
+            if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+                dxlt_mpiio_runtime_initialize(); \
+            } \
+        } \
         if(mpiio_runtime) break; \
     } \
     MPIIO_UNLOCK(); \
@@ -123,7 +140,13 @@ static int my_rank = -1;
     } \
     rec_id = darshan_core_gen_record_id(newpath); \
     rec_ref = darshan_lookup_record_ref(mpiio_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
-    if(!rec_ref) rec_ref = mpiio_track_new_file_record(rec_id, newpath); \
+    if(!rec_ref) { \
+        rec_ref = mpiio_track_new_file_record(rec_id, newpath); \
+        /* DXLT */ \
+        if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+            dxlt_mpiio_track_new_file_record(rec_id, newpath); \
+        } \
+    } \
     if(!rec_ref) { \
         if(newpath != __path) free(newpath); \
         break; \
@@ -143,6 +166,10 @@ static int my_rank = -1;
         __tm1, __tm2, rec_ref->last_meta_end); \
     darshan_add_record_ref(&(mpiio_runtime->fh_hash), &__fh, sizeof(MPI_File), rec_ref); \
     if(newpath != __path) free(newpath); \
+    /* DXLT */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_mpiio_add_record_ref(rec_id, __fh); \
+    } \
 } while(0)
 
 #define MPIIO_RECORD_READ(__ret, __fh, __count, __datatype, __counter, __tm1, __tm2) do { \
@@ -154,6 +181,10 @@ static int my_rank = -1;
     if(!rec_ref) break; \
     DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size);  \
     size = size * __count; \
+    /* DXLT to record detailed read tracing information */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_mpiio_read(__fh, size, __tm1, __tm2); \
+    } \
     DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
     darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
         &(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
@@ -183,6 +214,10 @@ static int my_rank = -1;
     if(!rec_ref) break; \
     DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size);  \
     size = size * __count; \
+     /* DXLT to record detailed write tracing information */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_mpiio_write(__fh, size, __tm1, __tm2); \
+    } \
     DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
     darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
         &(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \

darshan-runtime/lib/darshan-posix.c

@@ -157,6 +157,17 @@ static void posix_shutdown(
     MPI_Comm mod_comm, darshan_record_id *shared_recs,
     int shared_rec_count, void **posix_buf, int *posix_buf_sz);
 
+/* DXLT */
+extern void dxlt_posix_runtime_initialize();
+extern void dxlt_posix_track_new_file_record(
+    darshan_record_id rec_id, const char *path);
+extern void dxlt_posix_add_record_ref(
+    darshan_record_id rec_id, int fd);
+extern void dxlt_posix_write(int fd, int64_t offset, int64_t length,
+    double start_time, double end_time);
+extern void dxlt_posix_read(int fd, int64_t offset, int64_t length,
+    double start_time, double end_time);
+
 static struct posix_runtime *posix_runtime = NULL;
 static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
 static int instrumentation_disabled = 0;
@@ -169,7 +180,13 @@ static int darshan_mem_alignment = 1;
 #define POSIX_PRE_RECORD() do { \
     POSIX_LOCK(); \
     if(!instrumentation_disabled) { \
-        if(!posix_runtime) posix_runtime_initialize(); \
+        if(!posix_runtime) { \
+            posix_runtime_initialize(); \
+            /* DXLT */ \
+            if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+                dxlt_posix_runtime_initialize(); \
+            } \
+        } \
         if(posix_runtime) break; \
     } \
     POSIX_UNLOCK(); \
@@ -193,7 +210,13 @@ static int darshan_mem_alignment = 1;
     } \
     rec_id = darshan_core_gen_record_id(newpath); \
     rec_ref = darshan_lookup_record_ref(posix_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
-    if(!rec_ref) rec_ref = posix_track_new_file_record(rec_id, newpath); \
+    if(!rec_ref) { \
+        rec_ref = posix_track_new_file_record(rec_id, newpath); \
+        /* DXLT */ \
+        if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+            dxlt_posix_track_new_file_record(rec_id, newpath); \
+        } \
+    } \
     if(!rec_ref) { \
         if(newpath != __path) free(newpath); \
         break; \
@@ -213,6 +236,10 @@ static int darshan_mem_alignment = 1;
     darshan_add_record_ref(&(posix_runtime->fd_hash), &__ret, sizeof(int), rec_ref); \
     darshan_instrument_fs_data(rec_ref->fs_type, newpath, __ret); \
     if(newpath != __path) free(newpath); \
+    /* DXLT */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_posix_add_record_ref(rec_id, __ret); \
+    } \
 } while(0)
 
 #define POSIX_RECORD_READ(__ret, __fd, __pread_flag, __pread_offset, __aligned, __tm1, __tm2) do { \
@@ -228,6 +255,10 @@ static int darshan_mem_alignment = 1;
         this_offset = __pread_offset; \
     else \
         this_offset = rec_ref->offset; \
+    /* DXLT to record detailed read tracing information */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_posix_read(__fd, this_offset, __ret, __tm1, __tm2); \
+    } \
     if(this_offset > rec_ref->last_byte_read) \
         rec_ref->file_rec->counters[POSIX_SEQ_READS] += 1;  \
     if(this_offset == (rec_ref->last_byte_read + 1)) \
@@ -282,6 +313,10 @@ static int darshan_mem_alignment = 1;
         this_offset = __pwrite_offset; \
     else \
         this_offset = rec_ref->offset; \
+    /* DXLT to record detailed write tracing information */ \
+    if (getenv("ENABLE_DXLT_IO_TRACE")) { \
+        dxlt_posix_write(__fd, this_offset, __ret, __tm1, __tm2); \
+    } \
     if(this_offset > rec_ref->last_byte_written) \
         rec_ref->file_rec->counters[POSIX_SEQ_WRITES] += 1; \
     if(this_offset == (rec_ref->last_byte_written + 1)) \

darshan-runtime/lib/darshan-stdio.c

@@ -271,7 +271,7 @@ FILE* DARSHAN_DECL(fopen64)(const char *path, const char *mode)
     FILE* ret;
     double tm1, tm2;
 
-    MAP_OR_FAIL(fopen);
+    MAP_OR_FAIL(fopen64);
 
     tm1 = darshan_core_wtime();
     ret = __real_fopen64(path, mode);

darshan-util/Makefile.in

@@ -13,10 +13,10 @@ libdir = $(DESTDIR)@libdir@
 pkgconfigdir = $(DESTDIR)$(libdir)/pkgconfig
 
 DARSHAN_LOG_FORMAT = $(srcdir)/../darshan-log-format.h
-DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h $(srcdir)/../darshan-hdf5-log-format.h $(srcdir)/../darshan-pnetcdf-log-format.h $(srcdir)/../darshan-bgq-log-format.h $(srcdir)/../darshan-lustre-log-format.h $(srcdir)/../darshan-stdio-log-format.h
-DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h darshan-bgq-logutils.h darshan-lustre-logutils.h darshan-stdio-logutils.h
-DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o darshan-lustre-logutils.o darshan-stdio-logutils.o
-DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po darshan-lustre-logutils.po darshan-stdio-logutils.po
+DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h $(srcdir)/../darshan-hdf5-log-format.h $(srcdir)/../darshan-pnetcdf-log-format.h $(srcdir)/../darshan-bgq-log-format.h $(srcdir)/../darshan-lustre-log-format.h $(srcdir)/../darshan-stdio-log-format.h $(srcdir)/../darshan-dxlt-log-format.h
+DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h darshan-bgq-logutils.h darshan-lustre-logutils.h darshan-stdio-logutils.h darshan-dxlt-logutils.h
+DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o darshan-lustre-logutils.o darshan-stdio-logutils.o darshan-dxlt-logutils.o
+DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po darshan-lustre-logutils.po darshan-stdio-logutils.po darshan-dxlt-logutils.po
 
 DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
 
@@ -91,6 +91,11 @@ darshan-stdio-logutils.o: darshan-stdio-logutils.c darshan-logutils.h darshan-st
 darshan-stdio-logutils.po: darshan-stdio-logutils.c darshan-logutils.h darshan-stdio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-stdio-log-format.h | uthash-1.9.2
 	$(CC) $(CFLAGS_SHARED) -c  $< -o $@
 
+darshan-dxlt-logutils.o: darshan-dxlt-logutils.c darshan-logutils.h darshan-dxlt-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxlt-log-format.h | uthash-1.9.2
+	$(CC) $(CFLAGS) -c  $< -o $@
+darshan-dxlt-logutils.po: darshan-dxlt-logutils.c darshan-logutils.h darshan-dxlt-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxlt-log-format.h | uthash-1.9.2
+	$(CC) $(CFLAGS_SHARED) -c  $< -o $@
+
 libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS)
 	ar rcs libdarshan-util.a $^
 
@@ -147,6 +152,7 @@ endif
 	install -m 644 $(srcdir)/darshan-bgq-logutils.h $(includedir)
 	install -m 644 $(srcdir)/darshan-null-logutils.h $(includedir)
 	install -m 644 $(srcdir)/darshan-stdio-logutils.h $(includedir)
+	install -m 644 $(srcdir)/darshan-dxlt-logutils.h $(includedir)
 	install -m 644 $(srcdir)/../darshan-null-log-format.h $(includedir)
 	install -m 644 $(srcdir)/../darshan-posix-log-format.h $(includedir)
 	install -m 644 $(srcdir)/../darshan-mpiio-log-format.h $(includedir)
@@ -154,6 +160,7 @@ endif
 	install -m 644 $(srcdir)/../darshan-pnetcdf-log-format.h $(includedir)
 	install -m 644 $(srcdir)/../darshan-bgq-log-format.h $(includedir)
 	install -m 644 $(srcdir)/../darshan-stdio-log-format.h $(includedir)
+	install -m 644 $(srcdir)/../darshan-dxlt-log-format.h $(includedir)
 	install -d $(includedir)/uthash-1.9.2
 	install -d $(includedir)/uthash-1.9.2/src
 	install -m 644 uthash-1.9.2/src/uthash.h $(includedir)/uthash-1.9.2/src/

darshan-util/darshan-dxlt-logutils.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * Copyright (C) 2016 Intel Corporation.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_DXLT_LOG_UTILS_H
+#define __DARSHAN_DXLT_LOG_UTILS_H
+
+extern struct darshan_mod_logutil_funcs dxlt_posix_logutils;
+extern struct darshan_mod_logutil_funcs dxlt_mpiio_logutils;
+
+#endif

darshan-util/darshan-logutils.h

@@ -128,6 +128,9 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[];
 #include "darshan-lustre-logutils.h"
 #include "darshan-stdio-logutils.h"
 
+/* DXLT */
+#include "darshan-dxlt-logutils.h"
+
 darshan_fd darshan_log_open(const char *name);
 darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
     int partial_flag);

darshan-util/darshan-parser.c

@@ -121,6 +121,9 @@ void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver);
 
 void calc_perf(perf_data_t *pdata, int64_t nprocs);
 
+/* DXLT */
+extern void dxlt_logutils_cleanup();
+
 int usage (char *exename)
 {
     fprintf(stderr, "Usage: %s [options] <filename>\n", exename);
@@ -689,6 +692,9 @@ int main(int argc, char **argv)
             if(curr->rec_dat) free(curr->rec_dat);
             free(curr);
         }
+
+        /* DXLT */
+        dxlt_logutils_cleanup();
     }
     if(empty_mods == DARSHAN_MAX_MODS)
         printf("\n# no module data available.\n");