Merge branch 'lustre-mod'

Conflicts:
	darshan-bgq-log-format.h
	darshan-runtime/darshan.h
	darshan-runtime/lib/darshan-bgq.c
	darshan-runtime/lib/darshan-core.c
	darshan-runtime/lib/darshan-posix.c
	darshan-util/darshan-convert.c
	darshan-util/darshan-diff.c

darshan-log-format.h

@@ -111,6 +111,7 @@ struct darshan_base_record
 #include "darshan-hdf5-log-format.h"
 #include "darshan-pnetcdf-log-format.h"
 #include "darshan-bgq-log-format.h"
+#include "darshan-lustre-log-format.h"
 
 /* X-macro for keeping module ordering consistent */
 /* NOTE: first val used to define module enum values, 
@@ -128,7 +129,8 @@ struct darshan_base_record
     X(DARSHAN_MPIIO_MOD,    "MPI-IO",   DARSHAN_MPIIO_VER,      &mpiio_logutils) \
     X(DARSHAN_HDF5_MOD,     "HDF5",     DARSHAN_HDF5_VER,       &hdf5_logutils) \
     X(DARSHAN_PNETCDF_MOD,  "PNETCDF",  DARSHAN_PNETCDF_VER,    &pnetcdf_logutils) \
-    X(DARSHAN_BGQ_MOD,      "BG/Q",     DARSHAN_BGQ_VER,        &bgq_logutils)
+    X(DARSHAN_BGQ_MOD,      "BG/Q",     DARSHAN_BGQ_VER,        &bgq_logutils) \
+    X(DARSHAN_LUSTRE_MOD,   "LUSTRE",   DARSHAN_LUSTRE_VER,     &lustre_logutils)
 
 /* unique identifiers to distinguish between available darshan modules */
 /* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]

darshan-lustre-log-format.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_LUSTRE_LOG_FORMAT_H
+#define __DARSHAN_LUSTRE_LOG_FORMAT_H
+
+/* NOTE -- redefining the size of OST_ID will require changing the DARSHAN_BSWAP
+ * macro used in darshan-util/darshan-lustre-logutils.c as well
+ */
+typedef int64_t OST_ID;
+
+/* current Lustre log format version */
+#define DARSHAN_LUSTRE_VER 1
+
+#define LUSTRE_COUNTERS \
+    /* number of OSTs for file system */\
+    X(LUSTRE_OSTS) \
+    /* number of MDTs for file system */\
+    X(LUSTRE_MDTS) \
+    /* index of first OST for file */\
+    X(LUSTRE_STRIPE_OFFSET) \
+    /* bytes per stripe for file */\
+    X(LUSTRE_STRIPE_SIZE) \
+    /* number of stripes (OSTs) for file */\
+    X(LUSTRE_STRIPE_WIDTH) \
+    /* end of counters */\
+    X(LUSTRE_NUM_INDICES)
+
+#define X(a) a,
+/* integer statistics for Lustre file records */
+enum darshan_lustre_indices
+{
+    LUSTRE_COUNTERS
+};
+#undef X
+
+/* record structure for the Lustre module. a record is created and stored for
+ * every file opened that belongs to a Lustre file system. This record includes:
+ *      - a corresponding record identifier (created by hashing the file path)
+ *      - the rank of the process which opened the file (-1 for shared files)
+ *      - integer file I/O statistics (stripe size, width, # of OSTs, etc.)
+ */
+struct darshan_lustre_record
+{
+    darshan_record_id rec_id;
+    int64_t rank;
+    int64_t counters[LUSTRE_NUM_INDICES];
+    OST_ID ost_ids[1];
+};
+
+/*
+ *  helper function to calculate the size of a record
+ */
+#define LUSTRE_RECORD_SIZE( osts ) ( sizeof(struct darshan_lustre_record) + sizeof(OST_ID) * (osts - 1) )
+
+#endif /* __DARSHAN_LUSTRE_LOG_FORMAT_H */

darshan-runtime/Makefile.in

@@ -37,6 +37,10 @@ LIBS = -lz @LIBBZ2@
 DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o
 DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po
 
+# TODO: make the lustre module enabled using config options
+DARSHAN_STATIC_MOD_OBJS += lib/darshan-lustre.o
+DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-lustre.po
+
 ifdef DARSHAN_USE_BGQ
 DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o
 DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-bgq.po
@@ -50,63 +54,69 @@ lib::
 lib/darshan-core-init-finalize.o: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
 lib/darshan-core.o: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-common.o: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-common.o: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-common.po: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-common.po: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-null.o: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
+lib/darshan-null.o: lib/darshan-null.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-null.po: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
+lib/darshan-null.po: lib/darshan-null.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-posix.o: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
+lib/darshan-posix.o: lib/darshan-posix.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-posix.po: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
+lib/darshan-posix.po: lib/darshan-posix.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
-lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
 lib/darshan-hdf5-stubs.o: lib/darshan-hdf5-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
+lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
-lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
+lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
 lib/darshan-pnetcdf-stubs.o: lib/darshan-pnetcdf-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
+lib/darshan-lustre.o: lib/darshan-lustre.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-lustre.po: lib/darshan-lustre.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+
 lib/lookup3.o: lib/lookup3.c
 	$(CC) $(CFLAGS) -c $< -o $@
 

darshan-runtime/darshan-lustre.h

+struct lustre_record_runtime
+{
+    struct darshan_lustre_record *record;
+    size_t record_size;
+    UT_hash_handle hlink;
+};
+
+struct lustre_runtime
+{
+    int    record_count;       /* number of records defined */
+    size_t record_buffer_max;  /* size of the allocated buffer pointed to by record_buffer */
+    size_t record_buffer_used; /* size of the allocated buffer actually used */
+    void   *next_free_record;  /* pointer to end of record_buffer */
+    void   *record_buffer;     /* buffer in which records are created */
+    struct lustre_record_runtime *record_runtime_array;
+    struct lustre_record_runtime *record_runtime_hash;
+};
+
+

darshan-runtime/darshan.h

@@ -73,6 +73,15 @@ typedef void (*darshan_module_shutdown)(
     int *mod_buf_sz /* output parameter to save module buffer size */
 );
 
+/* stores FS info from statfs calls for a given mount point */
+struct darshan_fs_info
+{
+    int fs_type;
+    int block_size;
+    int ost_count;
+    int mdt_count;
+};
+
 /*****************************************************
 * darshan-core functions exported to darshan modules *
 *****************************************************/
@@ -122,17 +131,18 @@ darshan_record_id darshan_core_gen_record_id(
  * Darshan record (e.g., the full file path), which for now is just a
  * string. 'mod_id' is the identifier of the calling module. 'rec_len'
  * is the size of the record being registered with Darshan. If given,
- * 'file_alignment' is a pointer to an integer which on return will
- * contain the corresponding file system alignment of the file system
- * path 'name' resides on. Returns a pointer to the address the record
- * should be written to on success, NULL on failure.
+ * 'fs_info' is a pointer to a structure containing information on
+ * the underlying FS this record is associated with (determined by
+ * matching the file name prefix with Darshan's list of tracked mount
+ * points). Returns a pointer to the address the record should be
+ * written to on success, NULL on failure.
  */
 void *darshan_core_register_record(
     darshan_record_id rec_id,
     const char *name,
     darshan_module_id mod_id,
     int rec_len,
-    int *file_alignment);
+    struct darshan_fs_info *fs_info);
 
 /* darshan_core_wtime()
  *

darshan-runtime/lib/darshan-core-init-finalize.c

@@ -15,6 +15,7 @@
 
 #include "darshan.h"
 #include "darshan-core.h"
+#include "darshan-dynamic.h"
 
 #ifdef DARSHAN_PRELOAD
 

darshan-runtime/lib/darshan-core.c

@@ -19,6 +19,8 @@
 #include <limits.h>
 #include <pthread.h>
 #include <fcntl.h>
+#include <dirent.h>
+#include <sys/ioctl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
@@ -32,6 +34,9 @@
 #include "darshan-core.h"
 #include "darshan-dynamic.h"
 
+/* XXX stick this into autoconf .h */
+#include <lustre/lustre_user.h>
+
 extern char* __progname;
 extern char* __progname_full;
 
@@ -83,9 +88,9 @@ void (*mod_static_init_fns[])(void) =
 #define DARSHAN_MAX_MNT_TYPE 32
 struct mnt_data
 {
-    int block_size;
     char path[DARSHAN_MAX_MNT_PATH];
     char type[DARSHAN_MAX_MNT_TYPE];
+    struct darshan_fs_info fs_info;
 };
 static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
 static int mnt_data_count = 0;
@@ -99,11 +104,11 @@ static void darshan_log_record_hints_and_ver(
     struct darshan_core_runtime* core);
 static void darshan_get_exe_and_mounts(
     struct darshan_core_runtime *core, int argc, char **argv);
+static void darshan_fs_info_from_path(
+    const char *path, struct darshan_fs_info *fs_info);
 static int darshan_add_name_record_ref(
     struct darshan_core_runtime *core, darshan_record_id rec_id,
     const char *name, darshan_module_id mod_id);
-static int darshan_block_size_from_path(
-    const char *path);
 static void darshan_get_user_name(
     char *user);
 static void darshan_get_logfile_name(
@@ -919,12 +924,42 @@ static void add_entry(char* buf, int* space_left, struct mntent* entry)
 #define LL_SUPER_MAGIC 0x0BD00BD0
 #endif
     ret = statfs(entry->mnt_dir, &statfsbuf);
+    mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
     if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
-        mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
+        mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
     else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
-        mnt_data_array[mnt_data_count].block_size = 1024*1024;
+        mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
     else
-        mnt_data_array[mnt_data_count].block_size = 4096;
+        mnt_data_array[mnt_data_count].fs_info.block_size = 4096;
+
+    /* XXX */
+    /* attempt to retrieve OST and MDS counts from Lustre */
+    mnt_data_array[mnt_data_count].fs_info.ost_count = -1;
+    mnt_data_array[mnt_data_count].fs_info.mdt_count = -1;
+    if ( statfsbuf.f_type == LL_SUPER_MAGIC )
+    {
+        int n_ost, n_mdt;
+        int ret_ost, ret_mdt;
+        DIR *mount_dir;
+
+        mount_dir = opendir( entry->mnt_dir );
+        if ( mount_dir  ) 
+        {
+            /* n_ost and n_mdt are used for both input and output to ioctl */
+            n_ost = 0;
+            n_mdt = 1;
+
+            ret_ost = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_ost );
+            ret_mdt = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_mdt );
+
+            if ( !(ret_ost < 0 || ret_mdt < 0) )
+            {
+                mnt_data_array[mnt_data_count].fs_info.ost_count = n_ost;
+                mnt_data_array[mnt_data_count].fs_info.mdt_count = n_mdt;
+            }
+            closedir( mount_dir );
+        }
+    }
 
     /* store mount information with the job-level metadata in darshan log */
     ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
@@ -1059,21 +1094,55 @@ static void darshan_get_exe_and_mounts(struct darshan_core_runtime *core,
     return;
 }
 
-static int darshan_block_size_from_path(const char *path)
+static void darshan_fs_info_from_path(const char *path, struct darshan_fs_info *fs_info)
 {
     int i;
-    int block_size = -1;
+    fs_info->fs_type = -1;
+    fs_info->block_size = -1;
 
     for(i=0; i<mnt_data_count; i++)
     {
         if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
         {
-            block_size = mnt_data_array[i].block_size;
-            break;;
+            *fs_info = mnt_data_array[i].fs_info;
+            return;
         }
     }
 
-    return block_size;
+    return;
+}
+
+static int darshan_add_name_record_ref(struct darshan_core_runtime *core,
+    darshan_record_id rec_id, const char *name, darshan_module_id mod_id)
+{
+    struct darshan_core_name_record_ref *ref;
+    int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
+
+    if((record_size + core->name_mem_used) > DARSHAN_NAME_RECORD_BUF_SIZE)
+        return(0);
+
+    ref = malloc(sizeof(*ref));
+    if(!ref)
+        return(0);
+    memset(ref, 0, sizeof(*ref));
+
+    /* initialize the name record */
+    ref->name_record = (struct darshan_name_record *)
+        ((char *)core->log_name_p + core->name_mem_used);
+    memset(ref->name_record, 0, record_size);
+    ref->name_record->id = rec_id;
+    strcpy(ref->name_record->name, name);
+    DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
+
+    /* add the record to the hash table */
+    HASH_ADD(hlink, core->name_hash, name_record->id,
+        sizeof(darshan_record_id), ref);
+    core->name_mem_used += record_size;
+#ifdef __DARSHAN_ENABLE_MMAP_LOGS
+    core->log_hdr_p->name_map.len += record_size;
+#endif
+
+    return(1);
 }
 
 static void darshan_get_user_name(char *cuser)
@@ -1239,39 +1308,6 @@ static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* s
     return;
 }
 
-static int darshan_add_name_record_ref(struct darshan_core_runtime *core,
-    darshan_record_id rec_id, const char *name, darshan_module_id mod_id)
-{
-    struct darshan_core_name_record_ref *ref;
-    int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
-
-    if((record_size + core->name_mem_used) > DARSHAN_NAME_RECORD_BUF_SIZE)
-        return(0);
-
-    ref = malloc(sizeof(*ref));
-    if(!ref)
-        return(0);
-    memset(ref, 0, sizeof(*ref));
-
-    /* initialize the name record */
-    ref->name_record = (struct darshan_name_record *)
-        ((char *)core->log_name_p + core->name_mem_used);
-    memset(ref->name_record, 0, record_size);
-    ref->name_record->id = rec_id;
-    strcpy(ref->name_record->name, name);
-    DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
-
-    /* add the record to the hash table */
-    HASH_ADD(hlink, core->name_hash, name_record->id,
-        sizeof(darshan_record_id), ref);
-    core->name_mem_used += record_size;
-#ifdef __DARSHAN_ENABLE_MMAP_LOGS
-    core->log_hdr_p->name_map.len += record_size;
-#endif
-
-    return(1);
-}
-
 static void darshan_get_shared_records(struct darshan_core_runtime *core,
     darshan_record_id **shared_recs, int *shared_rec_cnt)
 {
@@ -1889,7 +1925,7 @@ void *darshan_core_register_record(
     const char *name,
     darshan_module_id mod_id,
     int rec_len,
-    int *file_alignment)
+    struct darshan_fs_info *fs_info)
 {
     struct darshan_core_name_record_ref *ref;
     void *rec_buf;
@@ -1942,8 +1978,8 @@ void *darshan_core_register_record(
 #endif
     DARSHAN_CORE_UNLOCK();
 
-    if(file_alignment)
-        *file_alignment = darshan_block_size_from_path(name);
+    if(fs_info)
+        darshan_fs_info_from_path(name, fs_info);
 
     return(rec_buf);;
 }

darshan-runtime/lib/darshan-hdf5.c

@@ -4,6 +4,9 @@
  *
  */
 
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
 #include "darshan-runtime-config.h"
 #include <stdio.h>
 #include <unistd.h>
@@ -16,7 +19,6 @@
 #include <errno.h>
 #include <search.h>
 #include <assert.h>
-#define __USE_GNU
 #include <pthread.h>
 
 #include "darshan.h"

darshan-runtime/lib/darshan-lustre.c

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
+#include "darshan-runtime-config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <pthread.h>
+#include <sys/ioctl.h>
+
+/* XXX stick this into autoconf .h */
+#include <lustre/lustre_user.h>
+
+#include "uthash.h"
+
+#include "darshan.h"
+#include "darshan-dynamic.h"
+#include "darshan-lustre.h"
+
+struct lustre_runtime *lustre_runtime = NULL;
+static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+static int instrumentation_disabled = 0;
+static int my_rank = -1;
+
+static void lustre_runtime_initialize(void);
+
+static void lustre_begin_shutdown(void);
+static void lustre_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
+    int shared_rec_count, void **lustre_buf, int *lustre_buf_sz);
+static void lustre_shutdown(void);
+static int lustre_record_compare(const void* a_p, const void* b_p);
+static void lustre_record_reduction_op(void* infile_v, void* inoutfile_v,
+    int *len, MPI_Datatype *datatype);
+
+#define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
+#define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
+
+void darshan_instrument_lustre_file(const char* filepath, int fd)
+{
+    struct lustre_record_runtime *rec_rt;
+    struct darshan_lustre_record *rec;
+    struct darshan_fs_info fs_info;
+    darshan_record_id rec_id;
+    int limit_flag;
+    int i;
+    struct lov_user_md *lum;
+    size_t lumsize = sizeof(struct lov_user_md) +
+        LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data);
+    size_t rec_size;
+    char *newname = NULL;
+
+    LUSTRE_LOCK();
+    if(instrumentation_disabled)
+    {
+        LUSTRE_UNLOCK();
+        return;
+    }
+
+    /* make sure the lustre module is already initialized */
+    lustre_runtime_initialize();
+    if(!lustre_runtime)
+    {
+        LUSTRE_UNLOCK();
+        return;
+    }
+
+    /* if we can't issue ioctl, we have no counter data at all */
+    if ( (lum = calloc(1, lumsize)) == NULL )
+    {
+        LUSTRE_UNLOCK();
+        return;
+    }
+
+    /* find out the OST count of this file so we can allocate memory */
+    lum->lmm_magic = LOV_USER_MAGIC;
+    lum->lmm_stripe_count = LOV_MAX_STRIPE_COUNT;
+
+    /* -1 means ioctl failed, likely because file isn't on Lustre */
+    if ( ioctl( fd, LL_IOC_LOV_GETSTRIPE, (void *)lum ) == -1 )
+    {
+        free(lum);
+        LUSTRE_UNLOCK();
+        return;
+    }
+
+    rec_size = LUSTRE_RECORD_SIZE( lum->lmm_stripe_count );
+
+    /* get fully qualified name for record */
+    newname = darshan_clean_file_path(filepath);
+    if(!newname)
+        newname = (char*)filepath;
+
+    {
+        /* broken out for clarity */
+        void *end_of_new_record = (char*)lustre_runtime->next_free_record + rec_size;
+        void *end_of_rec_buffer = (char*)lustre_runtime->record_buffer + lustre_runtime->record_buffer_max;
+        limit_flag = ( end_of_new_record > end_of_rec_buffer );
+    }
+
+    /* register a Lustre file record with Darshan */
+    fs_info.fs_type = -1;
+    darshan_core_register_record(
+        (void *)newname,
+        strlen(newname),
+        DARSHAN_LUSTRE_MOD,
+        1,
+        limit_flag,
+        &rec_id,
+        &fs_info);
+
+    /* if record id is 0, darshan has no more memory for instrumenting */
+    if(rec_id == 0)
+    {
+        free(lum);
+        LUSTRE_UNLOCK();
+        return;
+    }
+
+    /* search the hash table for this file record, and initialize if not found */
+    HASH_FIND(hlink, lustre_runtime->record_runtime_hash, &rec_id, sizeof(darshan_record_id), rec_rt );
+    if ( !rec_rt ) {
+        /* allocate a new lustre record and append it to the array */
+        rec_rt = &(lustre_runtime->record_runtime_array[lustre_runtime->record_count]);
+        rec_rt->record = lustre_runtime->next_free_record;
+        rec_rt->record_size = rec_size;
+        lustre_runtime->next_free_record = (char*)(lustre_runtime->next_free_record) + rec_size;
+        lustre_runtime->record_buffer_used += rec_size;
+        rec = rec_rt->record;
+        rec->rec_id = rec_id;
+        rec->rank = my_rank;
+
+        /* implicit assumption here that none of these counters will change
+         * after the first time a file is opened.  This may not always be
+         * true in the future */
+        if ( fs_info.fs_type != -1 ) 
+        {