initial runtime lustre module implementation

the POSIX module will call into the Lustre module for each file
record created that corresponds to a file from a Lustre FS. No
counters are currently stored but most of the module strucutre
has been stubbed.

darshan-runtime/darshan.h

@@ -80,6 +80,13 @@ struct darshan_module_funcs
     void (*shutdown)(void);
 };
 
+/* stores FS info from statfs calls for a given mount point */
+struct darshan_fs_info
+{
+    int fs_type;
+    int block_size;
+};
+
 /* paths that darshan will not trace */
 extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
 
@@ -134,7 +141,7 @@ void darshan_core_register_record(
     int printable_flag,
     int mod_limit_flag,
     darshan_record_id *rec_id,
-    int *file_alignment);
+    struct darshan_fs_info *fs_info);
 
 /* darshan_core_unregister_record()
  *

darshan-runtime/lib/darshan-core.c

@@ -79,9 +79,9 @@ void (*mod_static_init_fns[])(void) =
 #define DARSHAN_MAX_MNT_TYPE 32
 struct mnt_data
 {
-    int block_size;
     char path[DARSHAN_MAX_MNT_PATH];
     char type[DARSHAN_MAX_MNT_TYPE];
+    struct darshan_fs_info fs_info;
 };
 static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
 static int mnt_data_count = 0;
@@ -96,8 +96,8 @@ static void darshan_get_exe_and_mounts_root(
     int space_left);
 static char* darshan_get_exe_and_mounts(
     struct darshan_core_runtime *core);
-static void darshan_block_size_from_path(
-    const char *path, int *block_size);
+static void darshan_fs_info_from_path(
+    const char *path, struct darshan_fs_info *fs_info);
 static void darshan_get_shared_records(
     struct darshan_core_runtime *core, darshan_record_id **shared_recs,
     int *shared_rec_cnt);
@@ -952,12 +952,13 @@ static void add_entry(char* trailing_data, int* space_left, struct mntent *entry
 #define LL_SUPER_MAGIC 0x0BD00BD0
 #endif
     ret = statfs(entry->mnt_dir, &statfsbuf);
+    mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
     if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
-        mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
+        mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
     else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
-        mnt_data_array[mnt_data_count].block_size = 1024*1024;
+        mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
     else
-        mnt_data_array[mnt_data_count].block_size = 4096;
+        mnt_data_array[mnt_data_count].fs_info.block_size = 4096;
 
     /* store mount information for use in header of darshan log */
     ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
@@ -1097,16 +1098,17 @@ static char* darshan_get_exe_and_mounts(struct darshan_core_runtime *core)
     return(trailing_data);
 }
 
-static void darshan_block_size_from_path(const char *path, int *block_size)
+static void darshan_fs_info_from_path(const char *path, struct darshan_fs_info *fs_info)
 {
     int i;
-    *block_size = -1;
+    fs_info->fs_type = -1;
+    fs_info->block_size = -1;
 
     for(i=0; i<mnt_data_count; i++)
     {
         if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
         {
-            *block_size = mnt_data_array[i].block_size;
+            *fs_info = mnt_data_array[i].fs_info;
             return;
         }
     }
@@ -1695,7 +1697,7 @@ void darshan_core_register_record(
     int printable_flag,
     int mod_limit_flag,
     darshan_record_id *rec_id,
-    int *file_alignment)
+    struct darshan_fs_info *fs_info)
 {
     darshan_record_id tmp_rec_id;
     struct darshan_core_record_ref *ref;
@@ -1743,14 +1745,10 @@ void darshan_core_register_record(
     DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
     DARSHAN_CORE_UNLOCK();
 
-    /* TODO: call into lustre if functionality enabled and if lustre data has not
-     * already been captured for this file
-     */
-
-    if(file_alignment)
-        darshan_block_size_from_path(name, file_alignment);
-
     *rec_id = tmp_rec_id;
+    if(fs_info)
+        darshan_fs_info_from_path(name, fs_info);
+
     return;
 }
 

darshan-runtime/lib/darshan-lustre.c

@@ -22,15 +22,27 @@
 #include "darshan.h"
 #include "darshan-dynamic.h"
 
-/* Lustre record data is currently immutable, so once it's set it should be
- * final. Since we won't likely need to search for records after they've been
- * created, a simple array is fine for storing them instead of a hash table
+/* TODO: once mmap merge is complete, we can just use an array
+ * to store record data rather than a hash table -- in that
+ * branch, register_record() returns whether the record 
+ * already exists, at which point we won't need to instrument
+ * more data, since the Lustre record data is immutable. records
+ * could just be appended to the array if there is no need to
+ * search for a specific record.
  */
+struct lustre_record_ref
+{
+    struct darshan_lustre_record *record;
+    UT_hash_handle hlink;
+};
+
 struct lustre_runtime
 {
+    struct lustre_record_ref *ref_array;
     struct darshan_lustre_record *record_array;
     int record_array_size;
     int record_array_ndx;
+    struct lustre_record_ref *record_hash;
 };
 
 static struct lustre_runtime *lustre_runtime = NULL;
@@ -48,13 +60,58 @@ static void lustre_shutdown(void);
 #define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
 #define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
 
+/* TODO: is there any way we can further compact Lustre data to save space?
+ * e.g., are all files in the same directory guaranteed same striping parameters?
+ * if so, can we store stripe parameters on per-directory basis and the OST
+ * list on a per-file basis? maybe the storage savings are small enough this isn't
+ * worth it, but nice to keep in mind
+ */
+
 void darshan_instrument_lustre_file(char *filepath)
 {
+    struct lustre_record_ref *lustre_ref;
+    darshan_record_id rec_id;
+    int limit_flag;
+
+    LUSTRE_LOCK();
     /* make sure the lustre module is already initialized */
     lustre_runtime_initialize();
 
-    /* TODO: implement gathering of lustre data */
+    limit_flag = (lustre_runtime->record_array_ndx >= lustre_runtime->record_array_size);
 
+    /* register a Lustre file record with Darshan */
+    darshan_core_register_record(
+        (void *)filepath,
+        strlen(filepath),
+        DARSHAN_LUSTRE_MOD,
+        1,
+        limit_flag,
+        &rec_id,
+        NULL);
+
+    /* if record id is 0, darshan has no more memory for instrumenting */
+    if(rec_id == 0)
+        return;
+
+    HASH_FIND(hlink, lustre_runtime->record_hash, &rec_id,
+        sizeof(darshan_record_id), lustre_ref);
+    if(!lustre_ref)
+    {
+        /* no existing record, allocate a new one and add it to the hash */
+        lustre_ref = &(lustre_runtime->ref_array[lustre_runtime->record_array_ndx]);
+        lustre_ref->record = &(lustre_runtime->record_array[lustre_runtime->record_array_ndx]);
+        lustre_ref->record->rec_id = rec_id;
+        lustre_ref->record->rank = my_rank;
+
+        /* TODO: gather lustre data, store in record hash */
+        /* counters in lustre_ref->record->counters */
+
+        HASH_ADD(hlink, lustre_runtime->record_hash, record->rec_id,
+            sizeof(darshan_record_id), lustre_ref);
+        lustre_runtime->record_array_ndx++;
+    }
+
+    LUSTRE_UNLOCK();
     return;
 }
 
@@ -89,6 +146,24 @@ static void lustre_runtime_initialize()
         return;
     memset(lustre_runtime, 0, sizeof(*lustre_runtime));
 
+    /* allocate array of Lustre records according to the amount of memory
+     * assigned by Darshan
+     */
+    lustre_runtime->record_array_size = mem_limit / sizeof(struct darshan_lustre_record);
+
+    lustre_runtime->ref_array = malloc(lustre_runtime->record_array_size *
+                                       sizeof(struct lustre_record_ref));
+    lustre_runtime->record_array = malloc(lustre_runtime->record_array_size *
+                                          sizeof(struct darshan_lustre_record));
+    if(!lustre_runtime->ref_array || !lustre_runtime->record_array)
+    {
+        lustre_runtime->record_array_size = 0;
+        return;
+    }
+    memset(lustre_runtime->ref_array, 0, lustre_runtime->record_array_size *
+        sizeof(struct lustre_record_ref));
+    memset(lustre_runtime->record_array, 0, lustre_runtime->record_array_size *
+        sizeof(struct darshan_lustre_record));
 
     return;
 }

darshan-runtime/lib/darshan-posix.c

@@ -176,6 +176,11 @@ struct posix_runtime
     struct posix_file_runtime_ref* fd_hash;
 };
 
+/* XXX modules don't expose an API for other modules, so use extern to get
+ * Lustre instrumentation function
+ */
+extern void darshan_instrument_lustre_file(char *filepath);
+
 static struct posix_runtime *posix_runtime = NULL;
 static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
 static int instrumentation_disabled = 0;
@@ -1505,7 +1510,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
     struct posix_file_runtime *file = NULL;
     char *newname = NULL;
     darshan_record_id file_id;
-    int file_alignment;
+    struct darshan_fs_info fs_info;
     int limit_flag;
 
     if(!posix_runtime || instrumentation_disabled)
@@ -1525,7 +1530,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
         1,
         limit_flag,
         &file_id,
-        &file_alignment);
+        &fs_info);
 
     /* the file record id is set to 0 if no memory is available for tracking
      * new records -- just fall through and ignore this record
@@ -1552,12 +1557,19 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
     file->file_record->f_id = file_id;
     file->file_record->rank = my_rank;
     file->file_record->counters[POSIX_MEM_ALIGNMENT] = darshan_mem_alignment;
-    file->file_record->counters[POSIX_FILE_ALIGNMENT] = file_alignment;
+    file->file_record->counters[POSIX_FILE_ALIGNMENT] = fs_info.block_size;
 
     /* add new record to file hash table */
     HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
     posix_runtime->file_array_ndx++;
 
+    /* XXX: check for lustre and call in */
+#ifndef LL_SUPER_MAGIC
+#define LL_SUPER_MAGIC 0x0BD00BD0
+#endif
+    if(fs_info.fs_type == LL_SUPER_MAGIC)
+        darshan_instrument_lustre_file(newname);
+
     if(newname != name)
         free(newname);
     return(file);