first cut at mmap logic -- posix mod only for now

darshan-runtime/darshan-core.h

@@ -26,11 +26,15 @@
 /* Environment variable to override __DARSHAN_MEM_ALIGNMENT */
 #define DARSHAN_MEM_ALIGNMENT_OVERRIDE "DARSHAN_MEMALIGN"
 
+/* maximum number of records that can be tracked on a single process */
 #define DARSHAN_CORE_MAX_RECORDS 2048
 
 /* TODO: revisit this default size if we change memory per module */
 #define DARSHAN_CORE_COMP_BUF_SIZE (2 * 1024 * 1024)
 
+/* this controls the maximum mmapped memory each module can use */
+#define DARSHAN_MMAP_CHUNK_SIZE (4 * 1024)
+
 #define DARSHAN_CORE_MOD_SET(flags, id) (flags | (1 << id))
 #define DARSHAN_CORE_MOD_UNSET(flags, id) (flags & ~(1 << id))
 #define DARSHAN_CORE_MOD_ISSET(flags, id) (flags & (1 << id))
@@ -38,15 +42,18 @@
 /* in memory structure to keep up with job level data */
 struct darshan_core_runtime
 {
-    struct darshan_header log_header;
-    struct darshan_job log_job;
-    char exe[DARSHAN_EXE_LEN+1];
+    /* XXX-MMAP */
+    void *mmap_p;
+    struct darshan_job *mmap_job_p;
+    char *mmap_exe_mnt_p;
+    void *mmap_mod_p;
+    /* XXX-MMAP */
+
     struct darshan_core_record_ref *rec_hash;
     int rec_count;
     struct darshan_core_module* mod_array[DARSHAN_MAX_MODS];
     char comp_buf[DARSHAN_CORE_COMP_BUF_SIZE];
     double wtime_offset;
-    char *trailing_data;
 };
 
 struct darshan_core_module

darshan-runtime/darshan.h

@@ -80,9 +80,6 @@ struct darshan_module_funcs
     void (*shutdown)(void);
 };
 
-/* paths that darshan will not trace */
-extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
-
 /*****************************************************
 * darshan-core functions exported to darshan modules *
 *****************************************************/
@@ -103,6 +100,8 @@ void darshan_core_register_module(
     struct darshan_module_funcs *funcs,
     int *my_rank,
     int *mod_mem_limit,
+    void **mmap_buf,
+    int *mmap_buf_size,
     int *sys_mem_alignment);
 
 /* darshan_core_unregister_module()
@@ -151,4 +150,12 @@ void darshan_core_unregister_record(
  */
 double darshan_core_wtime(void);
 
+/* darshan_core_excluded_path()
+ *
+ * Returns true (1) if the given file path is in Darshan's list of
+ * excluded file paths, false (0) otherwise.
+ */
+int darshan_core_excluded_path(
+    const char * path);
+
 #endif /* __DARSHAN_H */

darshan-runtime/lib/darshan-bgq.c

@@ -39,6 +39,9 @@
 struct bgq_runtime
 {
     struct darshan_bgq_record record;
+
+    /* TODO: we don't need the mmap and regular buffer, both */
+    struct darshan_bgq_record *mmap_buf;
 };
 
 static struct bgq_runtime *bgq_runtime = NULL;
@@ -114,6 +117,8 @@ void bgq_runtime_initialize()
         .shutdown = bgq_shutdown
     };
     int mem_limit;
+    void *mmap_buf;
+    int mmap_buf_size;
     char *recname = "darshan-internal-bgq";
 
     BGQ_LOCK();
@@ -128,6 +133,8 @@ void bgq_runtime_initialize()
         &bgq_mod_fns,
         &my_rank,
         &mem_limit,
+        &mmap_buf,
+        &mmap_buf_size,
         &darshan_mem_alignment);
 
     /* return if no memory assigned by darshan-core */

darshan-runtime/lib/darshan-hdf5.c

@@ -59,6 +59,8 @@ struct hdf5_runtime
     int file_array_ndx;
     struct hdf5_file_runtime *file_hash;
     struct hdf5_file_runtime_ref* hid_hash;
+
+    struct darshan_hdf5_file *total_file;
 };
 
 static struct hdf5_runtime *hdf5_runtime = NULL;
@@ -197,13 +199,15 @@ herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
 /* initialize internal HDF5 module data strucutres and register with darshan-core */
 static void hdf5_runtime_initialize()
 {
-    int mem_limit;
     struct darshan_module_funcs hdf5_mod_fns =
     {
         .begin_shutdown = &hdf5_begin_shutdown,
         .get_output_data = &hdf5_get_output_data,
         .shutdown = &hdf5_shutdown
     };
+    void *mmap_buf;
+    int mmap_buf_size;
+    int mem_limit;
 
     /* don't do anything if already initialized or instrumenation is disabled */
     if(hdf5_runtime || instrumentation_disabled)
@@ -215,6 +219,8 @@ static void hdf5_runtime_initialize()
         &hdf5_mod_fns,
         &my_rank,
         &mem_limit,
+        &mmap_buf,
+        &mmap_buf_size,
         NULL);
 
     /* return if no memory assigned by darshan-core */
@@ -235,6 +241,7 @@ static void hdf5_runtime_initialize()
     /* allocate array of runtime file records */
     hdf5_runtime->file_runtime_array = malloc(hdf5_runtime->file_array_size *
                                               sizeof(struct hdf5_file_runtime));
+    /* XXX-MMAP */
     hdf5_runtime->file_record_array = malloc(hdf5_runtime->file_array_size *
                                              sizeof(struct darshan_hdf5_file));
     if(!hdf5_runtime->file_runtime_array || !hdf5_runtime->file_record_array)

darshan-runtime/lib/darshan-mpiio.c

@@ -103,6 +103,8 @@ struct mpiio_runtime
     int file_array_ndx;
     struct mpiio_file_runtime* file_hash;
     struct mpiio_file_runtime_ref* fh_hash;
+
+    struct darshan_mpiio_file* total_file;
 };
 
 static struct mpiio_runtime *mpiio_runtime = NULL;
@@ -820,13 +822,15 @@ int MPI_File_close(MPI_File *fh)
 /* initialize data structures and register with darshan-core component */
 static void mpiio_runtime_initialize()
 {
-    int mem_limit;
     struct darshan_module_funcs mpiio_mod_fns =
     {
         .begin_shutdown = &mpiio_begin_shutdown,
         .get_output_data = &mpiio_get_output_data,
         .shutdown = &mpiio_shutdown
     };
+    void *mmap_buf;
+    int mmap_buf_size;
+    int mem_limit;
 
     /* don't do anything if already initialized or instrumenation is disabled */
     if(mpiio_runtime || instrumentation_disabled)
@@ -838,6 +842,8 @@ static void mpiio_runtime_initialize()
         &mpiio_mod_fns,
         &my_rank,
         &mem_limit,
+        &mmap_buf,
+        &mmap_buf_size,
         NULL);
 
     /* return if no memory assigned by darshan core */

darshan-runtime/lib/darshan-null.c

@@ -96,6 +96,14 @@ struct null_runtime
      * currently maintained by the "NULL" module.
      */
     struct null_record_runtime* record_hash;
+
+    /* total_record is a pointer to a buffer for storing an aggregate record for
+     * all "NULL" module records. This buffer is pointed at the appropriate
+     * region of a mmapped partial log file managed by darshan-core -- this
+     * partial log file is used to get at least *some* i/o data out to a log
+     * file if Darshan does not shut down properly.
+     */
+    struct darshan_null_record *total_record;
 };
 
 /* null_runtime is the global data structure encapsulating "NULL" module state */
@@ -200,6 +208,7 @@ int DARSHAN_DECL(foo)(const char* name, int arg1, int arg2)
 /* Initialize internal POSIX module data structures and register with darshan-core. */
 static void null_runtime_initialize()
 {
+    int mmap_buf_size;
     /* struct of function pointers for interfacing with darshan-core */
     struct darshan_module_funcs null_mod_fns =
     {
@@ -219,6 +228,8 @@ static void null_runtime_initialize()
         &null_mod_fns,
         &my_rank,
         &mem_limit,
+        (void **)&null_runtime->total_record,
+        &mmap_buf_size,
         NULL);
 
     /* return if no memory assigned by darshan-core */

darshan-runtime/lib/darshan-pnetcdf.c

@@ -55,6 +55,8 @@ struct pnetcdf_runtime
     int file_array_ndx;
     struct pnetcdf_file_runtime *file_hash;
     struct pnetcdf_file_runtime_ref* ncid_hash;
+
+    struct darshan_pnetcdf_file *total_file;
 };
 
 static struct pnetcdf_runtime *pnetcdf_runtime = NULL;
@@ -209,13 +211,15 @@ int DARSHAN_DECL(ncmpi_close)(int ncid)
 /* initialize internal PNETCDF module data strucutres and register with darshan-core */
 static void pnetcdf_runtime_initialize()
 {
-    int mem_limit;
     struct darshan_module_funcs pnetcdf_mod_fns =
     {
         .begin_shutdown = &pnetcdf_begin_shutdown,
         .get_output_data = &pnetcdf_get_output_data,
         .shutdown = &pnetcdf_shutdown
     };
+    void *mmap_buf;
+    int mmap_buf_size;
+    int mem_limit;
 
     /* don't do anything if already initialized or instrumenation is disabled */
     if(pnetcdf_runtime || instrumentation_disabled)
@@ -227,6 +231,8 @@ static void pnetcdf_runtime_initialize()
         &pnetcdf_mod_fns,
         &my_rank,
         &mem_limit,
+        &mmap_buf,
+        &mmap_buf_size,
         NULL);
 
     /* return if no memory assigned by darshan-core */
@@ -247,6 +253,7 @@ static void pnetcdf_runtime_initialize()
     /* allocate array of runtime file records */
     pnetcdf_runtime->file_runtime_array = malloc(pnetcdf_runtime->file_array_size *
                                                  sizeof(struct pnetcdf_file_runtime));
+    /* XXX-MMAP */
     pnetcdf_runtime->file_record_array = malloc(pnetcdf_runtime->file_array_size *
                                                 sizeof(struct darshan_pnetcdf_file));
     if(!pnetcdf_runtime->file_runtime_array || !pnetcdf_runtime->file_record_array)

darshan-runtime/lib/darshan-posix.c

@@ -177,6 +177,8 @@ struct posix_runtime
     int file_array_ndx;
     struct posix_file_runtime* file_hash;
     struct posix_file_runtime_ref* fd_hash;
+
+    struct posix_file_runtime agg_file_runtime;
 };
 
 static struct posix_runtime *posix_runtime = NULL;
@@ -209,15 +211,8 @@ static void posix_shutdown(void);
 
 #define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
     struct posix_file_runtime* file; \
-    char* exclude; \
-    int tmp_index = 0; \
     if(__ret < 0) break; \
-    while((exclude = darshan_path_exclusions[tmp_index])) { \
-        if(!(strncmp(exclude, __path, strlen(exclude)))) \
-            break; \
-        tmp_index++; \
-    } \
-    if(exclude) break; \
+    if(darshan_core_excluded_path(__path)) break; \
     file = posix_file_by_name_setfd(__path, __ret); \
     if(!file) break; \
     if(__mode) \
@@ -337,15 +332,8 @@ static void posix_shutdown(void);
 } while(0)
 
 #define POSIX_LOOKUP_RECORD_STAT(__path, __statbuf, __tm1, __tm2) do { \
-    char* exclude; \
-    int tmp_index = 0; \
     struct posix_file_runtime* file; \
-    while((exclude = darshan_path_exclusions[tmp_index])) { \
-        if(!(strncmp(exclude, __path, strlen(exclude)))) \
-            break; \
-        tmp_index++; \
-    } \
-    if(exclude) break; \
+    if(darshan_core_excluded_path(__path)) break; \
     file = posix_file_by_name(__path); \
     if(file) \
     { \
@@ -1449,13 +1437,15 @@ int DARSHAN_DECL(lio_listio64)(int mode, struct aiocb64 *const aiocb_list[],
 /* initialize internal POSIX module data structures and register with darshan-core */
 static void posix_runtime_initialize()
 {
-    int mem_limit;
     struct darshan_module_funcs posix_mod_fns =
     {
         .begin_shutdown = &posix_begin_shutdown,
         .get_output_data = &posix_get_output_data,
         .shutdown = &posix_shutdown
     };
+    int mem_limit;
+    void *mmap_buf;
+    int mmap_buf_size;
 
     /* don't do anything if already initialized or instrumenation is disabled */
     if(posix_runtime || instrumentation_disabled)
@@ -1467,6 +1457,8 @@ static void posix_runtime_initialize()
         &posix_mod_fns,
         &my_rank,
         &mem_limit,
+        &mmap_buf,
+        &mmap_buf_size,
         &darshan_mem_alignment);
 
     /* return if no memory assigned by darshan core */
@@ -1499,6 +1491,17 @@ static void posix_runtime_initialize()
     memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
            sizeof(struct darshan_posix_file));
 
+    /* XXX-MMAP */
+    if(mmap_buf_size >= sizeof(struct darshan_posix_file))
+    {
+        memset(&(posix_runtime->agg_file_runtime), 0,
+            sizeof(struct posix_file_runtime));
+        posix_runtime->agg_file_runtime.file_record =
+            (struct darshan_posix_file *)mmap_buf;
+        posix_runtime->agg_file_runtime.file_record->f_id = DARSHAN_POSIX_MOD;
+        posix_runtime->agg_file_runtime.file_record->rank = my_rank;
+    }
+
     return;
 }
 
@@ -1513,6 +1516,8 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
     if(!posix_runtime || instrumentation_disabled)
         return(NULL);
 
+    return(&(posix_runtime->agg_file_runtime));
+#if 0
     newname = darshan_clean_file_path(name);
     if(!newname)
         newname = (char*)name;
@@ -1564,6 +1569,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
     if(newname != name)
         free(newname);
     return(file);
+#endif
 }
 
 /* get a POSIX file record for the given file path, and also create a
@@ -1580,6 +1586,7 @@ static struct posix_file_runtime* posix_file_by_name_setfd(const char* name, int
     /* find file record by name first */
     file = posix_file_by_name(name);
 
+#if 0
     if(!file)
         return(NULL);
 
@@ -1605,6 +1612,7 @@ static struct posix_file_runtime* posix_file_by_name_setfd(const char* name, int
     ref->file = file;
     ref->fd = fd;    
     HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);
+#endif
 
     return(file);
 }
@@ -1617,12 +1625,16 @@ static struct posix_file_runtime* posix_file_by_fd(int fd)
     if(!posix_runtime || instrumentation_disabled)
         return(NULL);
 
+    return(posix_file_by_name(NULL));
+
+#if 0
     /* search hash table for existing file ref for this fd */
     HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
     if(ref)
         return(ref->file);
 
     return(NULL);
+#endif
 }
 
 /* free up reference data structures for the given file descriptor */
@@ -1633,6 +1645,7 @@ static void posix_file_close_fd(int fd)
     if(!posix_runtime || instrumentation_disabled)
         return;
 
+#if 0
     /* search hash table for this fd */
     HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
     if(ref)
@@ -1641,6 +1654,7 @@ static void posix_file_close_fd(int fd)
         HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
         free(ref);
     }
+#endif
 
     return;
 }
@@ -1715,8 +1729,6 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
     struct darshan_posix_file *inoutfile = inoutfile_v;
     int i, j, k;
 
-    assert(posix_runtime);
-
     for(i=0; i<*len; i++)
     {
         memset(&tmp_file, 0, sizeof(struct darshan_posix_file));