Updates to bgq module core

darshan-runtime/lib/darshan-bgq.c

@@ -41,7 +41,6 @@ struct bgq_runtime
     struct darshan_bgq_record record;
 };
 
-/* null_runtime is the global data structure encapsulating "NULL" module state */
 static struct bgq_runtime *bgq_runtime = NULL;
 static pthread_mutex_t bgq_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
 
@@ -52,12 +51,14 @@ static int instrumentation_disabled = 0;
 static int my_rank = -1;
 
 /* internal helper functions for the "NULL" module */
-static void bgq_runtime_initialize(void);
+void bgq_runtime_initialize(void);
 
 /* forward declaration for module functions needed to interface with darshan-core */
 static void bgq_begin_shutdown(void);
 static void bgq_get_output_data(void **buffer, int *size);
 static void bgq_shutdown(void);
+static void bgq_setup_reduction(darshan_record_id *shared_recs,int *shared_rec_count,void **send_buf,void **recv_buf,int *rec_size);
+static void bgq_record_reduction_op(void* infile_v,void* inoutfile_v,int *len,MPI_Datatype *datatype);
 
 /* macros for obtaining/releasing the "NULL" module lock */
 #define BGQ_LOCK() pthread_mutex_lock(&bgq_runtime_mutex)
@@ -70,10 +71,13 @@ static void capture(struct darshan_bgq_record *rec)
 {
 #ifdef __bgq__
     Personality_t person;
+    int r;
 
     rec->counters[BGQ_CSJOBID] = Kernel_GetJobID();
     rec->counters[BGQ_RANKSPERNODE] = Kernel_ProcessCount();
 
+    rec->counters[BGQ_INODES] = MPIX_IO_node();
+
     r = Kernel_GetPersonality(&person, sizeof(person));
     if (r == 0)
     {
@@ -104,14 +108,14 @@ static void capture(struct darshan_bgq_record *rec)
  * Internal functions for manipulating BGQ module state *
  **********************************************************/
 
-static void bgq_runtime_initialize()
+void bgq_runtime_initialize()
 {
     /* struct of function pointers for interfacing with darshan-core */
     struct darshan_module_funcs bgq_mod_fns =
     {
         .begin_shutdown = bgq_begin_shutdown,
-        .setup_reduction = NULL, 
-        .record_reduction_op = NULL, 
+        .setup_reduction = bgq_setup_reduction, 
+        .record_reduction_op = bgq_record_reduction_op, 
         .get_output_data = bgq_get_output_data,
         .shutdown = bgq_shutdown
     };
@@ -198,9 +202,9 @@ static void bgq_get_output_data(
      * I/O records, and set the output size according to the number of records
      * currently being tracked.
      */
-    if (bgq_runtime)
+    if ((bgq_runtime) && (my_rank == 0))
     {
-        *buffer = (void *)&bgq_runtime->record;
+        *buffer = &bgq_runtime->record;
         *size = sizeof(struct darshan_bgq_record);
     }
     else
@@ -224,6 +228,70 @@ static void bgq_shutdown()
     return;
 }
 
+static void bgq_setup_reduction(
+    darshan_record_id *shared_recs,
+    int *shared_rec_count,
+    void **send_buf,
+    void **recv_buf,
+    int *rec_size)
+{
+    int i;
+    int found;
+
+    for (i = 0; i < *shared_rec_count; i++)
+    {
+        if (shared_recs[i] == bgq_runtime->record.f_id)
+        {
+            found = 1;
+            break;
+        }
+    }
+
+    if (found)
+    {
+        printf("found bgq shared record\n");
+        *rec_size = sizeof(struct darshan_bgq_record);
+        *shared_rec_count = 1;
+        *send_buf = &bgq_runtime->record;
+        *recv_buf = &bgq_runtime->record;
+    }
+
+    return;
+}
+
+static void bgq_record_reduction_op(
+    void* infile_v,
+    void* inoutfile_v,
+    int* len,
+    MPI_Datatype *datatype)
+{
+    int i;
+    int j;
+    struct darshan_bgq_record *infile = infile_v;
+    struct darshan_bgq_record *inoutfile = inoutfile_v;
+
+    for (i = 0; i<*len; i++)
+    {
+        for (j = 0; j < BGQ_NUM_INDICES; j++)
+        {
+            if (infile->counters[j] != inoutfile->counters[j])
+            {
+                // unexpected
+                fprintf(stderr,
+                        "%lu counter mismatch: %d [%lu] [%lu]\n",
+                        infile->f_id,
+                        j,
+                        infile->counters[j],
+                        inoutfile->counters[j]);
+            }
+        }
+        infile++;
+        inoutfile++;
+    }
+
+    return;
+}
+
 /*
  * Local variables:
  *  c-indent-level: 4

darshan-runtime/lib/darshan-core.c

@@ -208,6 +208,11 @@ void darshan_core_initialize(int argc, char **argv)
         }
     }
 
+#ifdef __bgq__
+    extern void bgq_runtime_initialize();
+    bgq_runtime_initialize();
+#endif
+
     return;
 }
 

darshan-runtime/lib/darshan-mpiio.c

@@ -453,16 +453,19 @@ static void mpiio_setup_reduction(
 {
     struct mpiio_file_runtime *file;
     int i;
+    int count;
 
     assert(mpiio_runtime);
 
     /* necessary initialization of shared records (e.g., change rank to -1) */
-    for(i = 0; i < *shared_rec_count; i++)
+    for(i = 0,count = 0; i < *shared_rec_count; i++)
     {
         HASH_FIND(hlink, mpiio_runtime->file_hash, &shared_recs[i],
             sizeof(darshan_record_id), file);
-        assert(file);
-
+        if (!file) {
+            continue;
+        }
+        count++;
         file->file_record->rank = -1;
     }
 
@@ -475,22 +478,18 @@ static void mpiio_setup_reduction(
 
     /* make *send_buf point to the shared files at the end of sorted array */
     *send_buf =
-        &(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-(*shared_rec_count)]);
+        &(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-(count)]);
 
     /* allocate memory for the reduction output on rank 0 */
-    if(my_rank == 0)
+    if((my_rank == 0) && (count > 0))
     {
-        *recv_buf = malloc(*shared_rec_count * sizeof(struct darshan_mpiio_file));
-        if(!(*recv_buf))
-            return;
+        *recv_buf = malloc(count * sizeof(struct darshan_mpiio_file));
+        mpiio_runtime->red_buf = *recv_buf;
     }
 
     *rec_size = sizeof(struct darshan_mpiio_file);
 
-    /* TODO: cleaner way to do this? */
-    if(my_rank == 0)
-        mpiio_runtime->red_buf = *recv_buf;
-    mpiio_runtime->shared_rec_count = *shared_rec_count;
+    mpiio_runtime->shared_rec_count = count;
 
     return;
 }

darshan-runtime/lib/darshan-posix.c

@@ -1655,15 +1655,19 @@ static void posix_setup_reduction(
     struct posix_file_runtime *file;
     int i;
     double posix_time;
+    int count;
 
     assert(posix_runtime);
 
     /* necessary initialization of shared records (e.g., change rank to -1) */
-    for(i = 0; i < *shared_rec_count; i++)
+    for(i = 0,count = 0; i < *shared_rec_count; i++)
     {
         HASH_FIND(hlink, posix_runtime->file_hash, &shared_recs[i],
             sizeof(darshan_record_id), file);
-        assert(file);
+        if (!file) {
+            continue;
+        }
+        count++;
 
         posix_time =
             file->file_record->fcounters[POSIX_F_READ_TIME] +
@@ -1702,22 +1706,22 @@ static void posix_setup_reduction(
 
     /* make *send_buf point to the shared files at the end of sorted array */
     *send_buf =
-        &(posix_runtime->file_record_array[posix_runtime->file_array_ndx-(*shared_rec_count)]);
+        &(posix_runtime->file_record_array[posix_runtime->file_array_ndx-(count)]);
 
     /* allocate memory for the reduction output on rank 0 */
-    if(my_rank == 0)
+    if((my_rank == 0) && (count > 0))
     {
-        *recv_buf = malloc(*shared_rec_count * sizeof(struct darshan_posix_file));
-        if(!(*recv_buf))
-            return;
+        printf("shared count = %d\n", count);
+        *recv_buf = malloc(count * sizeof(struct darshan_posix_file));
+        posix_runtime->red_buf = *recv_buf;
+        printf("recv_buf = %p\n", *recv_buf);
+//        if(!(*recv_buf))
+//            return;
     }
 
     *rec_size = sizeof(struct darshan_posix_file);
 
-    /* TODO: cleaner way to do this? */
-    if(my_rank == 0)
-        posix_runtime->red_buf = *recv_buf;
-    posix_runtime->shared_rec_count = *shared_rec_count;
+    posix_runtime->shared_rec_count = count;
 
     return;
 }
@@ -1962,7 +1966,7 @@ static void posix_get_output_data(
 
     /* TODO: cleaner way to do this? */
     /* clean up reduction state */
-    if(my_rank == 0)
+    if((my_rank == 0) && (posix_runtime->red_buf))
     {
         int tmp_ndx = posix_runtime->file_array_ndx - posix_runtime->shared_rec_count;
         memcpy(&(posix_runtime->file_record_array[tmp_ndx]), posix_runtime->red_buf,

darshan-util/Makefile.in

-all: libdarshan-util.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser
+all: libdarshan-util.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser darshan-bgq-parser
 
 DESTDIR =
 srcdir = @srcdir@
@@ -74,6 +74,9 @@ darshan-posix-parser: darshan-posix-parser.c darshan-logutils.h darshan-posix-lo
 darshan-mpiio-parser: darshan-mpiio-parser.c darshan-logutils.h darshan-mpiio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h libdarshan-util.a | uthash-1.9.2
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS) 
 
+darshan-bgq-parser: darshan-bgq-parser.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h libdarshan-util.a | uthash-1.9.2
+	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS) 
+
 #darshan-parser: darshan-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o | uthash-1.9.2
 #	$(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS) 
 
@@ -109,6 +112,7 @@ install:: all
 	install -m 755 darshan-base-parser $(bindir)
 	install -m 755 darshan-posix-parser $(bindir)
 	install -m 755 darshan-mpiio-parser $(bindir)
+	install -m 755 darshan-bgq-parser $(bindir)
 #	install -m 755 darshan-convert $(bindir)
 #	install -m 755 darshan-diff $(bindir)
 #	install -m 755 darshan-analyzer $(bindir)