Implement a separate dxt parser for parsing dxt logs

Signed-off-by: Cong Xu <cong.xu@intel.com>

darshan-util/darshan-dxt-logutils.c

@@ -21,8 +21,8 @@
 
 static int dxt_log_get_posix_file(darshan_fd fd, void** dxt_posix_buf);
 static int dxt_log_put_posix_file(darshan_fd fd, void* dxt_posix_buf);
-static void dxt_log_print_posix_file(void *file_rec, char *file_name,
-        char *mnt_pt, char *fs_type, struct darshan_name_record_ref *ref);
+void dxt_log_print_posix_file(void *file_rec, char *file_name,
+        char *mnt_pt, char *fs_type, struct darshan_lustre_record *ref);
 static void dxt_log_print_posix_description(int ver);
 static void dxt_log_print_posix_file_diff(void *file_rec1, char *file_name1,
     void *file_rec2, char *file_name2);
@@ -30,7 +30,7 @@ static void dxt_log_agg_posix_files(void *rec, void *agg_rec, int init_flag);
 
 static int dxt_log_get_mpiio_file(darshan_fd fd, void** dxt_mpiio_buf);
 static int dxt_log_put_mpiio_file(darshan_fd fd, void* dxt_mpiio_buf);
-static void dxt_log_print_mpiio_file(void *file_rec,
+void dxt_log_print_mpiio_file(void *file_rec,
         char *file_name, char *mnt_pt, char *fs_type);
 static void dxt_log_print_mpiio_description(int ver);
 static void dxt_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
@@ -44,7 +44,6 @@ struct darshan_mod_logutil_funcs dxt_posix_logutils =
 {
     .log_get_record = &dxt_log_get_posix_file,
     .log_put_record = &dxt_log_put_posix_file,
-    .log_print_record_dxt = &dxt_log_print_posix_file,
 //    .log_print_description = &dxt_log_print_posix_description,
     .log_print_diff = &dxt_log_print_posix_file_diff,
     .log_agg_records = &dxt_log_agg_posix_files,
@@ -54,7 +53,6 @@ struct darshan_mod_logutil_funcs dxt_mpiio_logutils =
 {
     .log_get_record = &dxt_log_get_mpiio_file,
     .log_put_record = &dxt_log_put_mpiio_file,
-    .log_print_record = &dxt_log_print_mpiio_file,
 //    .log_print_description = &dxt_log_print_mpiio_description,
     .log_print_diff = &dxt_log_print_mpiio_file_diff,
     .log_agg_records = &dxt_log_agg_mpiio_files,
@@ -196,8 +194,8 @@ static int dxt_log_put_mpiio_file(darshan_fd fd, void* dxt_mpiio_buf)
     return(0);
 }
 
-static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
-    char *mnt_pt, char *fs_type, struct darshan_name_record_ref *ref)
+void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
+    char *mnt_pt, char *fs_type, struct darshan_lustre_record *ref)
 {
     struct dxt_file_record *file_rec =
                 (struct dxt_file_record *)posix_file_rec;
@@ -222,18 +220,21 @@ static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
     int print_count;
     int ost_idx;
     
+    if (!ref) {
+        lustreFS = 0;
+    }
 
     printf("\n# DXT, file_id: %" PRIu64 ", file_name: %s\n", f_id, file_name);
     printf("# DXT, rank: %d, write_count: %d, read_count: %d\n",
                 rank, write_count, read_count);
 
     if (lustreFS) {
-        stripe_size = ref->stripe_size;
-        stripe_count = ref->stripe_count;
+        stripe_size = ref->counters[LUSTRE_STRIPE_SIZE];
+        stripe_count = ref->counters[LUSTRE_STRIPE_WIDTH];
 
         printf("# DXT, mnt_pt: %s, fs_type: %s\n", mnt_pt, fs_type);
         printf("# DXT, stripe_size: %d, stripe_count: %d\n", stripe_size, stripe_count);
-        for (i = 0; i < ref->stripe_count; i++) {
+        for (i = 0; i < stripe_count; i++) {
             printf("# DXT, OST: %d\n", (ref->ost_ids)[i]);
         }
     }
@@ -305,7 +306,7 @@ static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
     return;
 }
 
-static void dxt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
+void dxt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     struct dxt_file_record *file_rec =

darshan-util/darshan-logutils.h

@@ -53,11 +53,13 @@ struct darshan_name_record_ref
 {
     struct darshan_name_record *name_record;
     UT_hash_handle hlink;
+};
 
-	/* DXT Lustre */
-	int32_t stripe_size;
-	int32_t stripe_count;
-	OST_ID *ost_ids;
+/* DXT */
+struct lustre_record_ref
+{
+	struct darshan_lustre_record *rec;
+	UT_hash_handle hlink;
 };
 
 struct darshan_mnt_info
@@ -121,15 +123,6 @@ struct darshan_mod_logutil_funcs
         void *agg_rec,
         int init_flag
     );
-
-	/* DXT */
-    void (*log_print_record_dxt)(
-        void *rec,
-        char *file_name,
-        char *mnt_pt,
-        char *fs_type,
-		struct darshan_name_record_ref *ref
-    );
 };
 
 extern struct darshan_mod_logutil_funcs *mod_logutils[];

darshan-util/darshan-parser.c

@@ -121,9 +121,6 @@ void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver);
 
 void calc_perf(perf_data_t *pdata, int64_t nprocs);
 
-/* DXT */
-extern void dxt_logutils_cleanup();
-
 int usage (char *exename)
 {
     fprintf(stderr, "Usage: %s [options] <filename>\n", exename);
@@ -401,7 +398,8 @@ int main(int argc, char **argv)
         /* currently only POSIX, MPIIO, and STDIO modules support non-base
          * parsing
          */
-        else if((i != DARSHAN_POSIX_MOD) && (i != DARSHAN_MPIIO_MOD) && (i != DARSHAN_STDIO_MOD) && !(mask & OPTION_BASE))
+        else if((i != DARSHAN_POSIX_MOD) && (i != DARSHAN_MPIIO_MOD) &&
+                (i != DARSHAN_STDIO_MOD) && !(mask & OPTION_BASE))
             continue;
 
         /* this module has data to be parsed and printed */
@@ -428,22 +426,24 @@ int main(int argc, char **argv)
             }
         }
 
-        ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
-        if(ret != 1)
-        {
-            fprintf(stderr, "Error: failed to parse the first %s module record.\n",
-                darshan_module_names[i]);
-            ret = -1;
-            goto cleanup;
-        }
-
         /* loop over each of this module's records and print them */
-        do
+        while(1)
         {
             char *mnt_pt = NULL;
             char *fs_type = NULL;
             char *rec_name = NULL;
             hash_entry_t *hfile = NULL;
+
+            ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
+            if(ret < 1)
+            {
+                if(ret == -1)
+                {
+                    fprintf(stderr, "Error: failed to parse %s module record.\n",
+                        darshan_module_names[i]);
+                }
+                break;
+            }
             base_rec = (struct darshan_base_record *)mod_buf;
 
             /* get the pathname for this record */
@@ -476,32 +476,11 @@ int main(int argc, char **argv)
             if(!fs_type)
                 fs_type = "UNKNOWN";
 
-            /* DXT */
-            if (i == DARSHAN_LUSTRE_MOD && ref) {
-                /* LUSTRE MODULE */
-                struct darshan_lustre_record *file_rec = 
-                            (struct darshan_lustre_record *)mod_buf;
-
-                ref->stripe_size = file_rec->counters[LUSTRE_STRIPE_SIZE];
-                ref->stripe_count = file_rec->counters[LUSTRE_STRIPE_WIDTH];
-
-                int ost_ids_size = ref->stripe_count * sizeof(OST_ID);
-                ref->ost_ids = (OST_ID *) malloc(ost_ids_size);
-                memcpy((void *)(ref->ost_ids), (void *)(file_rec->ost_ids),
-                                    ost_ids_size);
-            }
-
             if(mask & OPTION_BASE)
             {
-                /* DXT */
-                if (i == DXT_POSIX_MOD) {
-                    mod_logutils[i]->log_print_record_dxt(mod_buf, rec_name,
-                            mnt_pt, fs_type, ref);
-                } else {
-                    /* print the corresponding module data for this record */
-                    mod_logutils[i]->log_print_record(mod_buf, rec_name,
-                            mnt_pt, fs_type);
-                }
+                /* print the corresponding module data for this record */
+                mod_logutils[i]->log_print_record(mod_buf, rec_name,
+                    mnt_pt, fs_type);
             }
 
             /* we calculate more detailed stats for POSIX, MPI-IO, and STDIO modules, 
@@ -552,13 +531,9 @@ int main(int argc, char **argv)
             }
 
             memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
-
-        } while((ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf)) == 1);
-        if (ret < 0)
-        {
-            ret = -1;
-            goto cleanup;
         }
+        if(ret == -1)
+            continue; /* move on to the next module if there was an error with this one */
 
         /* we calculate more detailed stats for POSIX and MPI-IO modules, 
          * if the parser is executed with more than the base option
@@ -713,9 +688,6 @@ int main(int argc, char **argv)
             if(curr->rec_dat) free(curr->rec_dat);
             free(curr);
         }
-
-        /* DXT */
-        dxt_logutils_cleanup();
     }
     if(empty_mods == DARSHAN_MAX_MODS)
         printf("\n# no module data available.\n");