moved the temporary record buffer used in darshan-{convert,parser} into the...

moved the temporary record buffer used in darshan-{convert,parser} into the heap; darshan_diff remains on the stack

darshan-util/darshan-convert.c

@@ -19,8 +19,6 @@
 
 #include "darshan-logutils.h"
 
-#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
-
 extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
 
 int usage (char *exename)
@@ -233,7 +231,7 @@ int main(int argc, char **argv)
     struct darshan_mnt_info *mnt_data_array;
     struct darshan_record_ref *rec_hash = NULL;
     struct darshan_record_ref *ref, *tmp;
-    char mod_buf[DEF_MOD_BUF_SIZE];
+    char *mod_buf;
     enum darshan_comp_type comp_type;
     int bzip2;
     int obfuscate;
@@ -338,6 +336,14 @@ int main(int argc, char **argv)
         return(-1);
     }
 
+    mod_buf = malloc(DEF_MOD_BUF_SIZE);
+    if (!mod_buf)
+    {
+        darshan_log_close(infile);
+        darshan_log_close(outfile);
+        return(-1);
+    }
+
     /* loop over each module and convert it's data to the new format */
     for(i=0; i<DARSHAN_MAX_MODS; i++)
     {
@@ -364,6 +370,7 @@ int main(int argc, char **argv)
             darshan_log_close(infile);
             darshan_log_close(outfile);
             unlink(outfile_name);
+            free(mod_buf);
             return(-1);
         }
 
@@ -384,6 +391,7 @@ int main(int argc, char **argv)
             }
         } while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1);
     }
+    free(mod_buf);
 
     darshan_log_close(infile);
     darshan_log_close(outfile);

darshan-util/darshan-diff.c

@@ -13,8 +13,6 @@
 #include "darshan-logutils.h"
 #include "uthash-1.9.2/src/uthash.h"
 
-#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
-
 /* XXX: this structure is a temporary hack to get at the rank for each module's record */
 struct darshan_base_rec
 {

darshan-util/darshan-logutils.h

@@ -17,6 +17,11 @@
 
 #include "darshan-log-format.h"
 
+/* Maximum size of a record - Lustre OST lists can get huge, but 81920 is enough
+ * for 10K OSTs 
+ */
+#define DEF_MOD_BUF_SIZE 81920 /* 640 KiB */
+
 struct darshan_fd_int_state;
 
 /* darshan file descriptor definition */

darshan-util/darshan-parser.c

@@ -20,8 +20,6 @@
 
 #include "darshan-logutils.h"
 
-#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
-
 /*
  * Options
  */
@@ -211,7 +209,7 @@ int main(int argc, char **argv)
     char *save;
     char buffer[DARSHAN_JOB_METADATA_LEN];
     int empty_mods = 0;
-    char mod_buf[DEF_MOD_BUF_SIZE];
+    char *mod_buf;
 
     hash_entry_t *file_hash = NULL;
     hash_entry_t *curr = NULL;
@@ -350,6 +348,12 @@ int main(int argc, char **argv)
         memset(pdata.rank_cumul_md_time, 0, sizeof(double)*job.nprocs);
     }
 
+    mod_buf = malloc(DEF_MOD_BUF_SIZE);
+    if (!mod_buf) {
+        darshan_log_close(fd);
+        return(-1);
+    }
+
     for(i=0; i<DARSHAN_MAX_MODS; i++)
     {
         darshan_record_id rec_id;
@@ -624,6 +628,7 @@ cleanup:
     darshan_log_close(fd);
     free(pdata.rank_cumul_io_time);
     free(pdata.rank_cumul_md_time);
+    free(mod_buf);
 
     /* free record hash data */
     HASH_ITER(hlink, rec_hash, ref, tmp_ref)