Commit 735fc530 authored by Glenn K. Lockwood's avatar Glenn K. Lockwood

moved the temporary record buffer used in darshan-{convert,parser} into the...

moved the temporary record buffer used in darshan-{convert,parser} into the heap; darshan_diff remains on the stack
parent 4264247f
...@@ -19,8 +19,6 @@ ...@@ -19,8 +19,6 @@
#include "darshan-logutils.h" #include "darshan-logutils.h"
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval); extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
int usage (char *exename) int usage (char *exename)
...@@ -233,7 +231,7 @@ int main(int argc, char **argv) ...@@ -233,7 +231,7 @@ int main(int argc, char **argv)
struct darshan_mnt_info *mnt_data_array; struct darshan_mnt_info *mnt_data_array;
struct darshan_record_ref *rec_hash = NULL; struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp; struct darshan_record_ref *ref, *tmp;
char mod_buf[DEF_MOD_BUF_SIZE]; char *mod_buf;
enum darshan_comp_type comp_type; enum darshan_comp_type comp_type;
int bzip2; int bzip2;
int obfuscate; int obfuscate;
...@@ -338,6 +336,14 @@ int main(int argc, char **argv) ...@@ -338,6 +336,14 @@ int main(int argc, char **argv)
return(-1); return(-1);
} }
mod_buf = malloc(DEF_MOD_BUF_SIZE);
if (!mod_buf)
{
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
/* loop over each module and convert it's data to the new format */ /* loop over each module and convert it's data to the new format */
for(i=0; i<DARSHAN_MAX_MODS; i++) for(i=0; i<DARSHAN_MAX_MODS; i++)
{ {
...@@ -364,6 +370,7 @@ int main(int argc, char **argv) ...@@ -364,6 +370,7 @@ int main(int argc, char **argv)
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name); unlink(outfile_name);
free(mod_buf);
return(-1); return(-1);
} }
...@@ -384,6 +391,7 @@ int main(int argc, char **argv) ...@@ -384,6 +391,7 @@ int main(int argc, char **argv)
} }
} while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1); } while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1);
} }
free(mod_buf);
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
......
...@@ -13,8 +13,6 @@ ...@@ -13,8 +13,6 @@
#include "darshan-logutils.h" #include "darshan-logutils.h"
#include "uthash-1.9.2/src/uthash.h" #include "uthash-1.9.2/src/uthash.h"
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
/* XXX: this structure is a temporary hack to get at the rank for each module's record */ /* XXX: this structure is a temporary hack to get at the rank for each module's record */
struct darshan_base_rec struct darshan_base_rec
{ {
......
...@@ -17,6 +17,11 @@ ...@@ -17,6 +17,11 @@
#include "darshan-log-format.h" #include "darshan-log-format.h"
/* Maximum size of a record - Lustre OST lists can get huge, but 81920 is enough
* for 10K OSTs
*/
#define DEF_MOD_BUF_SIZE 81920 /* 640 KiB */
struct darshan_fd_int_state; struct darshan_fd_int_state;
/* darshan file descriptor definition */ /* darshan file descriptor definition */
......
...@@ -20,8 +20,6 @@ ...@@ -20,8 +20,6 @@
#include "darshan-logutils.h" #include "darshan-logutils.h"
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
/* /*
* Options * Options
*/ */
...@@ -211,7 +209,7 @@ int main(int argc, char **argv) ...@@ -211,7 +209,7 @@ int main(int argc, char **argv)
char *save; char *save;
char buffer[DARSHAN_JOB_METADATA_LEN]; char buffer[DARSHAN_JOB_METADATA_LEN];
int empty_mods = 0; int empty_mods = 0;
char mod_buf[DEF_MOD_BUF_SIZE]; char *mod_buf;
hash_entry_t *file_hash = NULL; hash_entry_t *file_hash = NULL;
hash_entry_t *curr = NULL; hash_entry_t *curr = NULL;
...@@ -350,6 +348,12 @@ int main(int argc, char **argv) ...@@ -350,6 +348,12 @@ int main(int argc, char **argv)
memset(pdata.rank_cumul_md_time, 0, sizeof(double)*job.nprocs); memset(pdata.rank_cumul_md_time, 0, sizeof(double)*job.nprocs);
} }
mod_buf = malloc(DEF_MOD_BUF_SIZE);
if (!mod_buf) {
darshan_log_close(fd);
return(-1);
}
for(i=0; i<DARSHAN_MAX_MODS; i++) for(i=0; i<DARSHAN_MAX_MODS; i++)
{ {
darshan_record_id rec_id; darshan_record_id rec_id;
...@@ -624,6 +628,7 @@ cleanup: ...@@ -624,6 +628,7 @@ cleanup:
darshan_log_close(fd); darshan_log_close(fd);
free(pdata.rank_cumul_io_time); free(pdata.rank_cumul_io_time);
free(pdata.rank_cumul_md_time); free(pdata.rank_cumul_md_time);
free(mod_buf);
/* free record hash data */ /* free record hash data */
HASH_ITER(hlink, rec_hash, ref, tmp_ref) HASH_ITER(hlink, rec_hash, ref, tmp_ref)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment