Commit 8659e6a3 authored by Shane Snyder's avatar Shane Snyder

new implementation of darshan-logutils

parent 86d17440
...@@ -35,18 +35,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -35,18 +35,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
{ {
int ret; int ret;
darshan_fd file; darshan_fd file;
struct darshan_header header;
struct darshan_job job; struct darshan_job job;
struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD]; struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD];
struct darshan_posix_file *psx_buf, *psx_buf_p; struct darshan_posix_file psx_rec;
int psx_buf_sz, psx_buf_bytes_left;
struct darshan_posix_file *psx_rec;
darshan_record_id rec_id; darshan_record_id rec_id;
int f_count; int f_count;
double total_io_time; double total_io_time;
double total_job_time; double total_job_time;
assert(psx_mod); assert(psx_mod);
memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
file = darshan_log_open(fname); file = darshan_log_open(fname);
if (file == NULL) if (file == NULL)
...@@ -55,14 +53,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -55,14 +53,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
return -1; return -1;
} }
ret = darshan_log_getheader(file, &header);
if (ret < 0)
{
fprintf(stderr, "darshan_log_getheader() failed on file %s.\n", fname);
darshan_log_close(file);
return -1;
}
ret = darshan_log_getjob(file, &job); ret = darshan_log_getjob(file, &job);
if (ret < 0) if (ret < 0)
{ {
...@@ -71,49 +61,36 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -71,49 +61,36 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
return -1; return -1;
} }
psx_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
psx_buf = malloc(psx_buf_sz);
if (!psx_buf)
{
darshan_log_close(file);
return -1;
}
ret = darshan_log_getmod(file, DARSHAN_POSIX_MOD, (void *)psx_buf, &psx_buf_sz);
if (ret < 0)
{
fprintf(stderr, "darshan_log_getmod() failed on file %s.\n", fname);
darshan_log_close(file);
return -1;
}
f_count = 0; f_count = 0;
total_io_time = 0.0; total_io_time = 0.0;
psx_buf_bytes_left = psx_buf_sz; while((ret = psx_mod->log_get_record(file, &psx_rec, &rec_id)) == 1)
psx_buf_p = psx_buf;
while(psx_buf_bytes_left)
{ {
ret = psx_mod->log_get_record((void **)&psx_buf_p, &psx_buf_bytes_left,
(void **)&psx_rec, &rec_id, file->swap_flag);
f_count += 1; f_count += 1;
if (psx_rec->rank == -1) if (psx_rec.rank == -1)
*used_shared = 1; *used_shared = 1;
else else
*used_fpp = 1; *used_fpp = 1;
total_io_time += (psx_rec->fcounters[POSIX_F_READ_TIME] + total_io_time += (psx_rec.fcounters[POSIX_F_READ_TIME] +
psx_rec->fcounters[POSIX_F_WRITE_TIME] + psx_rec.fcounters[POSIX_F_WRITE_TIME] +
psx_rec->fcounters[POSIX_F_META_TIME]); psx_rec.fcounters[POSIX_F_META_TIME]);
memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
}
if (ret < 0)
{
fprintf(stderr, "Error: unable to read posix file record in log file %s.\n", fname);
darshan_log_close(file);
return -1;
} }
if (header.mod_map[DARSHAN_MPIIO_MOD].len > 0) if (file->mod_map[DARSHAN_MPIIO_MOD].len > 0)
*used_mpio += 1; *used_mpio += 1;
if (header.mod_map[DARSHAN_HDF5_MOD].len > 0) if (file->mod_map[DARSHAN_HDF5_MOD].len > 0)
*used_hdf5 += 1; *used_hdf5 += 1;
if (header.mod_map[DARSHAN_PNETCDF_MOD].len > 0) if (file->mod_map[DARSHAN_PNETCDF_MOD].len > 0)
*used_pnet += 1; *used_pnet += 1;
total_job_time = (double)job.end_time - (double)job.start_time; total_job_time = (double)job.end_time - (double)job.start_time;
...@@ -131,7 +108,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -131,7 +108,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
*io_ratio = 0.0; *io_ratio = 0.0;
} }
free(psx_buf);
darshan_log_close(file); darshan_log_close(file);
return 0; return 0;
......
...@@ -30,48 +30,65 @@ char *bgq_f_counter_names[] = { ...@@ -30,48 +30,65 @@ char *bgq_f_counter_names[] = {
}; };
#undef X #undef X
static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left, static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag); darshan_record_id* rec_id);
static void darshan_log_print_bgq_file(void *file_rec, static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf);
static void darshan_log_print_bgq_rec(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type); char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs bgq_logutils = struct darshan_mod_logutil_funcs bgq_logutils =
{ {
.log_get_record = &darshan_log_get_bgq_file, .log_get_record = &darshan_log_get_bgq_rec,
.log_print_record = &darshan_log_print_bgq_file, .log_put_record = &darshan_log_put_bgq_rec,
.log_print_record = &darshan_log_print_bgq_rec,
}; };
static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left, static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag) darshan_record_id* rec_id)
{ {
struct darshan_bgq_record *rec;
int i; int i;
struct darshan_bgq_record *file = (struct darshan_bgq_record *) int ret;
(*psx_buf_p);
if(*bytes_left < sizeof(struct darshan_bgq_record)) ret = darshan_log_getmod(fd, DARSHAN_BGQ_MOD, bgq_buf,
sizeof(struct darshan_bgq_record));
if(ret < 0)
return(-1); return(-1);
else if(ret < sizeof(struct darshan_bgq_record))
if(byte_swap_flag) return(0);
else
{ {
/* swap bytes if necessary */ rec = (struct darshan_bgq_record *)bgq_buf;
DARSHAN_BSWAP64(&file->f_id); if(fd->swap_flag)
DARSHAN_BSWAP64(&file->rank); {
for(i=0; i<BGQ_NUM_INDICES; i++) /* swap bytes if necessary */
DARSHAN_BSWAP64(&file->counters[i]); DARSHAN_BSWAP64(&rec->f_id);
for(i=0; i<BGQ_F_NUM_INDICES; i++) DARSHAN_BSWAP64(&rec->rank);
DARSHAN_BSWAP64(&file->fcounters[i]); for(i=0; i<BGQ_NUM_INDICES; i++)
DARSHAN_BSWAP64(&rec->counters[i]);
for(i=0; i<BGQ_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&rec->fcounters[i]);
}
*rec_id = rec->f_id;
return(1);
} }
}
/* update/set output variables */ static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf)
*file_rec = (void *)file; {
*rec_id = file->f_id; struct darshan_bgq_record *rec = (struct darshan_bgq_record *)bgq_buf;
*psx_buf_p = (file + 1); /* increment input buf by size of file record */ int ret;
*bytes_left -= sizeof(struct darshan_bgq_record);
ret = darshan_log_putmod(fd, DARSHAN_BGQ_MOD, rec,
sizeof(struct darshan_bgq_record));
if(ret < 0)
return(-1);
return(0); return(0);
} }
static void darshan_log_print_bgq_file(void *file_rec, char *file_name, static void darshan_log_print_bgq_rec(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type) char *mnt_pt, char *fs_type)
{ {
int i; int i;
......
...@@ -19,6 +19,8 @@ ...@@ -19,6 +19,8 @@
#include "darshan-logutils.h" #include "darshan-logutils.h"
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval); extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
int usage (char *exename) int usage (char *exename)
...@@ -221,7 +223,6 @@ int main(int argc, char **argv) ...@@ -221,7 +223,6 @@ int main(int argc, char **argv)
int ret; int ret;
char *infile_name; char *infile_name;
char *outfile_name; char *outfile_name;
struct darshan_header header;
struct darshan_job job; struct darshan_job job;
char tmp_string[4096] = {0}; char tmp_string[4096] = {0};
darshan_fd infile; darshan_fd infile;
...@@ -232,8 +233,7 @@ int main(int argc, char **argv) ...@@ -232,8 +233,7 @@ int main(int argc, char **argv)
char** fs_types; char** fs_types;
struct darshan_record_ref *rec_hash = NULL; struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp; struct darshan_record_ref *ref, *tmp;
char *mod_buf; char mod_buf[DEF_MOD_BUF_SIZE];
int mod_buf_sz;
enum darshan_comp_type comp_type; enum darshan_comp_type comp_type;
int bzip2; int bzip2;
int obfuscate; int obfuscate;
...@@ -247,42 +247,23 @@ int main(int argc, char **argv) ...@@ -247,42 +247,23 @@ int main(int argc, char **argv)
infile = darshan_log_open(infile_name); infile = darshan_log_open(infile_name);
if(!infile) if(!infile)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", infile_name);
return(-1); return(-1);
}
comp_type = bzip2 ? comp_type = DARSHAN_BZIP2_COMP : DARSHAN_ZLIB_COMP; comp_type = bzip2 ? comp_type = DARSHAN_BZIP2_COMP : DARSHAN_ZLIB_COMP;
outfile = darshan_log_create(outfile_name, comp_type); outfile = darshan_log_create(outfile_name, comp_type);
if(!outfile) if(!outfile)
{ {
fprintf(stderr, "darshan_log_create() failed to create %s\n.", outfile_name);
darshan_log_close(infile); darshan_log_close(infile);
return(-1); return(-1);
} }
/* read header from input file */
ret = darshan_log_getheader(infile, &header);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read header from input log file %s.\n", infile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
/* NOTE: we do not write the header to the output file until the end, as
* the mapping data stored in this structure may change in the conversion
* process (particularly, if we are converting between libz/bz2 compression)
*/
/* read job info */ /* read job info */
ret = darshan_log_getjob(infile, &job); ret = darshan_log_getjob(infile, &job);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read job information from log file.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name);
return(-1); return(-1);
} }
...@@ -293,7 +274,6 @@ int main(int argc, char **argv) ...@@ -293,7 +274,6 @@ int main(int argc, char **argv)
ret = darshan_log_putjob(outfile, &job); ret = darshan_log_putjob(outfile, &job);
if (ret < 0) if (ret < 0)
{ {
fprintf(stderr, "Error: unable to write job information to log file.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
...@@ -302,9 +282,9 @@ int main(int argc, char **argv) ...@@ -302,9 +282,9 @@ int main(int argc, char **argv)
ret = darshan_log_getexe(infile, tmp_string); ret = darshan_log_getexe(infile, tmp_string);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name);
return(-1); return(-1);
} }
...@@ -313,7 +293,6 @@ int main(int argc, char **argv) ...@@ -313,7 +293,6 @@ int main(int argc, char **argv)
ret = darshan_log_putexe(outfile, tmp_string); ret = darshan_log_putexe(outfile, tmp_string);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to write trailing job information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
...@@ -322,16 +301,15 @@ int main(int argc, char **argv) ...@@ -322,16 +301,15 @@ int main(int argc, char **argv)
ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count); ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name);
return(-1); return(-1);
} }
ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count); ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to write mount information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
...@@ -340,9 +318,9 @@ int main(int argc, char **argv) ...@@ -340,9 +318,9 @@ int main(int argc, char **argv)
ret = darshan_log_gethash(infile, &rec_hash); ret = darshan_log_gethash(infile, &rec_hash);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read darshan record hash.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name);
return(-1); return(-1);
} }
...@@ -355,104 +333,56 @@ int main(int argc, char **argv) ...@@ -355,104 +333,56 @@ int main(int argc, char **argv)
ret = darshan_log_puthash(outfile, rec_hash); ret = darshan_log_puthash(outfile, rec_hash);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to write darshan record hash.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
} }
mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ); /* loop over each module and convert it's data to the new format */
if(!mod_buf)
return(-1);
for(i=0; i<DARSHAN_MAX_MODS; i++) for(i=0; i<DARSHAN_MAX_MODS; i++)
{ {
int mod_bytes_left;
int mod_bytes_left_save;
void *mod_buf_p;
void *rec_p = NULL;
darshan_record_id rec_id; darshan_record_id rec_id;
memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* check each module for any data */ /* check each module for any data */
ret = darshan_log_getmod(infile, i, mod_buf, &mod_buf_sz); if(infile->mod_map[i].len == 0)
if(ret < 0)
{
fprintf(stderr, "Error: failed to get module %s data.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
else if(ret == 0)
{
/* skip modules not present in log file */
continue; continue;
} else if(!mod_logutils[i])
/* skip modules with no defined logutil handlers */
if(!mod_logutils[i])
{ {
fprintf(stderr, "Warning: no log utility handlers defined " fprintf(stderr, "Warning: no log utility handlers defined "
"for module %s, SKIPPING\n", darshan_module_names[i]); "for module %s, SKIPPING.\n", darshan_module_names[i]);
continue; continue;
} }
/* we have module data to convert */ /* we have module data to convert */
/* NOTE: it is necessary to iterate through each module's memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
* records to correct any endianness issues before writing
* this data back to file
*/
mod_bytes_left = mod_buf_sz;
mod_buf_p = mod_buf;
while(mod_bytes_left > 0)
{
mod_bytes_left_save = mod_bytes_left;
ret = mod_logutils[i]->log_get_record(&mod_buf_p, &mod_bytes_left,
&rec_p, &rec_id, infile->swap_flag);
if(ret < 0)
{
fprintf(stderr, "Error: failed to parse module %s data record\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
if(hash == rec_id) ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id);
{ if(ret != 1)
mod_buf_p = rec_p;
mod_buf_sz = mod_bytes_left_save - mod_bytes_left;
break;
}
else if(mod_bytes_left == 0)
{
mod_buf_p = mod_buf;
}
}
ret = darshan_log_putmod(outfile, i, mod_buf_p, mod_buf_sz);
if(ret < 0)
{ {
fprintf(stderr, "Error: failed to put module %s data.\n", fprintf(stderr, "Error: failed to parse the first %s module record.\n",
darshan_module_names[i]); darshan_module_names[i]);
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
unlink(outfile_name);
return(-1); return(-1);
} }
}
free(mod_buf);
/* write header to output file */ /* loop over each of the module's records and convert */
ret = darshan_log_putheader(outfile); do
if(ret < 0) {
{ if(!hash || hash == rec_id)
fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name); {
darshan_log_close(infile); ret = mod_logutils[i]->log_put_record(outfile, mod_buf);
darshan_log_close(outfile); if(ret < 0)
return(-1); {
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
}
} while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1);
} }
darshan_log_close(infile); darshan_log_close(infile);
......
...@@ -30,43 +30,60 @@ char *hdf5_f_counter_names[] = { ...@@ -30,43 +30,60 @@ char *hdf5_f_counter_names[] = {
}; };
#undef X #undef X
static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left, static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag); darshan_record_id* rec_id);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
static void darshan_log_print_hdf5_file(void *file_rec, static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type); char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs hdf5_logutils = struct darshan_mod_logutil_funcs hdf5_logutils =
{ {
.log_get_record = &darshan_log_get_hdf5_file, .log_get_record = &darshan_log_get_hdf5_file,
.log_put_record = &darshan_log_put_hdf5_file,
.log_print_record = &darshan_log_print_hdf5_file, .log_print_record = &darshan_log_print_hdf5_file,
}; };
static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left, static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag) darshan_record_id* rec_id)
{ {
struct darshan_hdf5_file *file;
int i; int i;
struct darshan_hdf5_file *file = (struct darshan_hdf5_file *) int ret;
(*hdf5_buf_p);
if(*bytes_left < sizeof(struct darshan_hdf5_file)) ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD, hdf5_buf,
sizeof(struct darshan_hdf5_file));
if(ret < 0)
return(-1); return(-1);
else if(ret < sizeof(struct darshan_hdf5_file))
if(byte_swap_flag) return(0);
else
{ {
/* swap bytes if necessary */ file = (struct darshan_hdf5_file *)hdf5_buf;
DARSHAN_BSWAP64(&file->f_id); if(fd->swap_flag)
DARSHAN_BSWAP64(&file->rank); {
for(i=0; i<HDF5_NUM_INDICES; i++) /* swap bytes if necessary */
DARSHAN_BSWAP64(&file->counters[i]); DARSHAN_BSWAP64(&file->f_id);
for(i=0; i<HDF5_F_NUM_INDICES; i++) DARSHAN_BSWAP64(&file->rank);
DARSHAN_BSWAP64(&file->fcounters[i]); for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
*rec_id = file->f_id;
return(1);
} }
}
/* update/set output variables */ static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf)
*file_rec = (void *)file; {
*rec_id = file->f_id; struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)hdf5_buf;
*hdf5_buf_p = (file + 1); /* increment input buf by size of file record */ int ret;
*bytes_left -= sizeof(struct darshan_hdf5_file);
ret = darshan_log_putmod(fd, DARSHAN_HDF5_MOD, file,
sizeof(struct darshan_hdf5_file));
if(ret < 0)
return(-1);
return(0); return(0);
} }
......
This diff is collapsed.
...@@ -16,9 +16,6 @@ ...@@ -16,9 +16,6 @@
#include "darshan-log-format.h" #include "darshan-log-format.h"
/* default to a buffer size of 4 MiB for compression/decompression */
#define DARSHAN_DEF_COMP_BUF_SZ (4*1024*1024)
/* TODO: can we refactor this def out of header? modules currently poke at swap_flag /* TODO: can we refactor this def out of header? modules currently poke at swap_flag
* directly, but other than that there's no reason for another module to know this * directly, but other than that there's no reason for another module to know this
* definition. * definition.
...@@ -26,14 +23,24 @@ ...@@ -26,14 +23,24 @@
struct darshan_fd_s struct darshan_fd_s
{ {
int fildes; int fildes;
int o_flags;
int64_t pos; int64_t pos;
enum darshan_comp_type comp_type; enum darshan_comp_type comp_type;
char logfile_path[PATH_MAX];
char version[8]; char version[8];
int swap_flag;