Skip to content

GitLab

Commit 8659e6a3 authored by Shane Snyder's avatar Shane Snyder
Browse files

new implementation of darshan-logutils

parent 86d17440
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 983 additions and 852 deletions
darshan-util/darshan-analyzer.c
View file @ 8659e6a3
......@@ -35,18 +35,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
{
int ret;
darshan_fd file;
struct darshan_header header;
struct darshan_job job;
struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD];
struct darshan_posix_file *psx_buf, *psx_buf_p;
int psx_buf_sz, psx_buf_bytes_left;
struct darshan_posix_file *psx_rec;
struct darshan_posix_file psx_rec;
darshan_record_id rec_id;
int f_count;
double total_io_time;
double total_job_time;
assert(psx_mod);
memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
file = darshan_log_open(fname);
if (file == NULL)
......@@ -55,14 +53,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
return -1;
}
ret = darshan_log_getheader(file, &header);
if (ret < 0)
{
fprintf(stderr, "darshan_log_getheader() failed on file %s.\n", fname);
darshan_log_close(file);
return -1;
}
ret = darshan_log_getjob(file, &job);
if (ret < 0)
{
......@@ -71,49 +61,36 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
return -1;
}
psx_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
psx_buf = malloc(psx_buf_sz);
if (!psx_buf)
{
darshan_log_close(file);
return -1;
}
ret = darshan_log_getmod(file, DARSHAN_POSIX_MOD, (void *)psx_buf, &psx_buf_sz);
if (ret < 0)
{
fprintf(stderr, "darshan_log_getmod() failed on file %s.\n", fname);
darshan_log_close(file);
return -1;
}
f_count = 0;
total_io_time = 0.0;
psx_buf_bytes_left = psx_buf_sz;
psx_buf_p = psx_buf;
while(psx_buf_bytes_left)
while((ret = psx_mod->log_get_record(file, &psx_rec, &rec_id)) == 1)
{
ret = psx_mod->log_get_record((void **)&psx_buf_p, &psx_buf_bytes_left,
(void **)&psx_rec, &rec_id, file->swap_flag);
f_count += 1;
if (psx_rec->rank == -1)
if (psx_rec.rank == -1)
*used_shared = 1;
else
*used_fpp = 1;
total_io_time += (psx_rec->fcounters[POSIX_F_READ_TIME] +
psx_rec->fcounters[POSIX_F_WRITE_TIME] +
psx_rec->fcounters[POSIX_F_META_TIME]);
total_io_time += (psx_rec.fcounters[POSIX_F_READ_TIME] +
psx_rec.fcounters[POSIX_F_WRITE_TIME] +
psx_rec.fcounters[POSIX_F_META_TIME]);
memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
}
if (ret < 0)
{
fprintf(stderr, "Error: unable to read posix file record in log file %s.\n", fname);
darshan_log_close(file);
return -1;
}
if (header.mod_map[DARSHAN_MPIIO_MOD].len > 0)
if (file->mod_map[DARSHAN_MPIIO_MOD].len > 0)
*used_mpio += 1;
if (header.mod_map[DARSHAN_HDF5_MOD].len > 0)
if (file->mod_map[DARSHAN_HDF5_MOD].len > 0)
*used_hdf5 += 1;
if (header.mod_map[DARSHAN_PNETCDF_MOD].len > 0)
if (file->mod_map[DARSHAN_PNETCDF_MOD].len > 0)
*used_pnet += 1;
total_job_time = (double)job.end_time - (double)job.start_time;
......@@ -131,7 +108,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
*io_ratio = 0.0;
}
free(psx_buf);
darshan_log_close(file);
return 0;
......
darshan-util/darshan-bgq-logutils.c
View file @ 8659e6a3
......@@ -30,48 +30,65 @@ char *bgq_f_counter_names[] = {
};
#undef X
static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag);
static void darshan_log_print_bgq_file(void *file_rec,
static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
darshan_record_id* rec_id);
static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf);
static void darshan_log_print_bgq_rec(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs bgq_logutils =
{
.log_get_record = &darshan_log_get_bgq_file,
.log_print_record = &darshan_log_print_bgq_file,
.log_get_record = &darshan_log_get_bgq_rec,
.log_put_record = &darshan_log_put_bgq_rec,
.log_print_record = &darshan_log_print_bgq_rec,
};
static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag)
static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
darshan_record_id* rec_id)
{
struct darshan_bgq_record *rec;
int i;
struct darshan_bgq_record *file = (struct darshan_bgq_record *)
(*psx_buf_p);
int ret;
if(*bytes_left < sizeof(struct darshan_bgq_record))
ret = darshan_log_getmod(fd, DARSHAN_BGQ_MOD, bgq_buf,
sizeof(struct darshan_bgq_record));
if(ret < 0)
return(-1);
if(byte_swap_flag)
else if(ret < sizeof(struct darshan_bgq_record))
return(0);
else
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<BGQ_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<BGQ_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
rec = (struct darshan_bgq_record *)bgq_buf;
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&rec->f_id);
DARSHAN_BSWAP64(&rec->rank);
for(i=0; i<BGQ_NUM_INDICES; i++)
DARSHAN_BSWAP64(&rec->counters[i]);
for(i=0; i<BGQ_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&rec->fcounters[i]);
}
*rec_id = rec->f_id;
return(1);
}
}
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*psx_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_bgq_record);
static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf)
{
struct darshan_bgq_record *rec = (struct darshan_bgq_record *)bgq_buf;
int ret;
ret = darshan_log_putmod(fd, DARSHAN_BGQ_MOD, rec,
sizeof(struct darshan_bgq_record));
if(ret < 0)
return(-1);
return(0);
}
static void darshan_log_print_bgq_file(void *file_rec, char *file_name,
static void darshan_log_print_bgq_rec(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
......
darshan-util/darshan-convert.c
View file @ 8659e6a3
......@@ -19,6 +19,8 @@
#include "darshan-logutils.h"
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
int usage (char *exename)
......@@ -221,7 +223,6 @@ int main(int argc, char **argv)
int ret;
char *infile_name;
char *outfile_name;
struct darshan_header header;
struct darshan_job job;
char tmp_string[4096] = {0};
darshan_fd infile;
......@@ -232,8 +233,7 @@ int main(int argc, char **argv)
char** fs_types;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
char *mod_buf;
int mod_buf_sz;
char mod_buf[DEF_MOD_BUF_SIZE];
enum darshan_comp_type comp_type;
int bzip2;
int obfuscate;
......@@ -247,42 +247,23 @@ int main(int argc, char **argv)
infile = darshan_log_open(infile_name);
if(!infile)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", infile_name);
return(-1);
}
comp_type = bzip2 ? comp_type = DARSHAN_BZIP2_COMP : DARSHAN_ZLIB_COMP;
outfile = darshan_log_create(outfile_name, comp_type);
if(!outfile)
{
fprintf(stderr, "darshan_log_create() failed to create %s\n.", outfile_name);
darshan_log_close(infile);
return(-1);
}
/* read header from input file */
ret = darshan_log_getheader(infile, &header);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read header from input log file %s.\n", infile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
/* NOTE: we do not write the header to the output file until the end, as
* the mapping data stored in this structure may change in the conversion
* process (particularly, if we are converting between libz/bz2 compression)
*/
/* read job info */
ret = darshan_log_getjob(infile, &job);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read job information from log file.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
......@@ -293,7 +274,6 @@ int main(int argc, char **argv)
ret = darshan_log_putjob(outfile, &job);
if (ret < 0)
{
fprintf(stderr, "Error: unable to write job information to log file.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
......@@ -302,9 +282,9 @@ int main(int argc, char **argv)
ret = darshan_log_getexe(infile, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
......@@ -313,7 +293,6 @@ int main(int argc, char **argv)
ret = darshan_log_putexe(outfile, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
......@@ -322,16 +301,15 @@ int main(int argc, char **argv)
ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write mount information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
......@@ -340,9 +318,9 @@ int main(int argc, char **argv)
ret = darshan_log_gethash(infile, &rec_hash);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read darshan record hash.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
......@@ -355,104 +333,56 @@ int main(int argc, char **argv)
ret = darshan_log_puthash(outfile, rec_hash);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write darshan record hash.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!mod_buf)
return(-1);
/* loop over each module and convert it's data to the new format */
for(i=0; i<DARSHAN_MAX_MODS; i++)
{
int mod_bytes_left;
int mod_bytes_left_save;
void *mod_buf_p;
void *rec_p = NULL;
darshan_record_id rec_id;
memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* check each module for any data */
ret = darshan_log_getmod(infile, i, mod_buf, &mod_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: failed to get module %s data.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
else if(ret == 0)
{
/* skip modules not present in log file */
if(infile->mod_map[i].len == 0)
continue;
}
/* skip modules with no defined logutil handlers */
if(!mod_logutils[i])
else if(!mod_logutils[i])
{
fprintf(stderr, "Warning: no log utility handlers defined "
"for module %s, SKIPPING\n", darshan_module_names[i]);
"for module %s, SKIPPING.\n", darshan_module_names[i]);
continue;
}
/* we have module data to convert */
/* NOTE: it is necessary to iterate through each module's
* records to correct any endianness issues before writing
* this data back to file
*/
mod_bytes_left = mod_buf_sz;
mod_buf_p = mod_buf;
while(mod_bytes_left > 0)
{
mod_bytes_left_save = mod_bytes_left;
ret = mod_logutils[i]->log_get_record(&mod_buf_p, &mod_bytes_left,
&rec_p, &rec_id, infile->swap_flag);
if(ret < 0)
{
fprintf(stderr, "Error: failed to parse module %s data record\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
if(hash == rec_id)
{
mod_buf_p = rec_p;
mod_buf_sz = mod_bytes_left_save - mod_bytes_left;
break;
}
else if(mod_bytes_left == 0)
{
mod_buf_p = mod_buf;
}
}
ret = darshan_log_putmod(outfile, i, mod_buf_p, mod_buf_sz);
if(ret < 0)
ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id);
if(ret != 1)
{
fprintf(stderr, "Error: failed to put module %s data.\n",
fprintf(stderr, "Error: failed to parse the first %s module record.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
unlink(outfile_name);
return(-1);
}
}
free(mod_buf);
/* write header to output file */
ret = darshan_log_putheader(outfile);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
/* loop over each of the module's records and convert */
do
{
if(!hash || hash == rec_id)
{
ret = mod_logutils[i]->log_put_record(outfile, mod_buf);
if(ret < 0)
{
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
}
} while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1);
}
darshan_log_close(infile);
......
darshan-util/darshan-hdf5-logutils.c
View file @ 8659e6a3
......@@ -30,43 +30,60 @@ char *hdf5_f_counter_names[] = {
};
#undef X
static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag);
static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
darshan_record_id* rec_id);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs hdf5_logutils =
{
.log_get_record = &darshan_log_get_hdf5_file,
.log_put_record = &darshan_log_put_hdf5_file,
.log_print_record = &darshan_log_print_hdf5_file,
};
static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left,
void** file_rec, darshan_record_id* rec_id, int byte_swap_flag)
static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
darshan_record_id* rec_id)
{
struct darshan_hdf5_file *file;
int i;
struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)
(*hdf5_buf_p);
int ret;
if(*bytes_left < sizeof(struct darshan_hdf5_file))
ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD, hdf5_buf,
sizeof(struct darshan_hdf5_file));
if(ret < 0)
return(-1);
if(byte_swap_flag)
else if(ret < sizeof(struct darshan_hdf5_file))
return(0);
else
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
file = (struct darshan_hdf5_file *)hdf5_buf;
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
*rec_id = file->f_id;
return(1);
}
}
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*hdf5_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_hdf5_file);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf)
{
struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)hdf5_buf;
int ret;
ret = darshan_log_putmod(fd, DARSHAN_HDF5_MOD, file,
sizeof(struct darshan_hdf5_file));
if(ret < 0)
return(-1);
return(0);
}
......
darshan-util/darshan-logutils.c
View file @ 8659e6a3
This diff is collapsed.
darshan-util/darshan-logutils.h
View file @ 8659e6a3
......@@ -16,9 +16,6 @@
#include "darshan-log-format.h"
/* default to a buffer size of 4 MiB for compression/decompression */
#define DARSHAN_DEF_COMP_BUF_SZ (4*1024*1024)
/* TODO: can we refactor this def out of header? modules currently poke at swap_flag
* directly, but other than that there's no reason for another module to know this
* definition.
......@@ -26,14 +23,24 @@
struct darshan_fd_s
{
int fildes;
int o_flags;
int64_t pos;
enum darshan_comp_type comp_type;
char logfile_path[PATH_MAX];
char version[8];
int swap_flag;
char *exe_mnt_data;
struct darshan_log_map job_map;
struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
int err;
/* XXX */
void *dz_strm;
int dz_size;
unsigned char *dz_buf;
int dz_eor;
int dz_prev_reg_id;
};
typedef struct darshan_fd_s* darshan_fd;
......@@ -46,11 +53,13 @@ struct darshan_record_ref
struct darshan_mod_logutil_funcs
{
int (*log_get_record)(
void** mod_buf_p,
int* mod_bytes_left,
void** file_rec,
darshan_record_id* rec_id,
int byte_swap_flag
darshan_fd fd,
void* buf,
darshan_record_id* rec_id
);
int (*log_put_record)(
darshan_fd fd,
void *buf
);
void (*log_print_record)(
void *file_rec,
......@@ -70,8 +79,6 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS];
darshan_fd darshan_log_open(const char *name);
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type);
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header);
int darshan_log_putheader(darshan_fd fd);
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job);
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job);
int darshan_log_getexe(darshan_fd fd, char *buf);
......@@ -83,7 +90,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts,
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash);
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash);
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
void *mod_buf, int *mod_buf_sz);
void *buf, int len);
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
void *mod_buf, int mod_buf_sz);
void darshan_log_close(darshan_fd file);
......
darshan-util/darshan-mpiio-logutils.c
View file @ 8659e6a3
......@@ -30,43 +30,60 @@ char *mpiio_f_counter_names[] = {
};
#undef X