Commit 9f202697 authored by Shane Snyder's avatar Shane Snyder

update logutils to use new compression format

parent b2268e7b
......@@ -30,49 +30,48 @@ char *hdf5_f_counter_names[] = {
};
#undef X
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id);
static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs hdf5_logutils =
{
.log_get_record = &darshan_log_get_hdf5_file,
.log_print_record = &darshan_log_print_hdf5_file,
};
int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id)
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id)
{
int i;
int ret;
struct darshan_hdf5_file *file = NULL;
*rec_id = 0;
struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)
(*hdf5_buf_p);
file = malloc(sizeof(*file));
if(!file)
if(*bytes_left < sizeof(struct darshan_hdf5_file))
return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD,
(void *)file, sizeof(*file));
if(ret == 1)
if(fd->swap_flag)
{
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
return(ret);
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*hdf5_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_hdf5_file);
return(0);
}
void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
static void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
......
......@@ -15,9 +15,4 @@ extern char *hdf5_f_counter_names[];
extern struct darshan_mod_logutil_funcs hdf5_logutils;
int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif
This diff is collapsed.
......@@ -16,15 +16,21 @@
#include "darshan-log-format.h"
/* default to a decompression buffer size of 4 MiB */
#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
/* default to a buffer size of 4 MiB for compression/decompression */
#define DARSHAN_DEF_COMP_BUF_SZ (4*1024*1024)
/* TODO: can we refactor this def out of header? modules currently poke at swap_flag
* directly, but other than that there's no reason for another module to know this
* definition.
*/
struct darshan_fd_s
{
gzFile gzf;
int fildes;
int64_t pos;
char version[8];
int swap_flag;
char *exe_mnt_data;
struct darshan_log_map job_map;
struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
};
......@@ -40,8 +46,10 @@ struct darshan_mod_logutil_funcs
{
int (*log_get_record)(
darshan_fd fd,
void **rec_dat,
darshan_record_id *rec_id
void** mod_buf_p,
int* mod_bytes_left,
void** file_rec,
darshan_record_id* rec_id
);
void (*log_print_record)(
void *file_rec,
......@@ -72,7 +80,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts,
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash);
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash);
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
void *mod_buf, int mod_buf_sz);
void *mod_buf, int *mod_buf_sz);
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
void *mod_buf, int mod_buf_sz);
void darshan_log_close(darshan_fd file);
......
......@@ -30,49 +30,48 @@ char *mpiio_f_counter_names[] = {
};
#undef X
static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id);
static void darshan_log_print_mpiio_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs mpiio_logutils =
{
.log_get_record = &darshan_log_get_mpiio_file,
.log_print_record = &darshan_log_print_mpiio_file,
};
int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id)
static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id)
{
int i;
int ret;
struct darshan_mpiio_file *file = NULL;
*rec_id = 0;
struct darshan_mpiio_file *file = (struct darshan_mpiio_file *)
(*mpiio_buf_p);
file = malloc(sizeof(*file));
if(!file)
if(*bytes_left < sizeof(struct darshan_mpiio_file))
return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_getmod(fd, DARSHAN_MPIIO_MOD,
(void *)file, sizeof(*file));
if(ret == 1)
if(fd->swap_flag)
{
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<MPIIO_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<MPIIO_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<MPIIO_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<MPIIO_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
return(ret);
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*mpiio_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_mpiio_file);
return(0);
}
void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
static void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
......
......@@ -15,9 +15,4 @@ extern char *mpiio_f_counter_names[];
extern struct darshan_mod_logutil_funcs mpiio_logutils;
int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif
......@@ -25,12 +25,12 @@ int main(int argc, char **argv)
int ret;
int i, j;
char *filename;
char tmp_string[4096];
char tmp_string[4096] = {0};
darshan_fd fd;
struct darshan_header header;
struct darshan_job job;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref;
struct darshan_record_ref *ref, *tmp;
int mount_count;
char** mnt_pts;
char** fs_types;
......@@ -39,6 +39,8 @@ int main(int argc, char **argv)
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
int empty_mods = 0;
char *mod_buf;
int mod_buf_sz;
/* TODO: argument parsing */
assert(argc == 2);
......@@ -132,10 +134,10 @@ int main(int argc, char **argv)
}
/* print breakdown of each log file component's contribution to file size */
printf("\n# log file component sizes (decompressed)\n");
printf("\n# log file component sizes (compressed)\n");
printf("# -------------------------------------------------------\n");
printf("# header: %zu bytes\n", sizeof(struct darshan_header));
printf("# job data: %zu bytes\n", sizeof(struct darshan_job));
printf("# header: %zu bytes (uncompressed)\n", sizeof(struct darshan_header));
printf("# job data: %zu bytes\n", header.rec_map.off - sizeof(struct darshan_header));
printf("# record table: %zu bytes\n", header.rec_map.len);
for(i=0; i<DARSHAN_MAX_MODS; i++)
{
......@@ -154,20 +156,37 @@ int main(int argc, char **argv)
printf("# mount entry:\t%s\t%s\n", mnt_pts[i], fs_types[i]);
}
mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!mod_buf)
return(-1);
DARSHAN_PRINT_HEADER();
/* TODO: does each module print header of what each counter means??? */
for(i=0; i<DARSHAN_MAX_MODS; i++)
{
int mod_bytes_left;
void *mod_buf_p;
void *rec_p = NULL;
darshan_record_id rec_id;
/* skip modules not present in log file */
if(header.mod_map[i].len == 0)
memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* check each module for any data */
ret = darshan_log_getmod(fd, i, mod_buf, &mod_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: failed to get module %s data\n",
darshan_module_names[i]);
fflush(stderr);
darshan_log_close(fd);
return(-1);
}
else if(ret == 0)
{
/* skip modules not present in log file */
empty_mods++;
if(empty_mods == DARSHAN_MAX_MODS)
printf("# no module data available.\n");
continue;
}
......@@ -180,15 +199,26 @@ int main(int argc, char **argv)
}
/* this module has data to be parsed and printed */
/* TODO: do modules print header of what each counter means??? */
mod_buf_p = mod_buf;
mod_bytes_left = mod_buf_sz;
/* loop over each of this module's records and print them */
while ((ret = mod_logutils[i]->log_get_record(fd, &rec_p, &rec_id)) == 1)
while (mod_bytes_left > 0)
{
char *mnt_pt = NULL;
char *fs_type = NULL;
ret = mod_logutils[i]->log_get_record(fd, &mod_buf_p, &mod_bytes_left,
&rec_p, &rec_id);
if(ret < 0)
{
fprintf(stderr, "Error: failed to parse module %s data record\n",
darshan_module_names[i]);
fflush(stderr);
darshan_log_close(fd);
return(-1);
}
/* get the pathname for this record */
HASH_FIND(hlink, rec_hash, &rec_id, sizeof(darshan_record_id), ref);
assert(ref);
......@@ -211,19 +241,17 @@ int main(int argc, char **argv)
/* print the corresponding module data for this record */
mod_logutils[i]->log_print_record(rec_p, ref->rec.name,
mnt_pt, fs_type);
/* NOTE: we have to free the given file record data */
free(rec_p);
}
}
if(empty_mods == DARSHAN_MAX_MODS)
printf("# no module data available.\n");
if(ret < 0)
{
fprintf(stderr, "Error: failed to parse module %s data\n",
darshan_module_names[i]);
fflush(stderr);
darshan_log_close(fd);
return(-1);
}
/* free record hash data */
HASH_ITER(hlink, rec_hash, ref, tmp)
{
HASH_DELETE(hlink, rec_hash, ref);
free(ref->rec.name);
free(ref);
}
/* free mount info */
......@@ -238,6 +266,7 @@ int main(int argc, char **argv)
free(fs_types);
}
free(mod_buf);
darshan_log_close(fd);
return(0);
......
......@@ -30,49 +30,48 @@ char *pnetcdf_f_counter_names[] = {
};
#undef X
static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id);
static void darshan_log_print_pnetcdf_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs pnetcdf_logutils =
{
.log_get_record = &darshan_log_get_pnetcdf_file,
.log_print_record = &darshan_log_print_pnetcdf_file,
};
int darshan_log_get_pnetcdf_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id)
static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id)
{
int i;
int ret;
struct darshan_pnetcdf_file *file = NULL;
*rec_id = 0;
struct darshan_pnetcdf_file *file = (struct darshan_pnetcdf_file *)
(*pnetcdf_buf_p);
file = malloc(sizeof(*file));
if(!file)
if(*bytes_left < sizeof(struct darshan_pnetcdf_file))
return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_getmod(fd, DARSHAN_PNETCDF_MOD,
(void *)file, sizeof(*file));
if(ret == 1)
if(fd->swap_flag)
{
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<PNETCDF_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<PNETCDF_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<PNETCDF_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<PNETCDF_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
return(ret);
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*pnetcdf_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_pnetcdf_file);
return(0);
}
void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
static void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
......
......@@ -15,9 +15,4 @@ extern char *pnetcdf_f_counter_names[];
extern struct darshan_mod_logutil_funcs pnetcdf_logutils;
int darshan_log_get_pnetcdf_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif
......@@ -30,49 +30,49 @@ char *posix_f_counter_names[] = {
};
#undef X
static int darshan_log_get_posix_file(darshan_fd fd, void** psx_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id);
static void darshan_log_print_posix_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs posix_logutils =
{
.log_get_record = &darshan_log_get_posix_file,
.log_print_record = &darshan_log_print_posix_file,
};
int darshan_log_get_posix_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id)
static int darshan_log_get_posix_file(darshan_fd fd, void** psx_buf_p,
int* bytes_left, void **file_rec, darshan_record_id* rec_id)
{
int i;
int ret;
struct darshan_posix_file *file = NULL;
*rec_id = 0;
struct darshan_posix_file *file = (struct darshan_posix_file *)
(*psx_buf_p);
file = malloc(sizeof(*file));
if(!file)
if(*bytes_left < sizeof(struct darshan_posix_file))
return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_getmod(fd, DARSHAN_POSIX_MOD,
(void *)file, sizeof(*file));
if(ret == 1)
if(fd->swap_flag)
{
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<POSIX_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<POSIX_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->f_id);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<POSIX_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<POSIX_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
return(ret);
/* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*psx_buf_p = (file + 1); /* increment input buf by size of file record */
//((struct darshan_posix_file *)(*psx_buf_p))++;
*bytes_left -= sizeof(struct darshan_posix_file);
return(0);
}
void darshan_log_print_posix_file(void *file_rec, char *file_name,
static void darshan_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
......
......@@ -15,9 +15,4 @@ extern char *posix_f_counter_names[];
extern struct darshan_mod_logutil_funcs posix_logutils;
int darshan_log_get_posix_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment