Commit 9f202697 authored by Shane Snyder's avatar Shane Snyder
Browse files

update logutils to use new compression format

parent b2268e7b
...@@ -30,49 +30,48 @@ char *hdf5_f_counter_names[] = { ...@@ -30,49 +30,48 @@ char *hdf5_f_counter_names[] = {
}; };
#undef X #undef X
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
int* bytes_left, void** file_rec, darshan_record_id* rec_id);
static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
struct darshan_mod_logutil_funcs hdf5_logutils = struct darshan_mod_logutil_funcs hdf5_logutils =
{ {
.log_get_record = &darshan_log_get_hdf5_file, .log_get_record = &darshan_log_get_hdf5_file,
.log_print_record = &darshan_log_print_hdf5_file, .log_print_record = &darshan_log_print_hdf5_file,
}; };
int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec, static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
darshan_record_id *rec_id) int* bytes_left, void** file_rec, darshan_record_id* rec_id)
{ {
int i; int i;
int ret; struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)
struct darshan_hdf5_file *file = NULL; (*hdf5_buf_p);
*rec_id = 0;
file = malloc(sizeof(*file)); if(*bytes_left < sizeof(struct darshan_hdf5_file))
if(!file)
return(-1); return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD, if(fd->swap_flag)
(void *)file, sizeof(*file));
if(ret == 1)
{ {
if(fd->swap_flag) /* swap bytes if necessary */
{ DARSHAN_BSWAP64(&file->f_id);
/* swap bytes if necessary */ DARSHAN_BSWAP64(&file->rank);
DARSHAN_BSWAP64(&file->f_id); for(i=0; i<HDF5_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->rank); DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<HDF5_NUM_INDICES; i++) for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]); DARSHAN_BSWAP64(&file->fcounters[i]);
for(i=0; i<HDF5_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
} }
return(ret); /* update/set output variables */
*file_rec = (void *)file;
*rec_id = file->f_id;
*hdf5_buf_p = (file + 1); /* increment input buf by size of file record */
*bytes_left -= sizeof(struct darshan_hdf5_file);
return(0);
} }
void darshan_log_print_hdf5_file(void *file_rec, char *file_name, static void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type) char *mnt_pt, char *fs_type)
{ {
int i; int i;
......
...@@ -15,9 +15,4 @@ extern char *hdf5_f_counter_names[]; ...@@ -15,9 +15,4 @@ extern char *hdf5_f_counter_names[];
extern struct darshan_mod_logutil_funcs hdf5_logutils; extern struct darshan_mod_logutil_funcs hdf5_logutils;
int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif #endif
...@@ -19,9 +19,13 @@ ...@@ -19,9 +19,13 @@
#include "darshan-logutils.h" #include "darshan-logutils.h"
/* TODO: for log reads, we need to make sure the header has been read prior */
static int darshan_log_seek(darshan_fd fd, off_t offset); static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len); static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len); static int darshan_log_write(darshan_fd fd, void *buf, int len);
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
char* decomp_buf, int* inout_decomp_buf_sz);
/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */ /* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] = struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
...@@ -52,20 +56,34 @@ struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] = ...@@ -52,20 +56,34 @@ struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
*/ */
darshan_fd darshan_log_open(const char *name, const char *mode) darshan_fd darshan_log_open(const char *name, const char *mode)
{ {
int o_flags;
darshan_fd tmp_fd; darshan_fd tmp_fd;
/* we only allows "w" or "r" modes, nothing fancy */ /* we only allow "w" or "r" modes, nothing fancy */
assert(strlen(mode) == 1); assert(strlen(mode) == 1);
assert(mode[0] == 'r' || mode[0] == 'w'); assert(mode[0] == 'r' || mode[0] == 'w');
if(mode[0] == 'r')
{
o_flags = O_RDONLY;
}
else if (mode[0] == 'w')
{
/* when writing, we create the log file making sure not to overwrite
* an existing log
*/
o_flags = O_WRONLY | O_CREAT | O_EXCL;
}
tmp_fd = malloc(sizeof(*tmp_fd)); tmp_fd = malloc(sizeof(*tmp_fd));
if(!tmp_fd) if(!tmp_fd)
return(NULL); return(NULL);
memset(tmp_fd, 0, sizeof(*tmp_fd)); memset(tmp_fd, 0, sizeof(*tmp_fd));
tmp_fd->gzf = gzopen(name, mode); tmp_fd->fildes = open(name, o_flags);
if(!tmp_fd->gzf) if(tmp_fd->fildes < 0)
{ {
perror("darshan_log_open: ");
free(tmp_fd); free(tmp_fd);
tmp_fd = NULL; tmp_fd = NULL;
} }
...@@ -77,6 +95,7 @@ darshan_fd darshan_log_open(const char *name, const char *mode) ...@@ -77,6 +95,7 @@ darshan_fd darshan_log_open(const char *name, const char *mode)
* *
* read the header of the darshan log and set internal data structures * read the header of the darshan log and set internal data structures
* NOTE: this function must be called before reading other portions of the log * NOTE: this function must be called before reading other portions of the log
* NOTE: this is the only portion of the darshan log which is uncompressed
* *
* returns 0 on success, -1 on failure * returns 0 on success, -1 on failure
*/ */
...@@ -92,7 +111,7 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header) ...@@ -92,7 +111,7 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
return(-1); return(-1);
} }
/* read header from log file */ /* read uncompressed header from log file */
ret = darshan_log_read(fd, header, sizeof(*header)); ret = darshan_log_read(fd, header, sizeof(*header));
if(ret != sizeof(*header)) if(ret != sizeof(*header))
{ {
...@@ -134,12 +153,15 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header) ...@@ -134,12 +153,15 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
} }
/* save the mapping of data within log file to this file descriptor */ /* save the mapping of data within log file to this file descriptor */
fd->job_map.off = sizeof(struct darshan_header);
fd->job_map.len = header->rec_map.off - fd->job_map.off;
memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map)); memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map)); memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
return(0); return(0);
} }
#if 0
/* darshan_log_putheader() /* darshan_log_putheader()
* *
* write a darshan header to log file * write a darshan header to log file
...@@ -171,6 +193,7 @@ int darshan_log_putheader(darshan_fd fd, struct darshan_header *header) ...@@ -171,6 +193,7 @@ int darshan_log_putheader(darshan_fd fd, struct darshan_header *header)
return(0); return(0);
} }
#endif
/* darshan_log_getjob() /* darshan_log_getjob()
* *
...@@ -180,22 +203,45 @@ int darshan_log_putheader(darshan_fd fd, struct darshan_header *header) ...@@ -180,22 +203,45 @@ int darshan_log_putheader(darshan_fd fd, struct darshan_header *header)
*/ */
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job) int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
{ {
char *comp_buf;
char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
int ret; int ret;
ret = darshan_log_seek(fd, sizeof(struct darshan_header)); /* allocate buffer to store compressed job info */
comp_buf = malloc(fd->job_map.len);
if(!comp_buf)
return(-1);
ret = darshan_log_seek(fd, fd->job_map.off);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to seek in darshan log file.\n"); fprintf(stderr, "Error: unable to seek in darshan log file.\n");
free(comp_buf);
return(-1); return(-1);
} }
/* read the job data from the log file */ /* read the compressed job data from the log file */
ret = darshan_log_read(fd, job, sizeof(*job)); ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
if(ret != sizeof(*job)) if(ret != fd->job_map.len)
{ {
fprintf(stderr, "Error: failed to read darshan log file job data.\n"); fprintf(stderr, "Error: failed to read darshan log file job data.\n");
free(comp_buf);
return(-1);
}
/* decompress the job data */
ret = darshan_decompress_buf(comp_buf, fd->job_map.len, job_buf, &job_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: failed to decompress darshan job data.\n");
free(comp_buf);
return(-1); return(-1);
} }
assert(job_buf_sz == DARSHAN_JOB_RECORD_SIZE);
free(comp_buf);
memcpy(job, job_buf, sizeof(*job));
if(fd->swap_flag) if(fd->swap_flag)
{ {
...@@ -207,9 +253,17 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job) ...@@ -207,9 +253,17 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
DARSHAN_BSWAP64(&job->jobid); DARSHAN_BSWAP64(&job->jobid);
} }
/* save trailing exe & mount information, so it can be retrieved later */
if(!fd->exe_mnt_data)
fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
if(!fd->exe_mnt_data)
return(-1);
memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
return(0); return(0);
} }
#if 0
/* darshan_log_putjob() /* darshan_log_putjob()
* *
* write job level metadat to darshan log file * write job level metadat to darshan log file
...@@ -253,6 +307,7 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job) ...@@ -253,6 +307,7 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
return(0); return(0);
} }
#endif
/* darshan_log_getexe() /* darshan_log_getexe()
* *
...@@ -262,33 +317,30 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job) ...@@ -262,33 +317,30 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
*/ */
int darshan_log_getexe(darshan_fd fd, char *buf) int darshan_log_getexe(darshan_fd fd, char *buf)
{ {
int tmp_off = sizeof(struct darshan_header) + sizeof(struct darshan_job);
int ret;
char *newline; char *newline;
int ret;
ret = darshan_log_seek(fd, tmp_off); /* if the exe/mount data has not been saved yet, read in the job info */
if(ret < 0) if(!fd->exe_mnt_data)
{ {
fprintf(stderr, "Error: unable to seek in darshan log file.\n"); struct darshan_job job;
return(-1); ret = darshan_log_getjob(fd, &job);
}
/* read the trailing exe data from the darshan log */ if(ret < 0 || !fd->exe_mnt_data)
ret = darshan_log_read(fd, buf, DARSHAN_EXE_LEN+1); return(-1);
if(ret != DARSHAN_EXE_LEN+1)
{
fprintf(stderr, "Error: failed to read exe string from darshan log file.\n");
return(-1);
} }
/* mount info is stored after the exe string, so truncate there */ /* exe string is located before the first line break */
newline = strchr(buf, '\n'); newline = strchr(fd->exe_mnt_data, '\n');
/* copy over the exe string */
if(newline) if(newline)
*newline = '\0'; memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
return (0); return (0);
} }
#if 0
/* darshan_log_putexe() /* darshan_log_putexe()
* *
* wrties the application exe name to darshan log file * wrties the application exe name to darshan log file
...@@ -319,6 +371,7 @@ int darshan_log_putexe(darshan_fd fd, char *buf) ...@@ -319,6 +371,7 @@ int darshan_log_putexe(darshan_fd fd, char *buf)
return(0); return(0);
} }
#endif
/* darshan_log_getmounts() /* darshan_log_getmounts()
* *
...@@ -331,30 +384,23 @@ int darshan_log_putexe(darshan_fd fd, char *buf) ...@@ -331,30 +384,23 @@ int darshan_log_putexe(darshan_fd fd, char *buf)
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts, int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
char*** fs_types, int* count) char*** fs_types, int* count)
{ {
int tmp_off = sizeof(struct darshan_header) + sizeof(struct darshan_job);
int ret;
char *pos; char *pos;
int array_index = 0; int array_index = 0;
char buf[DARSHAN_EXE_LEN+1]; int ret;
ret = darshan_log_seek(fd, tmp_off); /* if the exe/mount data has not been saved yet, read in the job info */
if(ret < 0) if(!fd->exe_mnt_data)
{ {
fprintf(stderr, "Error: unable to seek in darshan log file.\n"); struct darshan_job job;
return(-1); ret = darshan_log_getjob(fd, &job);
}
/* read the trailing mount data from the darshan log */ if(ret < 0 || !fd->exe_mnt_data)
ret = darshan_log_read(fd, buf, DARSHAN_EXE_LEN+1); return(-1);
if(ret != DARSHAN_EXE_LEN+1)
{
fprintf(stderr, "Error: failed to read mount info from darshan log file.\n");
return(-1);
} }
/* count entries */ /* count entries */
*count = 0; *count = 0;
pos = buf; pos = fd->exe_mnt_data;
while((pos = strchr(pos, '\n')) != NULL) while((pos = strchr(pos, '\n')) != NULL)
{ {
pos++; pos++;
...@@ -376,7 +422,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts, ...@@ -376,7 +422,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
/* work backwards through the table and parse each line (except for /* work backwards through the table and parse each line (except for
* first, which holds command line information) * first, which holds command line information)
*/ */
while((pos = strrchr(buf, '\n')) != NULL) while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
{ {
/* overestimate string lengths */ /* overestimate string lengths */
(*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN); (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
...@@ -399,6 +445,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts, ...@@ -399,6 +445,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
return(0); return(0);
} }
#if 0
/* darshan_log_putmounts() /* darshan_log_putmounts()
* *
* writes mount information to the darshan log file * writes mount information to the darshan log file
...@@ -436,6 +483,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co ...@@ -436,6 +483,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co
return(0); return(0);
} }
#endif
/* darshan_log_gethash() /* darshan_log_gethash()
* *
...@@ -445,8 +493,9 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co ...@@ -445,8 +493,9 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co
*/ */
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash) int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
{ {
char *comp_buf;
char *hash_buf; char *hash_buf;
int hash_buf_sz = fd->rec_map.len; int hash_buf_sz;
char *buf_ptr; char *buf_ptr;
darshan_record_id *rec_id_ptr; darshan_record_id *rec_id_ptr;
uint32_t *path_len_ptr; uint32_t *path_len_ptr;
...@@ -454,24 +503,43 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash) ...@@ -454,24 +503,43 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
struct darshan_record_ref *ref; struct darshan_record_ref *ref;
int ret; int ret;
/* allocate buffers to store compressed and decompressed record
* hash data
*/
comp_buf = malloc(fd->rec_map.len);
hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!comp_buf || !hash_buf)
return(-1);
hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
ret = darshan_log_seek(fd, fd->rec_map.off); ret = darshan_log_seek(fd, fd->rec_map.off);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to seek in darshan log file.\n"); fprintf(stderr, "Error: unable to seek in darshan log file.\n");
free(comp_buf);
return(-1); return(-1);
} }
hash_buf = malloc(hash_buf_sz);
if(!hash_buf)
return(-1);
/* read the record hash from the log file */ /* read the record hash from the log file */
ret = darshan_log_read(fd, hash_buf, fd->rec_map.len); ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
if(ret != fd->rec_map.len) if(ret != fd->rec_map.len)
{ {
fprintf(stderr, "Error: failed to read record hash from darshan log file.\n"); fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
free(comp_buf);
return(-1);
}
/* decompress the record hash buffer */
ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
hash_buf, &hash_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: unable to decompress darshan job data.\n");
free(comp_buf);
return(-1); return(-1);
} }
free(comp_buf);
buf_ptr = hash_buf; buf_ptr = hash_buf;
while(buf_ptr < (hash_buf + hash_buf_sz)) while(buf_ptr < (hash_buf + hash_buf_sz))
...@@ -519,9 +587,11 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash) ...@@ -519,9 +587,11 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
} }
} }
free(hash_buf);
return(0); return(0);
} }
#if 0
/* darshan_log_puthash() /* darshan_log_puthash()
* *
* writes the hash table of records to the darshan log file * writes the hash table of records to the darshan log file
...@@ -609,60 +679,66 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash) ...@@ -609,60 +679,66 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
return(0); return(0);
} }
#endif
/* darshan_log_getmod() /* darshan_log_getmod()
* *
* reads a chunk of data for a corresponding darshan instrumentation module * returns 1 on successful read of module data, 0 on no data, -1 on failure
* NOTE: mod_buf_sz is the uncompressed size of data we wish to read for the given
* module. chunks of data can be read sequentially for a given module by
* repeatedly calling this function with the desired chunk sizes.
*
* returns 1 on successful read of data, 0 on end of module region, -1 on failure
*/ */