Skip to content

GitLab

Commit 999ee9c8 authored by Shane Snyder's avatar Shane Snyder
Browse files

darshan-util edits for var. length name records

parent d6dad7f3
Hide whitespace changes
Inline Side-by-side
Showing with 222 additions and 236 deletions
darshan-util/Makefile.in
View file @ 999ee9c8
......@@ -159,7 +159,7 @@ endif
clean::
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-parser darshan-merge jenkins-hash-gen
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
......
darshan-util/darshan-analyzer.c
View file @ 999ee9c8
......@@ -52,10 +52,10 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
return -1;
}
ret = darshan_log_getjob(file, &job);
ret = darshan_log_get_job(file, &job);
if (ret < 0)
{
fprintf(stderr, "darshan_log_getjob() failed on file %s.\n", fname);
fprintf(stderr, "darshan_log_get_job() failed on file %s.\n", fname);
darshan_log_close(file);
return -1;
}
......
darshan-util/darshan-bgq-logutils.c
View file @ 999ee9c8
......@@ -55,7 +55,7 @@ static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf)
int i;
int ret;
ret = darshan_log_getmod(fd, DARSHAN_BGQ_MOD, bgq_buf,
ret = darshan_log_get_mod(fd, DARSHAN_BGQ_MOD, bgq_buf,
sizeof(struct darshan_bgq_record));
if(ret < 0)
return(-1);
......@@ -84,7 +84,7 @@ static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf, int ver)
struct darshan_bgq_record *rec = (struct darshan_bgq_record *)bgq_buf;
int ret;
ret = darshan_log_putmod(fd, DARSHAN_BGQ_MOD, rec,
ret = darshan_log_put_mod(fd, DARSHAN_BGQ_MOD, rec,
sizeof(struct darshan_bgq_record), ver);
if(ret < 0)
return(-1);
......
darshan-util/darshan-convert.c
View file @ 999ee9c8
......@@ -140,21 +140,25 @@ void obfuscate_exe(int key, char *exe)
return;
}
void obfuscate_filenames(int key, struct darshan_record_ref *rec_hash)
void obfuscate_filenames(int key, struct darshan_name_record_ref *name_hash)
{
struct darshan_record_ref *ref, *tmp;
struct darshan_name_record_ref *ref, *tmp;
uint32_t hashed;
char tmp_string[128] = {0};
darshan_record_id tmp_id;
HASH_ITER(hlink, rec_hash, ref, tmp)
HASH_ITER(hlink, name_hash, ref, tmp)
{
hashed = darshan_hashlittle(ref->name, strlen(ref->name), key);
tmp_id = ref->name_record->id;
hashed = darshan_hashlittle(ref->name_record->name,
strlen(ref->name_record->name), key);
sprintf(tmp_string, "%u", hashed);
free(ref->name);
ref->name = malloc(strlen(tmp_string) + 1);
assert(ref->name);
memcpy(ref->name, tmp_string, strlen(tmp_string));
ref->name[strlen(tmp_string)] = '\0';
free(ref->name_record);
ref->name_record = malloc(sizeof(struct darshan_name_record) +
strlen(tmp_string));
assert(ref->name_record);
ref->name_record->id = tmp_id;
strcpy(ref->name_record->name, tmp_string);
}
return;
......@@ -202,16 +206,17 @@ void add_annotation (char *annotation,
return;
}
static void remove_hash_recs(struct darshan_record_ref **rec_hash, darshan_record_id hash)
static void remove_hash_recs(struct darshan_name_record_ref **name_hash,
darshan_record_id hash)
{
struct darshan_record_ref *ref, *tmp;
struct darshan_name_record_ref *ref, *tmp;
HASH_ITER(hlink, *rec_hash, ref, tmp)
HASH_ITER(hlink, *name_hash, ref, tmp)
{
if(ref->id != hash)
if(ref->name_record->id != hash)
{
HASH_DELETE(hlink, *rec_hash, ref);
free(ref->name);
HASH_DELETE(hlink, *name_hash, ref);
free(ref->name_record);
free(ref);
}
}
......@@ -232,8 +237,8 @@ int main(int argc, char **argv)
int mount_count;
char** mnt_pts;
char** fs_types;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
struct darshan_name_record_ref *name_hash = NULL;
struct darshan_name_record_ref *ref, *tmp;
char mod_buf[DEF_MOD_BUF_SIZE];
enum darshan_comp_type comp_type;
int bzip2;
......@@ -259,7 +264,7 @@ int main(int argc, char **argv)
}
/* read job info */
ret = darshan_log_getjob(infile, &job);
ret = darshan_log_get_job(infile, &job);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -272,7 +277,7 @@ int main(int argc, char **argv)
if (obfuscate) obfuscate_job(key, &job);
if (annotation) add_annotation(annotation, &job);
ret = darshan_log_putjob(outfile, &job);
ret = darshan_log_put_job(outfile, &job);
if (ret < 0)
{
darshan_log_close(infile);
......@@ -280,7 +285,7 @@ int main(int argc, char **argv)
return(-1);
}
ret = darshan_log_getexe(infile, tmp_string);
ret = darshan_log_get_exe(infile, tmp_string);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -291,7 +296,7 @@ int main(int argc, char **argv)
if (obfuscate) obfuscate_exe(key, tmp_string);
ret = darshan_log_putexe(outfile, tmp_string);
ret = darshan_log_put_exe(outfile, tmp_string);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -299,7 +304,7 @@ int main(int argc, char **argv)
return(-1);
}
ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
ret = darshan_log_get_mounts(infile, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -308,7 +313,7 @@ int main(int argc, char **argv)
return(-1);
}
ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
ret = darshan_log_put_mounts(outfile, mnt_pts, fs_types, mount_count);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -316,7 +321,7 @@ int main(int argc, char **argv)
return(-1);
}
ret = darshan_log_gethash(infile, &rec_hash);
ret = darshan_log_get_namehash(infile, &name_hash);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -328,10 +333,10 @@ int main(int argc, char **argv)
/* NOTE: obfuscating filepaths breaks the ability to map files
* to the corresponding FS & mount info maintained by darshan
*/
if(obfuscate) obfuscate_filenames(key, rec_hash);
if(hash) remove_hash_recs(&rec_hash, hash);
if(obfuscate) obfuscate_filenames(key, name_hash);
if(hash) remove_hash_recs(&name_hash, hash);
ret = darshan_log_puthash(outfile, rec_hash);
ret = darshan_log_put_namehash(outfile, name_hash);
if(ret < 0)
{
darshan_log_close(infile);
......@@ -402,10 +407,10 @@ int main(int argc, char **argv)
free(fs_types);
}
HASH_ITER(hlink, rec_hash, ref, tmp)
HASH_ITER(hlink, name_hash, ref, tmp)
{
HASH_DELETE(hlink, rec_hash, ref);
free(ref->name);
HASH_DELETE(hlink, name_hash, ref);
free(ref->name_record);
free(ref);
}
......
darshan-util/darshan-diff.c
View file @ 999ee9c8
......@@ -53,8 +53,8 @@ int main(int argc, char *argv[])
darshan_fd file1, file2;
struct darshan_job job1, job2;
char exe1[4096], exe2[4096];
struct darshan_record_ref *name_hash1 = NULL, *name_hash2 = NULL;
struct darshan_record_ref *name_ref1, *name_ref2;
struct darshan_name_record_ref *name_hash1 = NULL, *name_hash2 = NULL;
struct darshan_name_record_ref *name_ref1, *name_ref2;
struct darshan_file_record_ref *rec_hash1 = NULL, *rec_hash2 = NULL;
struct darshan_file_record_ref *rec_ref1, *rec_ref2, *rec_tmp;
struct darshan_mod_record_ref *mod_rec1, *mod_rec2;
......@@ -89,7 +89,7 @@ int main(int argc, char *argv[])
}
/* get job data for each log file */
ret = darshan_log_getjob(file1, &job1);
ret = darshan_log_get_job(file1, &job1);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -98,7 +98,7 @@ int main(int argc, char *argv[])
return(-1);
}
ret = darshan_log_getjob(file2, &job2);
ret = darshan_log_get_job(file2, &job2);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -108,7 +108,7 @@ int main(int argc, char *argv[])
}
/* get exe string for each log file */
ret = darshan_log_getexe(file1, exe1);
ret = darshan_log_get_exe(file1, exe1);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -117,7 +117,7 @@ int main(int argc, char *argv[])
return(-1);
}
ret = darshan_log_getexe(file2, exe2);
ret = darshan_log_get_exe(file2, exe2);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -144,7 +144,7 @@ int main(int argc, char *argv[])
(int64_t)(job2.end_time - job2.start_time + 1));
/* get hash of record ids to file names for each log */
ret = darshan_log_gethash(file1, &name_hash1);
ret = darshan_log_get_namehash(file1, &name_hash1);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -153,7 +153,7 @@ int main(int argc, char *argv[])
return(-1);
}
ret = darshan_log_gethash(file2, &name_hash2);
ret = darshan_log_get_namehash(file2, &name_hash2);
if(ret < 0)
{
darshan_log_close(file1);
......@@ -246,14 +246,14 @@ int main(int argc, char *argv[])
HASH_FIND(hlink, name_hash1, &(base_rec1->id),
sizeof(darshan_record_id), name_ref1);
assert(name_ref1);
file_name1 = name_ref1->name;
file_name1 = name_ref1->name_record->name;
}
if(mod_buf2)
{
HASH_FIND(hlink, name_hash2, &(base_rec2->id),
sizeof(darshan_record_id), name_ref2);
assert(name_ref2);
file_name2 = name_ref2->name;
file_name2 = name_ref2->name_record->name;
}
mod_logutils[i]->log_print_diff(mod_buf1, file_name1, mod_buf2, file_name2);
......@@ -314,7 +314,7 @@ int main(int argc, char *argv[])
HASH_FIND(hlink, name_hash2, &(base_rec2->id),
sizeof(darshan_record_id), name_ref2);
assert(name_ref2);
file_name2 = name_ref2->name;
file_name2 = name_ref2->name_record->name;
mod_logutils[i]->log_print_diff(NULL, NULL, mod_rec2->mod_dat, file_name2);
......@@ -340,13 +340,13 @@ int main(int argc, char *argv[])
HASH_ITER(hlink, name_hash1, name_ref1, name_ref2)
{
HASH_DELETE(hlink, name_hash1, name_ref1);
free(name_ref1->name);
free(name_ref1->name_record);
free(name_ref1);
}
HASH_ITER(hlink, name_hash2, name_ref2, name_ref1)
{
HASH_DELETE(hlink, name_hash2, name_ref2);
free(name_ref2->name);
free(name_ref2->name_record);
free(name_ref2);
}
......
darshan-util/darshan-hdf5-logutils.c
View file @ 999ee9c8
......@@ -55,7 +55,7 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf)
int i;
int ret;
ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD, hdf5_buf,
ret = darshan_log_get_mod(fd, DARSHAN_HDF5_MOD, hdf5_buf,
sizeof(struct darshan_hdf5_file));
if(ret < 0)
return(-1);
......@@ -84,7 +84,7 @@ static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf, int ver)
struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)hdf5_buf;
int ret;
ret = darshan_log_putmod(fd, DARSHAN_HDF5_MOD, file,
ret = darshan_log_put_mod(fd, DARSHAN_HDF5_MOD, file,
sizeof(struct darshan_hdf5_file), ver);
if(ret < 0)
return(-1);
......
darshan-util/darshan-logutils.c
View file @ 999ee9c8
......@@ -27,7 +27,7 @@
*/
#define DARSHAN_HEADER_REGION_ID (-3)
#define DARSHAN_JOB_REGION_ID (-2)
#define DARSHAN_REC_MAP_REGION_ID (-1)
#define DARSHAN_NAME_MAP_REGION_ID (-1)
struct darshan_dz_state
{
......@@ -66,8 +66,8 @@ struct darshan_fd_int_state
struct darshan_dz_state dz;
};
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
......@@ -136,7 +136,7 @@ darshan_fd darshan_log_open(const char *name)
strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
/* read the header from the log file to init fd data structures */
ret = darshan_log_getheader(tmp_fd);
ret = darshan_log_get_header(tmp_fd);
if(ret < 0)
{
fprintf(stderr, "Error: failed to read darshan log file header.\n");
......@@ -229,13 +229,13 @@ darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type
return(tmp_fd);
}
/* darshan_log_getjob()
/* darshan_log_get_job()
*
* read job level metadata from the darshan log file
*
* returns 0 on success, -1 on failure
*/
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
{
struct darshan_fd_int_state *state = fd->state;
char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
......@@ -275,13 +275,13 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
return(0);
}
/* darshan_log_putjob()
/* darshan_log_put_job()
*
* write job level metadata to darshan log file
*
* returns 0 on success, -1 on failure
*/
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
{
struct darshan_fd_int_state *state = fd->state;
struct darshan_job job_copy;
......@@ -316,13 +316,13 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
return(0);
}
/* darshan_log_getexe()
/* darshan_log_get_exe()
*
* reads the application exe name from darshan log file
*
* returns 0 on success, -1 on failure
*/
int darshan_log_getexe(darshan_fd fd, char *buf)
int darshan_log_get_exe(darshan_fd fd, char *buf)
{
struct darshan_fd_int_state *state = fd->state;
char *newline;
......@@ -334,7 +334,7 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
if(!(state->exe_mnt_data))
{
struct darshan_job job;
ret = darshan_log_getjob(fd, &job);
ret = darshan_log_get_job(fd, &job);
if(ret < 0 || !(state->exe_mnt_data))
return(-1);
......@@ -350,7 +350,7 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
return (0);
}
/* darshan_log_putexe()
/* darshan_log_put_exe()
*
* wrties the application exe name to darshan log file
* NOTE: this needs to be called immediately following put_job as it
......@@ -359,7 +359,7 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
*
* returns 0 on success, -1 on failure
*/
int darshan_log_putexe(darshan_fd fd, char *buf)
int darshan_log_put_exe(darshan_fd fd, char *buf)
{
struct darshan_fd_int_state *state = fd->state;
int len = strlen(buf);
......@@ -378,7 +378,7 @@ int darshan_log_putexe(darshan_fd fd, char *buf)
return(0);
}
/* darshan_log_getmounts()
/* darshan_log_get_mounts()
*
* retrieves mount table information from the log. Note that mnt_pts and
* fs_types are arrays that will be allocated by the function and must be
......@@ -386,7 +386,7 @@ int darshan_log_putexe(darshan_fd fd, char *buf)
*
* returns 0 on success, -1 on failure
*/
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
int darshan_log_get_mounts(darshan_fd fd, char*** mnt_pts,
char*** fs_types, int* count)
{
struct darshan_fd_int_state *state = fd->state;
......@@ -400,7 +400,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
if(!(state->exe_mnt_data))
{
struct darshan_job job;
ret = darshan_log_getjob(fd, &job);
ret = darshan_log_get_job(fd, &job);
if(ret < 0 || !(state->exe_mnt_data))
return(-1);
......@@ -453,7 +453,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
return(0);
}
/* darshan_log_putmounts()
/* darshan_log_put_mounts()
*
* writes mount information to the darshan log file
* NOTE: this function call should follow immediately after the call
......@@ -462,7 +462,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
*
* returns 0 on success, -1 on failure
*/
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
int darshan_log_put_mounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
struct darshan_fd_int_state *state = fd->state;
int i;
......@@ -496,122 +496,112 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co
return(0);
}
/* darshan_log_gethash()
/* darshan_log_get_namehash()
*
* read the hash of records from the darshan log file
* read the hash of name records from the darshan log file
*
* returns 0 on success, -1 on failure
*/
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
{
struct darshan_fd_int_state *state = fd->state;
char *hash_buf;
int hash_buf_sz;
char *buf_ptr;
darshan_record_id *rec_id_ptr;
char *path_ptr;
char *name_rec_buf;
char *tmp_p;
struct darshan_record_ref *ref;
int name_rec_buf_sz;
int read;
int read_req_sz;
struct darshan_name_record_ref *ref;
struct darshan_name_record *name_rec;
int buf_rem = 0;
int rec_len;
assert(state);
/* just return if there is no record mapping data */
if(fd->rec_map.len == 0)
/* just return if there is no name record mapping data */
if(fd->name_map.len == 0)
{
*hash = NULL;
return(0);
}
/* default to hash buffer twice as big as default compression buf */
hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
if(!hash_buf)
/* default to buffer twice as big as default compression buf */
name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
name_rec_buf = malloc(name_rec_buf_sz);
if(!name_rec_buf)
return(-1);
memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
memset(name_rec_buf, 0, name_rec_buf_sz);
do
{
/* read chunks of the darshan record id -> file name mapping from log file,
/* read chunks of the darshan record id -> name mapping from log file,
* constructing a hash table in the process
*/
read_req_sz = hash_buf_sz - buf_rem;
read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
hash_buf + buf_rem, read_req_sz);
read_req_sz = name_rec_buf_sz - buf_rem;
read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
name_rec_buf + buf_rem, read_req_sz);
if(read < 0)
{
fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
free(hash_buf);
fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
free(name_rec_buf);
return(-1);
}
buf_rem += read;
/* work through the hash buffer -- deserialize the mapping data and
/* work through the name record buffer -- deserialize the mapping data and
* add to the output hash table
* NOTE: these mapping pairs are variable in length, so we have to be able
* to handle incomplete mappings temporarily here
*/
buf_ptr = hash_buf;
buf_rem += read;
while(buf_rem > (sizeof(darshan_record_id) + 1))
name_rec = (struct darshan_name_record *)name_rec_buf;
while(buf_rem > sizeof(darshan_record_id) + 1)
{
tmp_p = buf_ptr + sizeof(darshan_record_id);
while(tmp_p < (buf_ptr + buf_rem))
if(strnlen(name_rec->name, buf_rem - sizeof(darshan_record_id)) ==
(buf_rem - sizeof(darshan_record_id)))
{
/* look for terminating null character for record name */
if(*tmp_p == '\0')
break;
tmp_p++;
}
if(*tmp_p != '\0')
/* if this record name's terminating null character is not
* present, we need to read more of the buffer before continuing
*/
break;
/* get pointers for each field of this darshan record */
/* NOTE: darshan record hash serialization method:
* ... darshan_record_id | path '\0' ...
*/
rec_id_ptr = (darshan_record_id *)buf_ptr;
buf_ptr += sizeof(darshan_record_id);
path_ptr = (char *)buf_ptr;
}
if(fd->swap_flag)
{
/* we need to sort out endianness issues before deserializing */
DARSHAN_BSWAP64(rec_id_ptr);
DARSHAN_BSWAP64(&(name_rec->id));
}
HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
if(!ref)
{
rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
ref = malloc(sizeof(*ref));
if(!ref)
{
free(hash_buf);
free(name_rec_buf);
return(-1);
}
ref->name = malloc(strlen(path_ptr) + 1);
if(!ref->name)
ref->name_record = malloc(rec_len);
if(!ref->name_record)
{
free(ref);
free(hash_buf);
free(name_rec_buf);
return(-1);
}
/* set the fields for this record */
ref->id = *rec_id_ptr;
strcpy(ref->name, path_ptr);
/* copy the name record over from the hash buffer */
memcpy(ref->name_record, name_rec, rec_len);
/* add this record to the hash */
HASH_ADD(hlink, *hash, id, sizeof(darshan_record_id), ref);
HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
}