Skip to content

GitLab

Commit fcfcfe8a authored by Cong Xu's avatar Cong Xu
Browse files

Implement a separate dxt parser for parsing dxt logs 


Signed-off-by: Cong Xu's avatarCong Xu <cong.xu@intel.com>
parent 7458044e
Hide whitespace changes
Inline Side-by-side
Showing with 159 additions and 224 deletions
darshan-util/darshan-dxt-logutils.c
View file @ fcfcfe8a
......@@ -21,8 +21,8 @@
static int dxt_log_get_posix_file(darshan_fd fd, void** dxt_posix_buf);
static int dxt_log_put_posix_file(darshan_fd fd, void* dxt_posix_buf);
static void dxt_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct darshan_name_record_ref *ref);
void dxt_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct darshan_lustre_record *ref);
static void dxt_log_print_posix_description(int ver);
static void dxt_log_print_posix_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
......@@ -30,7 +30,7 @@ static void dxt_log_agg_posix_files(void *rec, void *agg_rec, int init_flag);
static int dxt_log_get_mpiio_file(darshan_fd fd, void** dxt_mpiio_buf);
static int dxt_log_put_mpiio_file(darshan_fd fd, void* dxt_mpiio_buf);
static void dxt_log_print_mpiio_file(void *file_rec,
void dxt_log_print_mpiio_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void dxt_log_print_mpiio_description(int ver);
static void dxt_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
......@@ -44,7 +44,6 @@ struct darshan_mod_logutil_funcs dxt_posix_logutils =
{
.log_get_record = &dxt_log_get_posix_file,
.log_put_record = &dxt_log_put_posix_file,
.log_print_record_dxt = &dxt_log_print_posix_file,
// .log_print_description = &dxt_log_print_posix_description,
.log_print_diff = &dxt_log_print_posix_file_diff,
.log_agg_records = &dxt_log_agg_posix_files,
......@@ -54,7 +53,6 @@ struct darshan_mod_logutil_funcs dxt_mpiio_logutils =
{
.log_get_record = &dxt_log_get_mpiio_file,
.log_put_record = &dxt_log_put_mpiio_file,
.log_print_record = &dxt_log_print_mpiio_file,
// .log_print_description = &dxt_log_print_mpiio_description,
.log_print_diff = &dxt_log_print_mpiio_file_diff,
.log_agg_records = &dxt_log_agg_mpiio_files,
......@@ -196,8 +194,8 @@ static int dxt_log_put_mpiio_file(darshan_fd fd, void* dxt_mpiio_buf)
return(0);
}
static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct darshan_name_record_ref *ref)
void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct darshan_lustre_record *ref)
{
struct dxt_file_record *file_rec =
(struct dxt_file_record *)posix_file_rec;
......@@ -222,18 +220,21 @@ static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
int print_count;
int ost_idx;
if (!ref) {
lustreFS = 0;
}
printf("\n# DXT, file_id: %" PRIu64 ", file_name: %s\n", f_id, file_name);
printf("# DXT, rank: %d, write_count: %d, read_count: %d\n",
rank, write_count, read_count);
if (lustreFS) {
stripe_size = ref->stripe_size;
stripe_count = ref->stripe_count;
stripe_size = ref->counters[LUSTRE_STRIPE_SIZE];
stripe_count = ref->counters[LUSTRE_STRIPE_WIDTH];
printf("# DXT, mnt_pt: %s, fs_type: %s\n", mnt_pt, fs_type);
printf("# DXT, stripe_size: %d, stripe_count: %d\n", stripe_size, stripe_count);
for (i = 0; i < ref->stripe_count; i++) {
for (i = 0; i < stripe_count; i++) {
printf("# DXT, OST: %d\n", (ref->ost_ids)[i]);
}
}
......@@ -305,7 +306,7 @@ static void dxt_log_print_posix_file(void *posix_file_rec, char *file_name,
return;
}
static void dxt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
void dxt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
struct dxt_file_record *file_rec =
......
darshan-util/darshan-dxt-parser.c
View file @ fcfcfe8a
/*
* Copyright (C) 2016 University of Chicago.
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
......@@ -20,13 +20,11 @@
#include "darshan-logutils.h"
struct lustre_record_ref
{
UT_hash_handle hlink;
struct darshan_lustre_record *rec;
};
extern void dxt_logutils_cleanup();
extern void dxt_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct darshan_lustre_record *ref);
extern void dxt_log_print_mpiio_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
int usage (char *exename)
{
......@@ -45,7 +43,7 @@ int main(int argc, char **argv)
darshan_fd fd;
struct darshan_job job;
struct darshan_name_record_ref *name_hash = NULL;
struct darshan_name_record_ref *name_rec_ref, *tmp_name_rec_ref;
struct darshan_name_record_ref *ref, *tmp_ref;
int mount_count;
struct darshan_mnt_info *mnt_data_array;
time_t tmp_time = 0;
......@@ -55,24 +53,21 @@ int main(int argc, char **argv)
char buffer[DARSHAN_JOB_METADATA_LEN];
struct lustre_record_ref *lustre_rec_ref, *tmp_lustre_rec_ref;
struct lustre_record_ref *lustre_rec_hash = NULL;
struct darshan_base_record *base_rec;
int empty_mods = 0;
char *mod_buf = NULL;
char *mnt_pt;
char *fs_type;
char *rec_name;
if(argc != 2)
if (argc != 2)
usage(argv[0]);
filename = argv[1];
fd = darshan_log_open(filename);
if(!fd)
if (!fd)
return(-1);
/* read darshan job info */
ret = darshan_log_get_job(fd, &job);
if(ret < 0)
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
......@@ -80,7 +75,7 @@ int main(int argc, char **argv)
/* get the original command line for this job */
ret = darshan_log_get_exe(fd, tmp_string);
if(ret < 0)
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
......@@ -88,7 +83,7 @@ int main(int argc, char **argv)
/* get the mount information for this log */
ret = darshan_log_get_mounts(fd, &mnt_data_array, &mount_count);
if(ret < 0)
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
......@@ -96,13 +91,16 @@ int main(int argc, char **argv)
/* read hash of darshan records */
ret = darshan_log_get_namehash(fd, &name_hash);
if(ret < 0)
goto cleanup;
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
}
/* print any warnings related to this log file version */
darshan_log_print_version_warnings(fd->version);
if(fd->comp_type == DARSHAN_ZLIB_COMP)
if (fd->comp_type == DARSHAN_ZLIB_COMP)
comp_str = "ZLIB";
else if (fd->comp_type == DARSHAN_BZIP2_COMP)
comp_str = "BZIP2";
......@@ -125,12 +123,12 @@ int main(int argc, char **argv)
tmp_time += job.end_time;
printf("# end_time_asci: %s", ctime(&tmp_time));
printf("# nprocs: %" PRId64 "\n", job.nprocs);
if(job.end_time >= job.start_time)
if (job.end_time >= job.start_time)
run_time = job.end_time - job.start_time + 1;
printf("# run time: %" PRId64 "\n", run_time);
for(token=strtok_r(job.metadata, "\n", &save);
for (token = strtok_r(job.metadata, "\n", &save);
token != NULL;
token=strtok_r(NULL, "\n", &save))
token = strtok_r(NULL, "\n", &save))
{
char *key;
char *value;
......@@ -155,9 +153,9 @@ int main(int argc, char **argv)
printf("# header: %zu bytes (uncompressed)\n", sizeof(struct darshan_header));
printf("# job data: %zu bytes (compressed)\n", fd->job_map.len);
printf("# record table: %zu bytes (compressed)\n", fd->name_map.len);
for(i=0; i<DARSHAN_MAX_MODS; i++)
for (i = 0; i < DARSHAN_MAX_MODS; i++)
{
if(fd->mod_map[i].len)
if (fd->mod_map[i].len)
{
printf("# %s module: %zu bytes (compressed), ver=%d\n",
darshan_module_names[i], fd->mod_map[i].len, fd->mod_ver[i]);
......@@ -167,7 +165,7 @@ int main(int argc, char **argv)
/* print table of mounted file systems */
printf("\n# mounted file systems (mount point and fs type)\n");
printf("# -------------------------------------------------------\n");
for(i=0; i<mount_count; i++)
for (i = 0; i < mount_count; i++)
{
printf("# mount entry:\t%s\t%s\n", mnt_data_array[i].mnt_path,
mnt_data_array[i].mnt_type);
......@@ -180,178 +178,149 @@ int main(int argc, char **argv)
goto cleanup;
}
if(fd->mod_map[DARSHAN_LUSTRE_MOD].len > 0 &&
mod_logutils[DARSHAN_LUSTRE_MOD])
mod_buf = malloc(DEF_MOD_BUF_SIZE);
if (!mod_buf) {
goto cleanup;
}
for (i = 0; i < DARSHAN_MAX_MODS; i++)
{
/* get lustre module data, put in hash table */
struct darshan_base_record *base_rec;
/* check each module for any data */
if (fd->mod_map[i].len == 0)
{
empty_mods++;
continue;
}
/* skip modules with no logutil definitions */
else if (!mod_logutils[i])
{
fprintf(stderr, "Warning: no log utility handlers defined "
"for module %s, SKIPPING.\n", darshan_module_names[i]);
continue;
}
/* this module has data to be parsed and printed */
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
if (i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD) {
printf("\n# ***************************************************\n");
printf("# %s module data\n", darshan_module_names[i]);
printf("# ***************************************************\n");
}
/* loop over each of this module's records and print them */
while(1)
{
lustre_rec_ref = malloc(sizeof(*lustre_rec_ref));
assert(lustre_rec_ref);
memset(lustre_rec_ref, 0, sizeof(*lustre_rec_ref));
char *mnt_pt = NULL;
char *fs_type = NULL;
char *rec_name = NULL;
if (i == DARSHAN_LUSTRE_MOD) {
lustre_rec_ref = malloc(sizeof(*lustre_rec_ref));
assert(lustre_rec_ref);
memset(lustre_rec_ref, 0, sizeof(*lustre_rec_ref));
ret = mod_logutils[i]->log_get_record(fd,
(void **)&(lustre_rec_ref->rec));
} else {
ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
}
ret = mod_logutils[DARSHAN_LUSTRE_MOD]->log_get_record(fd,
(void **)&(lustre_rec_ref->rec));
if(ret < 1)
if (ret < 1)
{
if(ret == -1)
if (ret == -1)
{
fprintf(stderr, "Error: failed to parse Lustre module record.\n");
fprintf(stderr, "Error: failed to parse %s module record.\n",
darshan_module_names[i]);
goto cleanup;
}
break;
}
else
{
if (i == DARSHAN_LUSTRE_MOD) {
HASH_ADD(hlink, lustre_rec_hash, rec->base_rec.id,
sizeof(darshan_record_id), lustre_rec_ref);
sizeof(darshan_record_id), lustre_rec_ref);
}
}
}
mod_buf = malloc(DEF_MOD_BUF_SIZE);
if (!mod_buf)
goto cleanup;
if ((i != DXT_POSIX_MOD) && (i != DXT_MPIIO_MOD))
continue;
/* iterate over DXT POSIX records and print them out */
if(fd->mod_map[DXT_POSIX_MOD].len > 0 && mod_logutils[DXT_POSIX_MOD])
{
while(1)
{
mnt_pt = NULL;
fs_type = NULL;
rec_name = NULL;
base_rec = (struct darshan_base_record *)mod_buf;
ret = mod_logutils[DXT_POSIX_MOD]->log_get_record(fd, (void **)&mod_buf);
if(ret < 1)
{
if(ret == -1)
{
fprintf(stderr, "Error: failed to parse DXT POSIX module record.\n");
goto cleanup;
}
break;
}
else
/* get the pathname for this record */
HASH_FIND(hlink, name_hash, &(base_rec->id),
sizeof(darshan_record_id), ref);
if (ref)
{
/* got a record */
base_rec = (struct darshan_base_record *)mod_buf;
rec_name = ref->name_record->name;
/* get the pathname for this record */
HASH_FIND(hlink, name_hash, &(base_rec->id),
sizeof(darshan_record_id), name_rec_ref);
if(name_rec_ref)
/* get mount point and fs type associated with this record */
for (j = 0; j < mount_count; j++)
{
rec_name = name_rec_ref->name_record->name;
/* get mount point and fs type associated with this record */
for(j=0; j<mount_count; j++)
if(strncmp(mnt_data_array[j].mnt_path, rec_name,
strlen(mnt_data_array[j].mnt_path)) == 0)
{
if(strncmp(mnt_data_array[j].mnt_path, rec_name,
strlen(mnt_data_array[j].mnt_path)) == 0)
{
mnt_pt = mnt_data_array[j].mnt_path;
fs_type = mnt_data_array[j].mnt_type;
break;
}
mnt_pt = mnt_data_array[j].mnt_path;
fs_type = mnt_data_array[j].mnt_type;
break;
}
}
if(!mnt_pt)
mnt_pt = "UNKNOWN";
if(!fs_type)
fs_type = "UNKNOWN";
/* look for corresponding lustre record and print DXT data */
HASH_FIND(hlink, lustre_rec_hash, &(base_rec->id),
sizeof(darshan_record_id), lustre_rec_ref);
if(lustre_rec_ref)
{
/* Lustre record found, data in lustre_rec_ref->rec */
}
else
{
/* no Lustre record found */
}
}
}
}
/* iterate over DXT MPI-IO records and print them out */
if(fd->mod_map[DXT_MPIIO_MOD].len > 0 && mod_logutils[DXT_MPIIO_MOD])
{
while(1)
{
mnt_pt = NULL;
fs_type = NULL;
rec_name = NULL;
if (!mnt_pt)
mnt_pt = "UNKNOWN";
if (!fs_type)
fs_type = "UNKNOWN";
ret = mod_logutils[DXT_MPIIO_MOD]->log_get_record(fd, (void **)&mod_buf);
if(ret < 1)
{
if(ret == -1)
{
fprintf(stderr, "Error: failed to parse DXT POSIX module record.\n");
goto cleanup;
}
break;
if (i == DXT_POSIX_MOD) {
/* look for corresponding lustre record and print DXT data */
HASH_FIND(hlink, lustre_rec_hash, &(base_rec->id),
sizeof(darshan_record_id), lustre_rec_ref);
dxt_log_print_posix_file(mod_buf, rec_name,
mnt_pt, fs_type, lustre_rec_ref->rec);
} else if (i == DXT_MPIIO_MOD){
dxt_log_print_mpiio_file(mod_buf, rec_name,
mnt_pt, fs_type);
}
else
{
/* got a record */
base_rec = (struct darshan_base_record *)mod_buf;
/* get the pathname for this record */
HASH_FIND(hlink, name_hash, &(base_rec->id),
sizeof(darshan_record_id), name_rec_ref);
if(name_rec_ref)
{
rec_name = name_rec_ref->name_record->name;
/* get mount point and fs type associated with this record */
for(j=0; j<mount_count; j++)
{
if(strncmp(mnt_data_array[j].mnt_path, rec_name,
strlen(mnt_data_array[j].mnt_path)) == 0)
{
mnt_pt = mnt_data_array[j].mnt_path;
fs_type = mnt_data_array[j].mnt_type;
break;
}
}
}
if(!mnt_pt)
mnt_pt = "UNKNOWN";
if(!fs_type)
fs_type = "UNKNOWN";
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
}
}
if (empty_mods == DARSHAN_MAX_MODS)
printf("\n# no module data available.\n");
ret = 0;
cleanup:
free(mod_buf);
/* DXT */
dxt_logutils_cleanup();
cleanup:
darshan_log_close(fd);
free(mod_buf);
/* free record hash data */
HASH_ITER(hlink, name_hash, name_rec_ref, tmp_name_rec_ref)
HASH_ITER(hlink, name_hash, ref, tmp_ref)
{
HASH_DELETE(hlink, name_hash, name_rec_ref);
free(name_rec_ref->name_record);
free(name_rec_ref);
HASH_DELETE(hlink, name_hash, ref);
free(ref->name_record);
free(ref);
}
/* free lustre record data */
HASH_ITER(hlink, lustre_rec_hash, lustre_rec_ref, tmp_lustre_rec_ref)
{
{
HASH_DELETE(hlink, lustre_rec_hash, lustre_rec_ref);
free(lustre_rec_ref->rec);
free(lustre_rec_ref);
}
}
/* free mount info */
if(mount_count > 0)
if (mount_count > 0)
{
free(mnt_data_array);
}
......
darshan-util/darshan-logutils.h
View file @ fcfcfe8a
......@@ -53,11 +53,13 @@ struct darshan_name_record_ref
{
struct darshan_name_record *name_record;
UT_hash_handle hlink;
};
/* DXT Lustre */
int32_t stripe_size;
int32_t stripe_count;
OST_ID *ost_ids;
/* DXT */
struct lustre_record_ref
{
struct darshan_lustre_record *rec;
UT_hash_handle hlink;
};
struct darshan_mnt_info
......@@ -121,15 +123,6 @@ struct darshan_mod_logutil_funcs
void *agg_rec,
int init_flag
);
/* DXT */
void (*log_print_record_dxt)(
void *rec,
char *file_name,
char *mnt_pt,
char *fs_type,
struct darshan_name_record_ref *ref
);
};
extern struct darshan_mod_logutil_funcs *mod_logutils[];
......
darshan-util/darshan-parser.c
View file @ fcfcfe8a
......@@ -121,9 +121,6 @@ void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver);
void calc_perf(perf_data_t *pdata, int64_t nprocs);
/* DXT */
extern void dxt_logutils_cleanup();
int usage (char *exename)
{
fprintf(stderr, "Usage: %s [options] <filename>\n", exename);
......@@ -401,7 +398,8 @@ int main(int argc, char **argv)
/* currently only POSIX, MPIIO, and STDIO modules support non-base
* parsing
*/
else if((i != DARSHAN_POSIX_MOD) && (i != DARSHAN_MPIIO_MOD) && (i != DARSHAN_STDIO_MOD) && !(mask & OPTION_BASE))
else if((i != DARSHAN_POSIX_MOD) && (i != DARSHAN_MPIIO_MOD) &&
(i != DARSHAN_STDIO_MOD) && !(mask & OPTION_BASE))
continue;
/* this module has data to be parsed and printed */
......@@ -428,22 +426,24 @@ int main(int argc, char **argv)
}
}
ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
if(ret != 1)
{
fprintf(stderr, "Error: failed to parse the first %s module record.\n",
darshan_module_names[i]);
ret = -1;
goto cleanup;
}
/* loop over each of this module's records and print them */
do
while(1)
{
char *mnt_pt = NULL;
char *fs_type = NULL;
char *rec_name = NULL;
hash_entry_t *hfile = NULL;
ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
if(ret < 1)
{
if(ret == -1)
{
fprintf(stderr, "Error: failed to parse %s module record.\n",
darshan_module_names[i]);
}
break;
}
base_rec = (struct darshan_base_record *)mod_buf;
/* get the pathname for this record */
......@@ -476,32 +476,11 @@ int main(int argc, char **argv)
if(!fs_type)
fs_type = "UNKNOWN";
/* DXT */
if (i == DARSHAN_LUSTRE_MOD && ref) {
/* LUSTRE MODULE */
struct darshan_lustre_record *file_rec =
(struct darshan_lustre_record *)mod_buf;
ref->stripe_size = file_rec->counters[LUSTRE_STRIPE_SIZE];
ref->stripe_count = file_rec->counters[LUSTRE_STRIPE_WIDTH];
int ost_ids_size = ref->stripe_count * sizeof(OST_ID);
ref->ost_ids = (OST_ID *) malloc(ost_ids_size);
memcpy((void *)(ref->ost_ids), (void *)(file_rec->ost_ids),
ost_ids_size);
}
if(mask & OPTION_BASE)
{
/* DXT */
if (i == DXT_POSIX_MOD) {
mod_logutils[i]->log_print_record_dxt(mod_buf, rec_name,
mnt_pt, fs_type, ref);
} else {
/* print the corresponding module data for this record */
mod_logutils[i]->log_print_record(mod_buf, rec_name,
mnt_pt, fs_type);
}
/* print the corresponding module data for this record */
mod_logutils[i]->log_print_record(mod_buf, rec_name,
mnt_pt, fs_type);
}
/* we calculate more detailed stats for POSIX, MPI-IO, and STDIO modules,
......@@ -552,13 +531,9 @@ int main(int argc, char **argv)