Commit ecec3bb8 authored by Philip Carns's avatar Philip Carns
Browse files

merged darshan-parser-file-list branch to trunk


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@754 3b7491f3-a168-0410-bf4b-c445ed680a29
parents b3c5a5f5 d1ed2052
...@@ -28,22 +28,28 @@ int usage (char *exename) ...@@ -28,22 +28,28 @@ int usage (char *exename)
fprintf(stderr, " --obfuscate Obfuscate items in the log.\n"); fprintf(stderr, " --obfuscate Obfuscate items in the log.\n");
fprintf(stderr, " --key <key> Key to use when obfuscating.\n"); fprintf(stderr, " --key <key> Key to use when obfuscating.\n");
fprintf(stderr, " --annotate <string> Additional metadata to add.\n"); fprintf(stderr, " --annotate <string> Additional metadata to add.\n");
fprintf(stderr, " --file <hash> Limit output to specified (hashed) file only.\n");
exit(1); exit(1);
} }
void parse_args (int argc, char **argv, char **infile, char **outfile, void parse_args (int argc, char **argv, char **infile, char **outfile,
int *obfuscate, int *key, char **annotate) int *obfuscate, int *key, char **annotate, uint64_t* hash)
{ {
int index; int index;
int ret;
static struct option long_opts[] = static struct option long_opts[] =
{ {
{"annotate", 1, NULL, 'a'}, {"annotate", 1, NULL, 'a'},
{"obfuscate", 0, NULL, 'o'}, {"obfuscate", 0, NULL, 'o'},
{"key", 1, NULL, 'k'}, {"key", 1, NULL, 'k'},
{"file", 1, NULL, 'f'},
{"help", 0, NULL, 0} {"help", 0, NULL, 0}
}; };
*hash = 0;
while(1) while(1)
{ {
int c = getopt_long(argc, argv, "", long_opts, &index); int c = getopt_long(argc, argv, "", long_opts, &index);
...@@ -61,6 +67,11 @@ void parse_args (int argc, char **argv, char **infile, char **outfile, ...@@ -61,6 +67,11 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
case 'k': case 'k':
*key = atoi(optarg); *key = atoi(optarg);
break; break;
case 'f':
ret = sscanf(optarg, "%" PRIu64, hash);
if(ret != 1)
usage(argv[0]);
break;
case 0: case 0:
case '?': case '?':
default: default:
...@@ -151,8 +162,9 @@ int main(int argc, char **argv) ...@@ -151,8 +162,9 @@ int main(int argc, char **argv)
int obfuscate = 0; int obfuscate = 0;
int key = 0; int key = 0;
char *annotation = NULL; char *annotation = NULL;
uint64_t hash;
parse_args(argc, argv, &infile_name, &outfile_name, &obfuscate, &key, &annotation); parse_args(argc, argv, &infile_name, &outfile_name, &obfuscate, &key, &annotation, &hash);
infile = darshan_log_open(infile_name, "r"); infile = darshan_log_open(infile_name, "r");
if(!infile) if(!infile)
...@@ -252,13 +264,16 @@ int main(int argc, char **argv) ...@@ -252,13 +264,16 @@ int main(int argc, char **argv)
if(cp_file.rank != -1) if(cp_file.rank != -1)
last_rank = cp_file.rank; last_rank = cp_file.rank;
if (obfuscate) obfuscate_file(key, &cp_file); if(!hash || hash == cp_file.hash)
ret = darshan_log_putfile(outfile, &job, &cp_file);
if (ret < 0)
{ {
fprintf(stderr, "Error: failed to write file record.\n"); if (obfuscate) obfuscate_file(key, &cp_file);
break;
ret = darshan_log_putfile(outfile, &job, &cp_file);
if (ret < 0)
{
fprintf(stderr, "Error: failed to write file record.\n");
break;
}
} }
} while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1); } while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1);
......
...@@ -461,7 +461,10 @@ int darshan_log_putjob(darshan_fd file, struct darshan_job *job) ...@@ -461,7 +461,10 @@ int darshan_log_putjob(darshan_fd file, struct darshan_job *job)
memcpy(&job_copy, job, sizeof(job_copy)); memcpy(&job_copy, job, sizeof(job_copy));
sprintf(pv_str, "prev_ver=%s\n", job->version_string); sprintf(pv_str, "prev_ver=%s\n", job->version_string);
sprintf(job_copy.version_string, "%s", CP_VERSION); sprintf(job_copy.version_string, "%s", CP_VERSION);
strncat(job_copy.metadata, pv_str, strlen(pv_str)); if(strlen(job_copy.metadata) + strlen(pv_str) < DARSHAN_JOB_METADATA_LEN)
strncat(job_copy.metadata, pv_str, strlen(pv_str));
else
sprintf(job_copy.metadata, "%s", pv_str);
job_copy.magic_nr = CP_MAGIC_NR; job_copy.magic_nr = CP_MAGIC_NR;
ret = darshan_log_write(file, &job_copy, sizeof(job_copy)); ret = darshan_log_write(file, &job_copy, sizeof(job_copy));
......
...@@ -26,11 +26,15 @@ ...@@ -26,11 +26,15 @@
#define OPTION_TOTAL (1 << 1) /* aggregated fields */ #define OPTION_TOTAL (1 << 1) /* aggregated fields */
#define OPTION_PERF (1 << 2) /* derived performance */ #define OPTION_PERF (1 << 2) /* derived performance */
#define OPTION_FILE (1 << 3) /* file count totals */ #define OPTION_FILE (1 << 3) /* file count totals */
#define OPTION_FILE_LIST (1 << 4) /* per-file summaries */
#define OPTION_FILE_LIST_DETAILED (1 << 6) /* per-file summaries with extra detail */
#define OPTION_ALL (\ #define OPTION_ALL (\
OPTION_BASE|\ OPTION_BASE|\
OPTION_TOTAL|\ OPTION_TOTAL|\
OPTION_PERF|\ OPTION_PERF|\
OPTION_FILE) OPTION_FILE|\
OPTION_FILE_LIST|\
OPTION_FILE_LIST_DETAILED)
#define FILETYPE_SHARED (1 << 0) #define FILETYPE_SHARED (1 << 0)
#define FILETYPE_UNIQUE (1 << 1) #define FILETYPE_UNIQUE (1 << 1)
...@@ -45,13 +49,15 @@ ...@@ -45,13 +49,15 @@
typedef struct hash_entry_s typedef struct hash_entry_s
{ {
UT_hash_handle hlink; UT_hash_handle hlink;
int64_t hash; uint64_t hash;
int64_t type; int64_t type;
int64_t procs; int64_t procs;
int64_t counters[CP_NUM_INDICES]; int64_t counters[CP_NUM_INDICES];
double fcounters[CP_F_NUM_INDICES]; double fcounters[CP_F_NUM_INDICES];
double cumul_time; double cumul_time;
double meta_time; double meta_time;
double slowest_time;
char name_suffix[CP_NAME_SUFFIX_LEN+1];
} hash_entry_t; } hash_entry_t;
typedef struct perf_data_s typedef struct perf_data_s
...@@ -100,8 +106,9 @@ typedef struct file_data_s ...@@ -100,8 +106,9 @@ typedef struct file_data_s
void accum_perf(struct darshan_file *, hash_entry_t *, perf_data_t *); void accum_perf(struct darshan_file *, hash_entry_t *, perf_data_t *);
void calc_perf(struct darshan_job *, hash_entry_t *, perf_data_t *); void calc_perf(struct darshan_job *, hash_entry_t *, perf_data_t *);
void accum_file(struct darshan_file *, hash_entry_t *, file_data_t *); void accum_file(struct darshan_job *, struct darshan_file *, hash_entry_t *, file_data_t *);
void calc_file(struct darshan_job *, hash_entry_t *, file_data_t *); void calc_file(struct darshan_job *, hash_entry_t *, file_data_t *);
void file_list(struct darshan_job *, hash_entry_t *, int);
int usage (char *exename) int usage (char *exename)
{ {
...@@ -109,6 +116,8 @@ int usage (char *exename) ...@@ -109,6 +116,8 @@ int usage (char *exename)
fprintf(stderr, " --all : all sub-options are enabled\n"); fprintf(stderr, " --all : all sub-options are enabled\n");
fprintf(stderr, " --base : darshan log field data [default]\n"); fprintf(stderr, " --base : darshan log field data [default]\n");
fprintf(stderr, " --file : total file counts\n"); fprintf(stderr, " --file : total file counts\n");
fprintf(stderr, " --file-list : per-file summaries\n");
fprintf(stderr, " --file-list-detailed : per-file summaries with additional detail\n");
fprintf(stderr, " --perf : derived perf data\n"); fprintf(stderr, " --perf : derived perf data\n");
fprintf(stderr, " --total : aggregated darshan field data\n"); fprintf(stderr, " --total : aggregated darshan field data\n");
...@@ -124,6 +133,8 @@ int parse_args (int argc, char **argv, char **filename) ...@@ -124,6 +133,8 @@ int parse_args (int argc, char **argv, char **filename)
{"all", 0, NULL, OPTION_ALL}, {"all", 0, NULL, OPTION_ALL},
{"base", 0, NULL, OPTION_BASE}, {"base", 0, NULL, OPTION_BASE},
{"file", 0, NULL, OPTION_FILE}, {"file", 0, NULL, OPTION_FILE},
{"file-list", 0, NULL, OPTION_FILE_LIST},
{"file-list-detailed", 0, NULL, OPTION_FILE_LIST_DETAILED},
{"perf", 0, NULL, OPTION_PERF}, {"perf", 0, NULL, OPTION_PERF},
{"total", 0, NULL, OPTION_TOTAL}, {"total", 0, NULL, OPTION_TOTAL},
{"help", 0, NULL, 0}, {"help", 0, NULL, 0},
...@@ -143,6 +154,8 @@ int parse_args (int argc, char **argv, char **filename) ...@@ -143,6 +154,8 @@ int parse_args (int argc, char **argv, char **filename)
case OPTION_ALL: case OPTION_ALL:
case OPTION_BASE: case OPTION_BASE:
case OPTION_FILE: case OPTION_FILE:
case OPTION_FILE_LIST:
case OPTION_FILE_LIST_DETAILED:
case OPTION_PERF: case OPTION_PERF:
case OPTION_TOTAL: case OPTION_TOTAL:
mask |= c; mask |= c;
...@@ -410,16 +423,18 @@ int main(int argc, char **argv) ...@@ -410,16 +423,18 @@ int main(int argc, char **argv)
/* init */ /* init */
memset(hfile, 0, sizeof(*hfile)); memset(hfile, 0, sizeof(*hfile));
hfile->hash = cp_file.hash; hfile->hash = cp_file.hash;
memcpy(hfile->name_suffix, cp_file.name_suffix, CP_NAME_SUFFIX_LEN+1);
hfile->type = 0; hfile->type = 0;
hfile->procs = 0; hfile->procs = 0;
hfile->cumul_time = 0.0; hfile->cumul_time = 0.0;
hfile->meta_time = 0.0; hfile->meta_time = 0.0;
hfile->slowest_time = 0.0;
HASH_ADD(hlink,file_hash,hash,sizeof(int64_t),hfile); HASH_ADD(hlink,file_hash,hash,sizeof(int64_t),hfile);
} }
accum_file(&cp_file, &total, NULL); accum_file(&job, &cp_file, &total, NULL);
accum_file(&cp_file, hfile, &fdata); accum_file(&job, &cp_file, hfile, &fdata);
accum_perf(&cp_file, hfile, &pdata); accum_perf(&cp_file, hfile, &pdata);
if ((mask & OPTION_BASE)) if ((mask & OPTION_BASE))
...@@ -501,6 +516,14 @@ int main(int argc, char **argv) ...@@ -501,6 +516,14 @@ int main(int argc, char **argv)
fdata.shared_max); fdata.shared_max);
} }
if ((mask & OPTION_FILE_LIST) || mask & OPTION_FILE_LIST_DETAILED)
{
if(mask & OPTION_FILE_LIST_DETAILED)
file_list(&job, file_hash, 1);
else
file_list(&job, file_hash, 0);
}
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: failed to parse log file.\n"); fprintf(stderr, "Error: failed to parse log file.\n");
...@@ -531,7 +554,8 @@ int main(int argc, char **argv) ...@@ -531,7 +554,8 @@ int main(int argc, char **argv)
return(0); return(0);
} }
void accum_file(struct darshan_file *dfile, void accum_file(struct darshan_job *job,
struct darshan_file *dfile,
hash_entry_t *hfile, hash_entry_t *hfile,
file_data_t *fdata) file_data_t *fdata)
{ {
...@@ -541,7 +565,46 @@ void accum_file(struct darshan_file *dfile, ...@@ -541,7 +565,46 @@ void accum_file(struct darshan_file *dfile,
if (dfile->rank == -1) if (dfile->rank == -1)
{ {
if(job->version_string[0] == '1')
{
hfile->slowest_time =
max((dfile->fcounters[CP_F_READ_END_TIMESTAMP]
- dfile->fcounters[CP_F_OPEN_TIMESTAMP]),
(dfile->fcounters[CP_F_WRITE_END_TIMESTAMP]
- dfile->fcounters[CP_F_OPEN_TIMESTAMP]));
if(hfile->slowest_time < 0)
hfile->slowest_time = 0;
}
else
{
hfile->slowest_time = dfile->fcounters[CP_F_SLOWEST_RANK_TIME];
}
}
else
{
if(dfile->counters[CP_INDEP_OPENS] || dfile->counters[CP_COLL_OPENS])
{
/* MPI file */
hfile->slowest_time = max(hfile->slowest_time,
(dfile->fcounters[CP_F_MPI_META_TIME] +
dfile->fcounters[CP_F_MPI_READ_TIME] +
dfile->fcounters[CP_F_MPI_WRITE_TIME]));
}
else
{
/* POSIX file */
hfile->slowest_time = max(hfile->slowest_time,
(dfile->fcounters[CP_F_POSIX_META_TIME] +
dfile->fcounters[CP_F_POSIX_READ_TIME] +
dfile->fcounters[CP_F_POSIX_WRITE_TIME]));
}
}
if (dfile->rank == -1)
{
hfile->procs = job->nprocs;
hfile->type |= FILETYPE_SHARED; hfile->type |= FILETYPE_SHARED;
} }
else if (hfile->procs > 1) else if (hfile->procs > 1)
{ {
...@@ -553,6 +616,19 @@ void accum_file(struct darshan_file *dfile, ...@@ -553,6 +616,19 @@ void accum_file(struct darshan_file *dfile,
hfile->type |= FILETYPE_UNIQUE; hfile->type |= FILETYPE_UNIQUE;
} }
if(dfile->counters[CP_INDEP_OPENS] || dfile->counters[CP_COLL_OPENS])
{
hfile->cumul_time += dfile->fcounters[CP_F_MPI_META_TIME] +
dfile->fcounters[CP_F_MPI_READ_TIME] +
dfile->fcounters[CP_F_MPI_WRITE_TIME];
}
else
{
hfile->cumul_time += dfile->fcounters[CP_F_POSIX_META_TIME] +
dfile->fcounters[CP_F_POSIX_READ_TIME] +
dfile->fcounters[CP_F_POSIX_WRITE_TIME];
}
for (i = 0; i < CP_NUM_INDICES; i++) for (i = 0; i < CP_NUM_INDICES; i++)
{ {
switch(i) switch(i)
...@@ -629,6 +705,24 @@ void accum_file(struct darshan_file *dfile, ...@@ -629,6 +705,24 @@ void accum_file(struct darshan_file *dfile,
{ {
switch(i) switch(i)
{ {
case CP_F_OPEN_TIMESTAMP:
case CP_F_READ_START_TIMESTAMP:
case CP_F_WRITE_START_TIMESTAMP:
if(hfile->fcounters[i] == 0 ||
hfile->fcounters[i] > dfile->fcounters[i])
{
hfile->fcounters[i] = dfile->fcounters[i];
}
break;
case CP_F_CLOSE_TIMESTAMP:
case CP_F_READ_END_TIMESTAMP:
case CP_F_WRITE_END_TIMESTAMP:
if(hfile->fcounters[i] == 0 ||
hfile->fcounters[i] < dfile->fcounters[i])
{
hfile->fcounters[i] = dfile->fcounters[i];
}
break;
case CP_F_FASTEST_RANK_TIME: case CP_F_FASTEST_RANK_TIME:
case CP_F_SLOWEST_RANK_TIME: case CP_F_SLOWEST_RANK_TIME:
case CP_F_VARIANCE_RANK_TIME: case CP_F_VARIANCE_RANK_TIME:
...@@ -660,6 +754,99 @@ void accum_file(struct darshan_file *dfile, ...@@ -660,6 +754,99 @@ void accum_file(struct darshan_file *dfile,
return; return;
} }
void file_list(struct darshan_job *djob, hash_entry_t *file_hash, int detail_flag)
{
hash_entry_t *curr = NULL;
hash_entry_t *tmp = NULL;
char* type;
int i;
/* TODO: list of columns:
*
* normal mode?
* - hash
* - suffix
* - MPI or POSIX
* - nprocs
* - slowest I/O time
* - average cumulative I/O time
*
* detailed mode?
* - first open
* - first read
* - first write
* - last close
* - last read
* - last write
* - MPI indep opens
* - MPI coll opens
* - POSIX opens
* - r histogram (POSIX)
* - w histogram (POSIX)
*/
if(detail_flag)
printf("\n# Per-file summary of I/O activity (detailed).\n");
else
printf("\n# Per-file summary of I/O activity.\n");
printf("# <hash>: hash of file name\n");
printf("# <suffix>: last %d characters of file name\n", CP_NAME_SUFFIX_LEN);
printf("# <type>: MPI or POSIX\n");
printf("# <nprocs>: number of processes that opened the file\n");
printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
printf("# <avg>: average time in seconds consumed in IO per process\n");
if(detail_flag)
{
printf("# <start_{open/read/write}>: start timestamp of first open, read, or write\n");
printf("# <end_{open/read/write}>: end timestamp of last open, read, or write\n");
printf("# <mpi_indep_opens>: independent MPI_File_open calls\n");
printf("# <mpi_coll_opens>: collective MPI_File_open calls\n");
printf("# <posix_opens>: POSIX open calls\n");
printf("# <CP_SIZE_READ_*>: POSIX read size histogram\n");
printf("# <CP_SIZE_WRITE_*>: POSIX write size histogram\n");
}
printf("\n# <hash>\t<suffix>\t<type>\t<nprocs>\t<slowest>\t<avg>");
if(detail_flag)
{
printf("\t<start_open>\t<start_read>\t<start_write>");
printf("\t<end_open>\t<end_read>\t<end_write>");
printf("\t<mpi_indep_opens>\t<mpi_coll_opens>\t<posix_opens>");
for(i=CP_SIZE_READ_0_100; i<= CP_SIZE_WRITE_1G_PLUS; i++)
printf("\t%s", darshan_names[i]);
}
printf("\n");
HASH_ITER(hlink, file_hash, curr, tmp)
{
if(curr->counters[CP_INDEP_OPENS] || curr->counters[CP_COLL_OPENS])
type = "MPI";
else
type = "POSIX";
printf("%" PRIu64 "\t%s\t%s\t%" PRId64 "\t%f\t%f",
curr->hash,
curr->name_suffix,
type,
curr->procs,
curr->slowest_time,
curr->cumul_time/(double)curr->procs);
if(detail_flag)
{
for(i=CP_F_OPEN_TIMESTAMP; i<=CP_F_WRITE_END_TIMESTAMP; i++)
{
printf("\t%f", curr->fcounters[i]);
}
printf("\t%" PRId64 "\t%" PRId64 "\t%" PRId64, curr->counters[CP_INDEP_OPENS], curr->counters[CP_COLL_OPENS], curr->counters[CP_POSIX_OPENS]);
for(i=CP_SIZE_READ_0_100; i<= CP_SIZE_WRITE_1G_PLUS; i++)
printf("\t%" PRId64, curr->counters[i]);
}
printf("\n");
}
return;
}
void calc_file(struct darshan_job *djob, void calc_file(struct darshan_job *djob,
hash_entry_t *file_hash, hash_entry_t *file_hash,
...@@ -828,38 +1015,19 @@ void accum_perf(struct darshan_file *dfile, ...@@ -828,38 +1015,19 @@ void accum_perf(struct darshan_file *dfile,
{ {
if (mpi_file) if (mpi_file)
{ {
#if 0
hfile->cumul_time += dfile->fcounters[CP_F_MPI_META_TIME] +
dfile->fcounters[CP_F_MPI_READ_TIME] +
dfile->fcounters[CP_F_MPI_WRITE_TIME];
hfile->meta_time += dfile->fcounters[CP_F_MPI_META_TIME];
#else
pdata->rank_cumul_io_time[dfile->rank] += dfile->fcounters[CP_F_MPI_META_TIME] + pdata->rank_cumul_io_time[dfile->rank] += dfile->fcounters[CP_F_MPI_META_TIME] +
dfile->fcounters[CP_F_MPI_READ_TIME] + dfile->fcounters[CP_F_MPI_READ_TIME] +
dfile->fcounters[CP_F_MPI_WRITE_TIME]; dfile->fcounters[CP_F_MPI_WRITE_TIME];
pdata->rank_cumul_md_time[dfile->rank] += dfile->fcounters[CP_F_MPI_META_TIME]; pdata->rank_cumul_md_time[dfile->rank] += dfile->fcounters[CP_F_MPI_META_TIME];
#endif
} }
else else
{ {
#if 0
hfile->cumul_time += dfile->fcounters[CP_F_POSIX_META_TIME] +
dfile->fcounters[CP_F_POSIX_READ_TIME] +
dfile->fcounters[CP_F_POSIX_WRITE_TIME];
hfile->meta_time += dfile->fcounters[CP_F_POSIX_META_TIME];
#else
pdata->rank_cumul_io_time[dfile->rank] += dfile->fcounters[CP_F_POSIX_META_TIME] + pdata->rank_cumul_io_time[dfile->rank] += dfile->fcounters[CP_F_POSIX_META_TIME] +
dfile->fcounters[CP_F_POSIX_READ_TIME] + dfile->fcounters[CP_F_POSIX_READ_TIME] +
dfile->fcounters[CP_F_POSIX_WRITE_TIME]; dfile->fcounters[CP_F_POSIX_WRITE_TIME];
pdata->rank_cumul_md_time[dfile->rank] += dfile->fcounters[CP_F_POSIX_META_TIME]; pdata->rank_cumul_md_time[dfile->rank] += dfile->fcounters[CP_F_POSIX_META_TIME];
#endif
} }
#if 0
pdata->rank_cumul_io_time[dfile->rank] += hfile->cumul_time;
pdata->rank_cumul_md_time[dfile->rank] += hfile->meta_time;
#endif
} }
return; return;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment