Commit 8eea697a authored by Shane Snyder's avatar Shane Snyder

refactor logutil code, create base darshan-parser

parent a0b3f9cc
all: libdarshan-util.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser
all: libdarshan-util.a darshan-parser
DESTDIR =
srcdir = @srcdir@
......@@ -65,18 +65,9 @@ libdarshan-util.a: darshan-logutils.o darshan-posix-logutils.o darshan-mpiio-log
jenkins: util/bin/jenkins.o lookup3.o
$(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
darshan-base-parser: darshan-base-parser.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) libdarshan-util.a | uthash-1.9.2
darshan-parser: darshan-parser.c darshan-logutils.h darshan-posix-logutils.h darshan-mpiio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
darshan-posix-parser: darshan-posix-parser.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
darshan-mpiio-parser: darshan-mpiio-parser.c darshan-logutils.h darshan-mpiio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
#darshan-parser: darshan-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o | uthash-1.9.2
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
#darshan-convert: darshan-convert.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o lookup3.o
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o lookup3.o -o $@ $(LIBS)
......@@ -105,10 +96,7 @@ install:: all
install -d $(libdir)
install -d $(includedir)
install -d $(pkgconfigdir)
# install -m 755 darshan-parser $(bindir)
install -m 755 darshan-base-parser $(bindir)
install -m 755 darshan-posix-parser $(bindir)
install -m 755 darshan-mpiio-parser $(bindir)
install -m 755 darshan-parser $(bindir)
# install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir)
# install -m 755 darshan-analyzer $(bindir)
......@@ -131,7 +119,7 @@ endif
clean::
rm -f *.o *.a darshan-base-parser darshan-posix-parser darshan-mpiio-parser
rm -f *.o *.a darshan-parser darshan-posix-parser darshan-mpiio-parser
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
......
......@@ -21,7 +21,28 @@
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
//static int darshan_log_write(darshan_fd fd, void *buf, int len);
/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
NULL, /* NULL */
&posix_logutils, /* POSIX */
&mpiio_logutils, /* MPI-IO */
NULL, /* HDF5 */
NULL, /* PNETCDF */
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL
};
/* darshan_log_open()
*
......@@ -42,16 +63,6 @@ darshan_fd darshan_log_open(const char *name, const char *mode)
return(NULL);
memset(tmp_fd, 0, sizeof(*tmp_fd));
if(strcmp(mode, "r") == 0)
{
/* TODO: check for bz2 */
}
if(strcmp(mode, "w") == 0)
{
/* TODO: check for bz2 */
}
tmp_fd->gzf = gzopen(name, mode);
if(!tmp_fd->gzf)
{
......@@ -406,8 +417,6 @@ void darshan_log_close(darshan_fd fd)
if(fd->gzf)
gzclose(fd->gzf);
/* TODO: check bz2 */
free(fd);
return;
......@@ -437,11 +446,10 @@ static int darshan_log_seek(darshan_fd fd, off_t offset)
return(-1);
}
/* TODO: check bz2 */
return(-1);
}
#if 0
/* return amount written on success, -1 on failure.
*/
static int darshan_log_write(darshan_fd fd, void* buf, int len)
......@@ -456,10 +464,9 @@ static int darshan_log_write(darshan_fd fd, void* buf, int len)
return(ret);
}
/* TODO: check bz2 */
return(-1);
}
#endif
/* return amount read on success, 0 on EOF, -1 on failure.
*/
......@@ -475,8 +482,6 @@ static int darshan_log_read(darshan_fd fd, void* buf, int len)
return(ret);
}
/* TODO: check bz2 */
return(-1);
}
......
......@@ -12,14 +12,13 @@
#include <bzlib.h>
#endif
#include "darshan-log-format.h"
#include "uthash-1.9.2/src/uthash.h"
/* default to a compression buffer size of 4 MiB */
#include "darshan-log-format.h"
/* default to a decompression buffer size of 4 MiB */
#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
/* TODO: we need to refactor s.t. utilities don't know implementation
of this, but module-specific functions do */
struct darshan_fd_s
{
gzFile gzf;
......@@ -37,6 +36,26 @@ struct darshan_record_ref
UT_hash_handle hlink;
};
struct darshan_mod_logutil_funcs
{
int (*log_get_record)(
darshan_fd fd,
void **rec_dat,
darshan_record_id *rec_id
);
void (*log_print_record)(
void *file_rec,
char *file_name,
char *mnt_pt,
char *fs_type
);
};
extern struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS];
#include "darshan-posix-logutils.h"
#include "darshan-mpiio-logutils.h"
darshan_fd darshan_log_open(const char *name, const char* mode);
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header);
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job);
......@@ -48,19 +67,25 @@ int darshan_log_get_moddat(darshan_fd fd, darshan_module_id mod_id,
void *moddat_buf, int moddat_buf_sz);
void darshan_log_close(darshan_fd file);
/* convenience macros for printing out counters */
#define CP_PRINT_HEADER() printf("#<rank>\t<file>\t<counter>\t<value>\t<name suffix>\t<mount pt>\t<fs type>\n")
#define CP_PRINT(__job, __file, __counter, __mnt_pt, __fs_type) do {\
printf("%" PRId64 "\t%" PRIu64 "\t%s\t%" PRId64 "\t...%s\t%s\t%s\n", \
(__file)->rank, (__file)->hash, darshan_names[__counter], \
(__file)->counters[__counter], (__file)->name_suffix, \
__mnt_pt, __fs_type); \
/* convenience macros for printing Darshan counters */
#define DARSHAN_PRINT_HEADER() \
printf("\n#<module>\t<rank>\t<file>\t<counter>\t<value>" \
"\t<file name>\t<mount pt>\t<fs type>\n")
#define DARSHAN_COUNTER_PRINT(__mod_name, __rank, __file_id, \
__counter, __counter_val, __file_name, \
__mnt_pt, __fs_type) do { \
printf("%s\t%" PRId64 "\t%" PRIu64 "\t%s\t%" PRId64 "\t%s\t%s\t%s\n", \
__mod_name, __rank, __file_id, __counter, __counter_val, \
__file_name, __mnt_pt, __fs_type); \
} while(0)
#define CP_F_PRINT(__job, __file, __counter, __mnt_pt, __fs_type) do {\
printf("%" PRId64 "\t%" PRIu64 "\t%s\t%f\t...%s\t%s\t%s\n", \
(__file)->rank, (__file)->hash, darshan_f_names[__counter], \
(__file)->fcounters[__counter], (__file)->name_suffix, \
__mnt_pt, __fs_type); \
#define DARSHAN_F_COUNTER_PRINT(__mod_name, __rank, __file_id, \
__counter, __counter_val, __file_name, \
__mnt_pt, __fs_type) do { \
printf("%s\t%" PRId64 "\t%" PRIu64 "\t%s\t%f\t%s\t%s\t%s\n", \
__mod_name, __rank, __file_id, __counter, __counter_val, \
__file_name, __mnt_pt, __fs_type); \
} while(0)
/* naive byte swap implementation */
......
......@@ -30,14 +30,22 @@ char *mpiio_f_counter_names[] = {
};
#undef X
int darshan_log_get_mpiio_file(darshan_fd fd, struct darshan_mpiio_file *file)
struct darshan_mod_logutil_funcs mpiio_logutils =
{
.log_get_record = &darshan_log_get_mpiio_file,
.log_print_record = &darshan_log_print_mpiio_file,
};
int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id)
{
int i;
int ret;
struct darshan_mpiio_file *file = NULL;
/* reset file record, so that diff compares against a zero'd out record
* if file is missing
*/
file = malloc(sizeof(*file));
if(!file)
return(-1);
memset(file, 0, sizeof(*file));
ret = darshan_log_get_moddat(fd, DARSHAN_MPIIO_MOD,
......@@ -56,9 +64,36 @@ int darshan_log_get_mpiio_file(darshan_fd fd, struct darshan_mpiio_file *file)
}
}
/* pass the file record back */
*file_rec = (void *)file;
*rec_id = file->f_id;
return(ret);
}
void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type)
{
int i;
struct darshan_mpiio_file *mpiio_file_rec =
(struct darshan_mpiio_file *)file_rec;
for(i=0; i<MPIIO_NUM_INDICES; i++)
{
DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MPIIO_MOD],
mpiio_file_rec->rank, mpiio_file_rec->f_id, mpiio_counter_names[i],
mpiio_file_rec->counters[i], file_name, mnt_pt, fs_type);
}
for(i=0; i<MPIIO_F_NUM_INDICES; i++)
{
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MPIIO_MOD],
mpiio_file_rec->rank, mpiio_file_rec->f_id, mpiio_f_counter_names[i],
mpiio_file_rec->fcounters[i], file_name, mnt_pt, fs_type);
}
return;
}
/*
* Local variables:
* c-indent-level: 4
......
......@@ -13,18 +13,11 @@
extern char *mpiio_counter_names[];
extern char *mpiio_f_counter_names[];
int darshan_log_get_mpiio_file(darshan_fd fd, struct darshan_mpiio_file *file);
extern struct darshan_mod_logutil_funcs mpiio_logutils;
#define MPIIO_COUNTER_PRINT(__file_rec, __counter) do { \
printf("%" PRId64 "\t%" PRIu64 "\t%s\t%" PRId64 "\n", \
(__file_rec)->rank, (__file_rec)->f_id, mpiio_counter_names[__counter], \
(__file_rec)->counters[__counter]); \
} while(0)
#define MPIIO_F_COUNTER_PRINT(__file_rec, __counter) do { \
printf("%" PRId64 "\t%" PRIu64 "\t%s\t%f\n", \
(__file_rec)->rank, (__file_rec)->f_id, mpiio_f_counter_names[__counter], \
(__file_rec)->fcounters[__counter]); \
} while(0)
int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
darshan_record_id *rec_id);
void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type);
#endif
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <assert.h>
#include "darshan-logutils.h"
#include "darshan-mpiio-logutils.h"
#include "uthash-1.9.2/src/uthash.h"
int main(int argc, char **argv)
{
int ret;
int i;
char *filename;
char tmp_string[4096];
darshan_fd fd;
struct darshan_header header;
struct darshan_job job;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref;
int mount_count;
char** mnt_pts;
char** fs_types;
time_t tmp_time = 0;
char *token;
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
struct darshan_mpiio_file next_file;
assert(argc == 2);
filename = argv[1];
struct stat sbuf;
stat(filename, &sbuf);
fd = darshan_log_open(filename, "r");
if(!fd)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", filename);
return(-1);
}
/**************************************************************/
/* TODO: some of this code should be shared or in a separate executable
* instead of repeated in each module parser
*/
/* read darshan log header */
ret = darshan_log_getheader(fd, &header);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getheader() failed to read log header.\n");
darshan_log_close(fd);
return(-1);
}
/* read darshan job info */
ret = darshan_log_getjob(fd, &job);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getjob() failed to read job data.\n");
darshan_log_close(fd);
return(-1);
}
/* get the original command line for this job */
ret = darshan_log_getexe(fd, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(fd);
return(-1);
}
/* print job summary */
printf("# darshan log version: %s\n", header.version_string);
printf("# size of MPIIO file statistics: %zu bytes\n", sizeof(struct darshan_mpiio_file));
printf("# size of job statistics: %zu bytes\n", sizeof(struct darshan_job));
printf("# exe: %s\n", tmp_string);
printf("# uid: %" PRId64 "\n", job.uid);
printf("# jobid: %" PRId64 "\n", job.jobid);
printf("# start_time: %" PRId64 "\n", job.start_time);
tmp_time += job.start_time;
printf("# start_time_asci: %s", ctime(&tmp_time));
printf("# end_time: %" PRId64 "\n", job.end_time);
tmp_time = 0;
tmp_time += job.end_time;
printf("# end_time_asci: %s", ctime(&tmp_time));
printf("# nprocs: %" PRId64 "\n", job.nprocs);
printf("# run time: %" PRId64 "\n", job.end_time - job.start_time + 1);
for(token=strtok_r(job.metadata, "\n", &save);
token != NULL;
token=strtok_r(NULL, "\n", &save))
{
char *key;
char *value;
/* NOTE: we intentionally only split on the first = character.
* There may be additional = characters in the value portion
* (for example, when storing mpi-io hints).
*/
strcpy(buffer, token);
key = buffer;
value = index(buffer, '=');
if(!value)
continue;
/* convert = to a null terminator to split key and value */
value[0] = '\0';
value++;
printf("# metadata: %s = %s\n", key, value);
}
/* get the mount information for this log */
ret = darshan_log_getmounts(fd, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmounts() failed to read mount information.\n");
darshan_log_close(fd);
return(-1);
}
/* print table of mounted file systems */
printf("\n# mounted file systems (mount point and fs type)\n");
printf("# -------------------------------------------------------\n");
for(i=0; i<mount_count; i++)
{
printf("# mount entry:\t%s\t%s\n", mnt_pts[i], fs_types[i]);
}
/* read hash of darshan records */
ret = darshan_log_gethash(fd, &rec_hash);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmap() failed to read record map.\n");
darshan_log_close(fd);
return(-1);
}
/* end TODO */
/*******************************************/
ret = darshan_log_get_mpiio_file(fd, &next_file);
if(ret < 0)
{
fprintf(stderr, "darshan_log_get_mpiio_file() failed to read next record.\n");
darshan_log_close(fd);
return(-1);
}
if(ret == 0)
{
printf("# no files opened.\n");
darshan_log_close(fd);
return(0);
}
/* loop over each stored MPIIO file record and print counters */
do
{
/* get the pathname for this record */
HASH_FIND(hlink, rec_hash, &next_file.f_id, sizeof(darshan_record_id), ref);
assert(ref);
for(i=0; i<MPIIO_NUM_INDICES; i++)
{
MPIIO_COUNTER_PRINT(&next_file, i);
}
for(i=0; i<MPIIO_F_NUM_INDICES; i++)
{
MPIIO_F_COUNTER_PRINT(&next_file, i);
}
} while((ret = darshan_log_get_mpiio_file(fd, &next_file)) == 1);
/* free mount info */
for(i=0; i<mount_count; i++)
{
free(mnt_pts[i]);
free(fs_types[i]);
}
if(mount_count > 0)
{
free(mnt_pts);
free(fs_types);
}
darshan_log_close(fd);
return(0);
}
/*
* (C) 2009 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdio.h>
......@@ -15,241 +16,88 @@
#include <getopt.h>
#include <assert.h>
#include "darshan-logutils.h"
#include "uthash-1.9.2/src/uthash.h"
/*
* Options
*/
#define OPTION_BASE (1 << 0) /* darshan log fields */
#define OPTION_TOTAL (1 << 1) /* aggregated fields */
#define OPTION_PERF (1 << 2) /* derived performance */
#define OPTION_FILE (1 << 3) /* file count totals */
#define OPTION_FILE_LIST (1 << 4) /* per-file summaries */
#define OPTION_FILE_LIST_DETAILED (1 << 6) /* per-file summaries with extra detail */
#define OPTION_ALL (\
OPTION_BASE|\
OPTION_TOTAL|\
OPTION_PERF|\
OPTION_FILE|\
OPTION_FILE_LIST|\
OPTION_FILE_LIST_DETAILED)
#define FILETYPE_SHARED (1 << 0)
#define FILETYPE_UNIQUE (1 << 1)
#define FILETYPE_PARTSHARED (1 << 2)
#define max(a,b) (((a) > (b)) ? (a) : (b))
#define max3(a,b,c) (((a) > (b)) ? (((a) > (c)) ? (a) : (c)) : (((b) > (c)) ? (b) : (c)))
/*
* Datatypes
*/
typedef struct hash_entry_s
{
UT_hash_handle hlink;
uint64_t hash;
int64_t type;
int64_t procs;
int64_t counters[CP_NUM_INDICES];
double fcounters[CP_F_NUM_INDICES];
double cumul_time;
double meta_time;
double slowest_time;
char name_suffix[CP_NAME_SUFFIX_LEN+1];
} hash_entry_t;
typedef struct perf_data_s
{
int64_t total_bytes;
double slowest_rank_time;
double slowest_rank_meta_time;
int slowest_rank_rank;
double shared_time_by_cumul;
double shared_time_by_open;
double shared_time_by_open_lastio;
double shared_time_by_slowest;
double shared_meta_time;
double agg_perf_by_cumul;
double agg_perf_by_open;
double agg_perf_by_open_lastio;
double agg_perf_by_slowest;
double *rank_cumul_io_time;
double *rank_cumul_md_time;
} perf_data_t;
typedef struct file_data_s
{
int64_t total;
int64_t total_size;
int64_t total_max;
int64_t read_only;
int64_t read_only_size;
int64_t read_only_max;
int64_t write_only;
int64_t write_only_size;
int64_t write_only_max;
int64_t read_write;
int64_t read_write_size;
int64_t read_write_max;
int64_t unique;
int64_t unique_size;
int64_t unique_max;
int64_t shared;
int64_t shared_size;
int64_t shared_max;
} file_data_t;
/*
* Prototypes
*/
void accum_perf(struct darshan_file *, hash_entry_t *, perf_data_t *);
void calc_perf(struct darshan_job *, hash_entry_t *, perf_data_t *);
void accum_file(struct darshan_job *, struct darshan_file *, hash_entry_t *, file_data_t *);
void calc_file(struct darshan_job *, hash_entry_t *, file_data_t *);
void file_list(struct darshan_job *, hash_entry_t *, int);
int usage (char *exename)
{
fprintf(stderr, "Usage: %s [options] <filename>\n", exename);
fprintf(stderr, " --all : all sub-options are enabled\n");
fprintf(stderr, " --base : darshan log field data [default]\n");
fprintf(stderr, " --file : total file counts\n");
fprintf(stderr, " --file-list : per-file summaries\n");
fprintf(stderr, " --file-list-detailed : per-file summaries with additional detail\n");
fprintf(stderr, " --perf : derived perf data\n");
fprintf(stderr, " --total : aggregated darshan field data\n");
exit(1);
}
int parse_args (int argc, char **argv, char **filename)
{
int index;
int mask;
static struct option long_opts[] =
{
{"all", 0, NULL, OPTION_ALL},
{"base", 0, NULL, OPTION_BASE},
{"file", 0, NULL, OPTION_FILE},
{"file-list", 0, NULL, OPTION_FILE_LIST},
{"file-list-detailed", 0, NULL, OPTION_FILE_LIST_DETAILED},
{"perf", 0, NULL, OPTION_PERF},
{"total", 0, NULL, OPTION_TOTAL},
{"help", 0, NULL, 0},
{0, 0, 0, 0}
};
mask = 0;
while(1)
{
int c = getopt_long(argc, argv, "", long_opts, &index);
if (c == -1) break;
switch(c)
{
case OPTION_ALL:
case OPTION_BASE:
case OPTION_FILE:
case OPTION_FILE_LIST:
case OPTION_FILE_LIST_DETAILED:
case OPTION_PERF:
case OPTION_TOTAL:
mask |= c;
break;
case 0:
case '?':
default:
usage(argv[0]);
break;
}
}
if (optind < argc)
{
*filename = argv[optind];
}
else
{
usage(argv[0]);
}
/* default mask value if none specified */
if (mask == 0)
{
mask = OPTION_BASE;
}
return mask;
}
#include "darshan-logutils.h"
int main(int argc, char **argv)
{
int ret;
int mask;
int i, j;
char *filename;
struct darshan_job job;
struct darshan_file cp_file;
char tmp_string[4096];
time_t tmp_time = 0;
darshan_fd file;
int i;
darshan_fd fd;
struct darshan_header header;
struct darshan_job job;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref;
int mount_count;
int64_t* devs;
char** mnt_pts;
char** fs_types;
int last_rank = 0;
time_t tmp_time = 0;
char *token;
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
int empty_mods = 0;
hash_entry_t *file_hash = NULL;
hash_entry_t *curr = NULL;
hash_entry_t *tmp = NULL;
hash_entry_t total;
perf_data_t pdata;
file_data_t fdata;