Commit d4db2298 authored by Shane Snyder's avatar Shane Snyder

update current utilities to build properly

parent 093b9f22
all: libdarshan-util.a darshan-parser all: libdarshan-util.a darshan-analyzer darshan-convert darshan-parser
DESTDIR = DESTDIR =
srcdir = @srcdir@ srcdir = @srcdir@
...@@ -106,8 +106,8 @@ install:: all ...@@ -106,8 +106,8 @@ install:: all
install -d $(libdir) install -d $(libdir)
install -d $(includedir) install -d $(includedir)
install -d $(pkgconfigdir) install -d $(pkgconfigdir)
# install -m 755 darshan-analyzer $(bindir) install -m 755 darshan-analyzer $(bindir)
# install -m 755 darshan-convert $(bindir) install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir) # install -m 755 darshan-diff $(bindir)
install -m 755 darshan-parser $(bindir) install -m 755 darshan-parser $(bindir)
# install -m 755 $(srcdir)/darshan-summary-per-file.sh $(bindir) # install -m 755 $(srcdir)/darshan-summary-per-file.sh $(bindir)
......
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <assert.h>
#include <ftw.h> #include <ftw.h>
#include <zlib.h> #include <zlib.h>
...@@ -13,8 +20,7 @@ ...@@ -13,8 +20,7 @@
char * base = NULL; char * base = NULL;
int total_single = 0; int total_shared = 0;
int total_multi = 0;
int total_mpio = 0; int total_mpio = 0;
int total_pnet = 0; int total_pnet = 0;
int total_hdf5 = 0; int total_hdf5 = 0;
...@@ -27,38 +33,47 @@ int bucket4 = 0; ...@@ -27,38 +33,47 @@ int bucket4 = 0;
int bucket5 = 0; int bucket5 = 0;
int fail = 0; int fail = 0;
int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_pnet, int *used_hdf5, int *used_multi, int *used_single) int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_pnet, int *used_hdf5, int *used_shared)
{ {
struct darshan_job job;
struct darshan_file cp_file;
char tmp_string[4096];
darshan_fd zfile; darshan_fd zfile;
struct darshan_header header;
struct darshan_job job;
struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD];
struct darshan_mod_logutil_funcs *mpiio_mod = mod_logutils[DARSHAN_MPIIO_MOD];
struct darshan_mod_logutil_funcs *hdf5_mod = mod_logutils[DARSHAN_HDF5_MOD];
struct darshan_mod_logutil_funcs *pnetcdf_mod = mod_logutils[DARSHAN_PNETCDF_MOD];
struct darshan_posix_file *psx_rec;
darshan_record_id rec_id;
int ret; int ret;
int f_count; int f_count;
double total_io_time; double total_io_time;
double total_job_time; double total_job_time;
assert(psx_mod);
assert(mpiio_mod);
assert(hdf5_mod);
assert(pnetcdf_mod);
zfile = darshan_log_open(fname, "r"); zfile = darshan_log_open(fname, "r");
if (zfile == NULL) if (zfile == NULL)
{ {
fprintf(stderr, "darshan_log_open() failed to open %s\n.", fname); fprintf(stderr, "darshan_log_open() failed to open %s.\n", fname);
return -1; return -1;
} }
ret = darshan_log_getjob(zfile, &job); #if 0
ret = darshan_log_getheader(zfile, &header);
if (ret < 0) if (ret < 0)
{ {
perror("darshan_log_getjob"); fprintf(stderr, "darshan_log_getheader() failed on file %s.\n", fname);
fprintf(stderr, "%s\n", fname);
darshan_log_close(zfile); darshan_log_close(zfile);
return -1; return -1;
} }
ret = darshan_log_getexe(zfile, tmp_string); ret = darshan_log_getjob(zfile, &job);
if (ret < 0) if (ret < 0)
{ {
perror("darshan_log_getexe"); fprintf(stderr, "darshan_log_getjob() failed on file %s.\n", fname);
fprintf(stderr, "%s\n", fname);
darshan_log_close(zfile); darshan_log_close(zfile);
return -1; return -1;
} }
...@@ -66,14 +81,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -66,14 +81,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
f_count = 0; f_count = 0;
total_io_time = 0.0; total_io_time = 0.0;
while ((ret = darshan_log_getfile(zfile, &job, &cp_file)) == 1) while ((ret = psx_mod->log_get_record(zfile, (void **)&psx_rec, &rec_id)) == 1)
{ {
void *tmp_rec;
f_count += 1; f_count += 1;
if (cp_file.rank == -1) if (psx_rec->rank == -1)
*used_single = 1; *used_shared = 1;
else
*used_multi = 1;
while((ret = mpiio_mod->log_get_record(
*used_mpio += cp_file.counters[CP_INDEP_OPENS]; *used_mpio += cp_file.counters[CP_INDEP_OPENS];
*used_mpio += cp_file.counters[CP_COLL_OPENS]; *used_mpio += cp_file.counters[CP_COLL_OPENS];
...@@ -100,6 +117,7 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p ...@@ -100,6 +117,7 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
{ {
*io_ratio = 0.0; *io_ratio = 0.0;
} }
#endif
darshan_log_close(zfile); darshan_log_close(zfile);
...@@ -112,20 +130,19 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -112,20 +130,19 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
int used_mpio = 0; int used_mpio = 0;
int used_pnet = 0; int used_pnet = 0;
int used_hdf5 = 0; int used_hdf5 = 0;
int used_multi = 0; int used_shared = 0;
int used_single = 0;
if (typeflag != FTW_F) return 0; if (typeflag != FTW_F) return 0;
process_log(fpath,&io_ratio,&used_mpio,&used_pnet,&used_hdf5,&used_multi,&used_single); process_log(fpath,&io_ratio,&used_mpio,&used_pnet,&used_hdf5,&used_shared);
/* XXX */
total_count++; total_count++;
if (used_mpio > 0) total_mpio++; if (used_mpio > 0) total_mpio++;
if (used_pnet > 0) total_pnet++; if (used_pnet > 0) total_pnet++;
if (used_hdf5 > 0) total_hdf5++; if (used_hdf5 > 0) total_hdf5++;
if (used_single > 0) total_single++; if (used_shared > 0) total_shared++;
if (used_multi > 0) total_multi++;
if (io_ratio <= BUCKET1) if (io_ratio <= BUCKET1)
bucket1++; bucket1++;
...@@ -152,7 +169,7 @@ int main(int argc, char **argv) ...@@ -152,7 +169,7 @@ int main(int argc, char **argv)
if(argc != 2) if(argc != 2)
{ {
fprintf(stderr, "Error: bad arguments.\n"); fprintf(stderr, "Error: directory of Darshan logs required as argument.\n");
return(-1); return(-1);
} }
...@@ -165,13 +182,13 @@ int main(int argc, char **argv) ...@@ -165,13 +182,13 @@ int main(int argc, char **argv)
return(-1); return(-1);
} }
printf (" log: %s\n", base); /* XXX */
printf (" total: %d\n", total_count); printf ("log dir: %s\n", base);
printf ("single: %lf [%d]\n", (double)total_single/(double)total_count, total_single); printf (" total: %d\n", total_count);
printf (" multi: %lf [%d]\n", (double)total_multi/(double)total_count, total_multi); printf (" single: %lf [%d]\n", (double)total_shared/(double)total_count, total_shared);
printf (" mpio: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio); printf (" mpio: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio);
printf (" pnet: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet); printf (" pnet: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet);
printf (" hdf5: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5); printf (" hdf5: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5);
printf ("%.2lf-%.2lf: %d\n", (double)0.0, (double)BUCKET1, bucket1); printf ("%.2lf-%.2lf: %d\n", (double)0.0, (double)BUCKET1, bucket1);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET1, (double)BUCKET2, bucket2); printf ("%.2lf-%.2lf: %d\n", (double)BUCKET1, (double)BUCKET2, bucket2);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET2, (double)BUCKET3, bucket3); printf ("%.2lf-%.2lf: %d\n", (double)BUCKET2, (double)BUCKET3, bucket3);
...@@ -179,3 +196,12 @@ int main(int argc, char **argv) ...@@ -179,3 +196,12 @@ int main(int argc, char **argv)
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET4, (double)100.0, bucket5); printf ("%.2lf-%.2lf: %d\n", (double)BUCKET4, (double)100.0, bucket5);
return 0; return 0;
} }
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/* /*
* (C) 2011 by Argonne National Laboratory. * Copyright (C) 2015 University of Chicago.
* See COPYRIGHT in top-level directory. * See COPYRIGHT notice in top-level directory.
*
*/ */
#include <stdio.h> #include <stdio.h>
...@@ -40,8 +41,6 @@ void parse_args (int argc, char **argv, char **infile, char **outfile, ...@@ -40,8 +41,6 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
int index; int index;
int ret; int ret;
*reset_md = 0;
static struct option long_opts[] = static struct option long_opts[] =
{ {
{"annotate", 1, NULL, 'a'}, {"annotate", 1, NULL, 'a'},
...@@ -53,6 +52,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile, ...@@ -53,6 +52,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
{ 0, 0, 0, 0 } { 0, 0, 0, 0 }
}; };
*reset_md = 0;
*hash = 0; *hash = 0;
while(1) while(1)
...@@ -101,6 +101,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile, ...@@ -101,6 +101,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
return; return;
} }
#if 0
static void reset_md_job(struct darshan_job *job) static void reset_md_job(struct darshan_job *job)
{ {
job->metadata[0] = '\0'; job->metadata[0] = '\0';
...@@ -181,23 +182,26 @@ void add_annotation (char *annotation, ...@@ -181,23 +182,26 @@ void add_annotation (char *annotation,
return; return;
} }
#endif
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
int ret; int ret;
char *infile_name; char *infile_name;
char *outfile_name; char *outfile_name;
struct darshan_header header;
struct darshan_job job; struct darshan_job job;
struct darshan_file cp_file;
char tmp_string[4096]; char tmp_string[4096];
darshan_fd infile; darshan_fd infile;
darshan_fd outfile; darshan_fd outfile;
int i; int i;
int mount_count; int mount_count;
int64_t* devs;
char** mnt_pts; char** mnt_pts;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
char *mod_buf;
int mod_buf_sz;
char** fs_types; char** fs_types;
int last_rank = 0;
int obfuscate = 0; int obfuscate = 0;
int key = 0; int key = 0;
char *annotation = NULL; char *annotation = NULL;
...@@ -213,38 +217,50 @@ int main(int argc, char **argv) ...@@ -213,38 +217,50 @@ int main(int argc, char **argv)
return(-1); return(-1);
} }
/* TODO: safety check that outfile_name doesn't exist; we don't want to
* overwrite something by accident.
*/
outfile = darshan_log_open(outfile_name, "w"); outfile = darshan_log_open(outfile_name, "w");
if(!outfile) if(!outfile)
{ {
fprintf(stderr, "darshan_log_open() failed to open %s\n.", outfile_name); fprintf(stderr, "darshan_log_open() failed to open %s\n.", outfile_name);
darshan_log_close(infile);
return(-1);
}
/* read header from input file */
ret = darshan_log_getheader(infile, &header);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read header from input log file %s.\n", infile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1); return(-1);
} }
/* TODO: for now this tool is just reading the input file and throwing /* NOTE: we do not write the header to the output file until the end, as
* away the data. Need to write the log_put*() functions and use this * the mapping data stored in this structure may change in the conversion
* program as a test harness * process (particularly, if we are converting between libz/bz2 compression)
*/ */
/* read job info */ /* read job info */
ret = darshan_log_getjob(infile, &job); ret = darshan_log_getjob(infile, &job);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read job information from log file.\n"); fprintf(stderr, "Error: unable to read job information from log file.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile);
return(-1); return(-1);
} }
#if 0
if (reset_md) reset_md_job(&job); if (reset_md) reset_md_job(&job);
if (obfuscate) obfuscate_job(key, &job); if (obfuscate) obfuscate_job(key, &job);
if (annotation) add_annotation(annotation, &job); if (annotation) add_annotation(annotation, &job);
#endif
ret = darshan_log_putjob(outfile, &job); ret = darshan_log_putjob(outfile, &job);
if (ret < 0) if (ret < 0)
{ {
fprintf(stderr, "Error: unable to write job information to log file.\n"); fprintf(stderr, "Error: unable to write job information to log file.\n");
darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
} }
...@@ -254,77 +270,97 @@ int main(int argc, char **argv) ...@@ -254,77 +270,97 @@ int main(int argc, char **argv)
{ {
fprintf(stderr, "Error: unable to read trailing job information.\n"); fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile);
return(-1); return(-1);
} }
#if 0
if (obfuscate) obfuscate_exe(key, tmp_string); if (obfuscate) obfuscate_exe(key, tmp_string);
#endif
ret = darshan_log_putexe(outfile, tmp_string); ret = darshan_log_putexe(outfile, tmp_string);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to write trailing job information.\n"); fprintf(stderr, "Error: unable to write trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
} }
ret = darshan_log_getmounts(infile, &devs, &mnt_pts, &fs_types, &mount_count); ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to read trailing job information.\n"); fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile);
return(-1); return(-1);
} }
ret = darshan_log_putmounts(outfile, devs, mnt_pts, fs_types, mount_count); ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: unable to write mount information.\n"); fprintf(stderr, "Error: unable to write mount information.\n");
darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(-1); return(-1);
} }
ret = darshan_log_getfile(infile, &job, &cp_file); ret = darshan_log_gethash(infile, &rec_hash);
if(ret < 0) if(ret < 0)
{ {
fprintf(stderr, "Error: failed to process log file.\n"); fprintf(stderr, "Error: unable to read darshan record hash.\n");
fflush(stderr); darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
} }
if(ret == 0)
ret = darshan_log_puthash(outfile, rec_hash);
if(ret < 0)
{ {
goto done; fprintf(stderr, "Error: unable to write darshan record hash.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
} }
do mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!mod_buf)
return(-1);
for(i=0; i<DARSHAN_MAX_MODS; i++)
{ {
if(cp_file.rank != -1 && cp_file.rank < last_rank) memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* check each module for any data */
ret = darshan_log_getmod(infile, i, mod_buf, &mod_buf_sz);
if(ret < 0)
{ {
fprintf(stderr, "Error: log file contains out of order rank data.\n"); fprintf(stderr, "Error: failed to get module %s data.\n",
fflush(stderr); darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1); return(-1);
} }
if(cp_file.rank != -1) else if(ret == 0)
last_rank = cp_file.rank;
if(!hash || hash == cp_file.hash)
{ {
if (obfuscate) obfuscate_file(key, &cp_file); /* skip modules not present in log file */
continue;
ret = darshan_log_putfile(outfile, &job, &cp_file);
if (ret < 0)
{
fprintf(stderr, "Error: failed to write file record.\n");
break;
}
} }
} while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1);
if(ret < 0) /* we have module data to convert */
{ ret = darshan_log_putmod(outfile, i, mod_buf, mod_buf_sz);
fprintf(stderr, "Error: failed to process log file.\n"); if(ret < 0)
fflush(stderr); {
fprintf(stderr, "Error: failed to put module %s data.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
} }
free(mod_buf);
done:
for(i=0; i<mount_count; i++) for(i=0; i<mount_count; i++)
{ {
free(mnt_pts[i]); free(mnt_pts[i]);
...@@ -332,15 +368,38 @@ done: ...@@ -332,15 +368,38 @@ done:
} }
if(mount_count > 0) if(mount_count > 0)
{ {
free(devs);
free(mnt_pts); free(mnt_pts);
free(fs_types); free(fs_types);
} }
HASH_ITER(hlink, rec_hash, ref, tmp)
{
HASH_DELETE(hlink, rec_hash, ref);
free(ref->rec.name);
free(ref);
}
/* write header to output file */
ret = darshan_log_putheader(outfile);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
darshan_log_close(infile); darshan_log_close(infile);
darshan_log_close(outfile); darshan_log_close(outfile);
return(ret); return(ret);
} }
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
...@@ -246,6 +246,8 @@ int main(int argc, char **argv) ...@@ -246,6 +246,8 @@ int main(int argc, char **argv)
if(empty_mods == DARSHAN_MAX_MODS) if(empty_mods == DARSHAN_MAX_MODS)
printf("# no module data available.\n"); printf("# no module data available.\n");
free(mod_buf);
/* free record hash data */ /* free record hash data */
HASH_ITER(hlink, rec_hash, ref, tmp) HASH_ITER(hlink, rec_hash, ref, tmp)
{ {
...@@ -266,7 +268,6 @@ int main(int argc, char **argv) ...@@ -266,7 +268,6 @@ int main(int argc, char **argv)
free(fs_types); free(fs_types);
} }
free(mod_buf);
darshan_log_close(fd); darshan_log_close(fd);
return(0); return(0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment