Skip to content

GitLab

Commit d4db2298 authored by Shane Snyder's avatar Shane Snyder
Browse files

update current utilities to build properly

parent 093b9f22
Hide whitespace changes
Inline Side-by-side
darshan-util/Makefile.in
View file @ d4db2298
all: libdarshan-util.a darshan-parser
all: libdarshan-util.a darshan-analyzer darshan-convert darshan-parser
DESTDIR =
srcdir = @srcdir@
......@@ -106,8 +106,8 @@ install:: all
install -d $(libdir)
install -d $(includedir)
install -d $(pkgconfigdir)
# install -m 755 darshan-analyzer $(bindir)
# install -m 755 darshan-convert $(bindir)
install -m 755 darshan-analyzer $(bindir)
install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir)
install -m 755 darshan-parser $(bindir)
# install -m 755 $(srcdir)/darshan-summary-per-file.sh $(bindir)
......
darshan-util/darshan-analyzer.c
View file @ d4db2298
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <ftw.h>
#include <zlib.h>
......@@ -13,8 +20,7 @@
char * base = NULL;
int total_single = 0;
int total_multi = 0;
int total_shared = 0;
int total_mpio = 0;
int total_pnet = 0;
int total_hdf5 = 0;
......@@ -27,38 +33,47 @@ int bucket4 = 0;
int bucket5 = 0;
int fail = 0;
int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_pnet, int *used_hdf5, int *used_multi, int *used_single)
int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_pnet, int *used_hdf5, int *used_shared)
{
struct darshan_job job;
struct darshan_file cp_file;
char tmp_string[4096];
darshan_fd zfile;
struct darshan_header header;
struct darshan_job job;
struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD];
struct darshan_mod_logutil_funcs *mpiio_mod = mod_logutils[DARSHAN_MPIIO_MOD];
struct darshan_mod_logutil_funcs *hdf5_mod = mod_logutils[DARSHAN_HDF5_MOD];
struct darshan_mod_logutil_funcs *pnetcdf_mod = mod_logutils[DARSHAN_PNETCDF_MOD];
struct darshan_posix_file *psx_rec;
darshan_record_id rec_id;
int ret;
int f_count;
double total_io_time;
double total_job_time;
assert(psx_mod);
assert(mpiio_mod);
assert(hdf5_mod);
assert(pnetcdf_mod);
zfile = darshan_log_open(fname, "r");
if (zfile == NULL)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", fname);
fprintf(stderr, "darshan_log_open() failed to open %s.\n", fname);
return -1;
}
ret = darshan_log_getjob(zfile, &job);
#if 0
ret = darshan_log_getheader(zfile, &header);
if (ret < 0)
{
perror("darshan_log_getjob");
fprintf(stderr, "%s\n", fname);
fprintf(stderr, "darshan_log_getheader() failed on file %s.\n", fname);
darshan_log_close(zfile);
return -1;
}
ret = darshan_log_getexe(zfile, tmp_string);
ret = darshan_log_getjob(zfile, &job);
if (ret < 0)
{
perror("darshan_log_getexe");
fprintf(stderr, "%s\n", fname);
fprintf(stderr, "darshan_log_getjob() failed on file %s.\n", fname);
darshan_log_close(zfile);
return -1;
}
......@@ -66,14 +81,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
f_count = 0;
total_io_time = 0.0;
while ((ret = darshan_log_getfile(zfile, &job, &cp_file)) == 1)
while ((ret = psx_mod->log_get_record(zfile, (void **)&psx_rec, &rec_id)) == 1)
{
void *tmp_rec;
f_count += 1;
if (cp_file.rank == -1)
*used_single = 1;
else
*used_multi = 1;
if (psx_rec->rank == -1)
*used_shared = 1;
while((ret = mpiio_mod->log_get_record(
*used_mpio += cp_file.counters[CP_INDEP_OPENS];
*used_mpio += cp_file.counters[CP_COLL_OPENS];
......@@ -100,6 +117,7 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
{
*io_ratio = 0.0;
}
#endif
darshan_log_close(zfile);
......@@ -112,20 +130,19 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
int used_mpio = 0;
int used_pnet = 0;
int used_hdf5 = 0;
int used_multi = 0;
int used_single = 0;
int used_shared = 0;
if (typeflag != FTW_F) return 0;
process_log(fpath,&io_ratio,&used_mpio,&used_pnet,&used_hdf5,&used_multi,&used_single);
process_log(fpath,&io_ratio,&used_mpio,&used_pnet,&used_hdf5,&used_shared);
/* XXX */
total_count++;
if (used_mpio > 0) total_mpio++;
if (used_pnet > 0) total_pnet++;
if (used_hdf5 > 0) total_hdf5++;
if (used_single > 0) total_single++;
if (used_multi > 0) total_multi++;
if (used_shared > 0) total_shared++;
if (io_ratio <= BUCKET1)
bucket1++;
......@@ -152,7 +169,7 @@ int main(int argc, char **argv)
if(argc != 2)
{
fprintf(stderr, "Error: bad arguments.\n");
fprintf(stderr, "Error: directory of Darshan logs required as argument.\n");
return(-1);
}
......@@ -165,13 +182,13 @@ int main(int argc, char **argv)
return(-1);
}
printf (" log: %s\n", base);
printf (" total: %d\n", total_count);
printf ("single: %lf [%d]\n", (double)total_single/(double)total_count, total_single);
printf (" multi: %lf [%d]\n", (double)total_multi/(double)total_count, total_multi);
printf (" mpio: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio);
printf (" pnet: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet);
printf (" hdf5: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5);
/* XXX */
printf ("log dir: %s\n", base);
printf (" total: %d\n", total_count);
printf (" single: %lf [%d]\n", (double)total_shared/(double)total_count, total_shared);
printf (" mpio: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio);
printf (" pnet: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet);
printf (" hdf5: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5);
printf ("%.2lf-%.2lf: %d\n", (double)0.0, (double)BUCKET1, bucket1);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET1, (double)BUCKET2, bucket2);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET2, (double)BUCKET3, bucket3);
......@@ -179,3 +196,12 @@ int main(int argc, char **argv)
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET4, (double)100.0, bucket5);
return 0;
}
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
darshan-util/darshan-convert.c
View file @ d4db2298
/*
* (C) 2011 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdio.h>
......@@ -40,8 +41,6 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
int index;
int ret;
*reset_md = 0;
static struct option long_opts[] =
{
{"annotate", 1, NULL, 'a'},
......@@ -53,6 +52,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
{ 0, 0, 0, 0 }
};
*reset_md = 0;
*hash = 0;
while(1)
......@@ -101,6 +101,7 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
return;
}
#if 0
static void reset_md_job(struct darshan_job *job)
{
job->metadata[0] = '\0';
......@@ -181,23 +182,26 @@ void add_annotation (char *annotation,
return;
}
#endif
int main(int argc, char **argv)
{
int ret;
char *infile_name;
char *outfile_name;
struct darshan_header header;
struct darshan_job job;
struct darshan_file cp_file;
char tmp_string[4096];
darshan_fd infile;
darshan_fd outfile;
int i;
int mount_count;
int64_t* devs;
char** mnt_pts;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
char *mod_buf;
int mod_buf_sz;
char** fs_types;
int last_rank = 0;
int obfuscate = 0;
int key = 0;
char *annotation = NULL;
......@@ -213,38 +217,50 @@ int main(int argc, char **argv)
return(-1);
}
/* TODO: safety check that outfile_name doesn't exist; we don't want to
* overwrite something by accident.
*/
outfile = darshan_log_open(outfile_name, "w");
if(!outfile)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", outfile_name);
darshan_log_close(infile);
return(-1);
}
/* read header from input file */
ret = darshan_log_getheader(infile, &header);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read header from input log file %s.\n", infile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
/* TODO: for now this tool is just reading the input file and throwing
* away the data. Need to write the log_put*() functions and use this
* program as a test harness
/* NOTE: we do not write the header to the output file until the end, as
* the mapping data stored in this structure may change in the conversion
* process (particularly, if we are converting between libz/bz2 compression)
*/
/* read job info */
ret = darshan_log_getjob(infile, &job);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read job information from log file.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
#if 0
if (reset_md) reset_md_job(&job);
if (obfuscate) obfuscate_job(key, &job);
if (annotation) add_annotation(annotation, &job);
#endif
ret = darshan_log_putjob(outfile, &job);
if (ret < 0)
{
fprintf(stderr, "Error: unable to write job information to log file.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
......@@ -254,77 +270,97 @@ int main(int argc, char **argv)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
#if 0
if (obfuscate) obfuscate_exe(key, tmp_string);
#endif
ret = darshan_log_putexe(outfile, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
ret = darshan_log_getmounts(infile, &devs, &mnt_pts, &fs_types, &mount_count);
ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
ret = darshan_log_putmounts(outfile, devs, mnt_pts, fs_types, mount_count);
ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write mount information.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
ret = darshan_log_getfile(infile, &job, &cp_file);
ret = darshan_log_gethash(infile, &rec_hash);
if(ret < 0)
{
fprintf(stderr, "Error: failed to process log file.\n");
fflush(stderr);
fprintf(stderr, "Error: unable to read darshan record hash.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
if(ret == 0)
ret = darshan_log_puthash(outfile, rec_hash);
if(ret < 0)
{
goto done;
fprintf(stderr, "Error: unable to write darshan record hash.\n");
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
do
mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!mod_buf)
return(-1);
for(i=0; i<DARSHAN_MAX_MODS; i++)
{
if(cp_file.rank != -1 && cp_file.rank < last_rank)
memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* check each module for any data */
ret = darshan_log_getmod(infile, i, mod_buf, &mod_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: log file contains out of order rank data.\n");
fflush(stderr);
fprintf(stderr, "Error: failed to get module %s data.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
if(cp_file.rank != -1)
last_rank = cp_file.rank;
if(!hash || hash == cp_file.hash)
else if(ret == 0)
{
if (obfuscate) obfuscate_file(key, &cp_file);
ret = darshan_log_putfile(outfile, &job, &cp_file);
if (ret < 0)
{
fprintf(stderr, "Error: failed to write file record.\n");
break;
}
/* skip modules not present in log file */
continue;
}
} while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1);
if(ret < 0)
{
fprintf(stderr, "Error: failed to process log file.\n");
fflush(stderr);
/* we have module data to convert */
ret = darshan_log_putmod(outfile, i, mod_buf, mod_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: failed to put module %s data.\n",
darshan_module_names[i]);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
}
free(mod_buf);
done:
for(i=0; i<mount_count; i++)
{
free(mnt_pts[i]);
......@@ -332,15 +368,38 @@ done:
}
if(mount_count > 0)
{
free(devs);
free(mnt_pts);
free(fs_types);
}
HASH_ITER(hlink, rec_hash, ref, tmp)
{
HASH_DELETE(hlink, rec_hash, ref);
free(ref->rec.name);
free(ref);
}
/* write header to output file */
ret = darshan_log_putheader(outfile);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
darshan_log_close(infile);
darshan_log_close(outfile);
return(-1);
}
darshan_log_close(infile);
darshan_log_close(outfile);
return(ret);
}
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
darshan-util/darshan-parser.c
View file @ d4db2298
......@@ -246,6 +246,8 @@ int main(int argc, char **argv)
if(empty_mods == DARSHAN_MAX_MODS)
printf("# no module data available.\n");
free(mod_buf);
/* free record hash data */
HASH_ITER(hlink, rec_hash, ref, tmp)
{
......@@ -266,7 +268,6 @@ int main(int argc, char **argv)
free(fs_types);
}
free(mod_buf);
darshan_log_close(fd);
return(0);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment