Commit 694c6a9a authored by Shane Snyder's avatar Shane Snyder

bunch of bug fixes for gzip madness

parent 35dcc69a
......@@ -53,12 +53,6 @@ static char *darshan_module_names[] =
"PNETCDF"
};
enum darshan_comp_type
{
DARSHAN_GZ_COMP,
DARSHAN_BZ2_COMP, /* TODO: no bz2 support util side, yet */
};
struct darshan_log_map
{
uint64_t off;
......@@ -69,7 +63,6 @@ struct darshan_header
{
char version_string[8];
int64_t magic_nr;
unsigned char comp_type;
struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
};
......
......@@ -30,6 +30,7 @@
/* in memory structure to keep up with job level data */
struct darshan_core_runtime
{
struct darshan_header log_header;
struct darshan_job log_job;
char exe[DARSHAN_EXE_LEN+1];
struct darshan_core_record_ref *rec_hash;
......
This diff is collapsed.
all: darshan-posix-parser darshan-util-lib
all: darshan-util-lib darshan-posix-parser
#all: darshan-parser darshan-convert darshan-diff darshan-analyzer darshan-log-params darshan-util-lib
DESTDIR =
......@@ -42,8 +42,27 @@ mktestdir::
uthash-1.9.2:
tar xjvf $(srcdir)/extern/uthash-1.9.2.tar.bz2
darshan-posix-parser: darshan-posix-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o
$(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
darshan-logutils.o: darshan-logutils.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
$(CC) $(CFLAGS) -c $< -o $@
darshan-logutils.po: darshan-logutils.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
darshan-posix-logutils.o: darshan-posix-logutils.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS) -c $< -o $@
darshan-posix-logutils.po: darshan-posix-logutils.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
libdarshan-util.so: darshan-logutils.po darshan-posix-logutils.po
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
darshan-util-lib: darshan-logutils.o darshan-posix-logutils.o
ar rcs libdarshan-util.a $^
jenkins: util/bin/jenkins.o lookup3.o
$(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
darshan-posix-parser: darshan-posix-parser.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h darshan-util-lib | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) darshan-posix-parser.c libdarshan-util.a -o $@ $(LIBS)
#darshan-parser: darshan-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o | uthash-1.9.2
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
......@@ -57,23 +76,10 @@ darshan-posix-parser: darshan-posix-parser.c $(DARSHAN_LOG_FORMAT) darshan-logut
#darshan-log-params: darshan-log-params.c $(DARSHAN_LOG_FORMAT)
# $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ $(LIBS)
jenkins: util/bin/jenkins.o lookup3.o
$(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
#darshan-diff: darshan-diff.o $(DARSHAN_LOG_FORMAT) darshan-logutils.o darshan-logutils.h
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
#darshan-diff.o: darshan-diff.c
# $(CC) $(CFLAGS) -c $< -o $@
darshan-logutils.o: darshan-logutils.c | uthash-1.9.2
$(CC) $(CFLAGS) -c $< -o $@
darshan-logutils.po: darshan-logutils.c
$(CC) $(CFLAGS_SHARED) -c $< -o $@
libdarshan-util.so: darshan-logutils.po
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
darshan-util-lib: darshan-logutils.o
ar rcs libdarshan-util.a $<
#test/gztest: test/gztest.c mktestdir
# $(CC) $(CFLAGS) $(LDFLAGS) -lz $< -o $@
......@@ -90,7 +96,7 @@ install:: all
install -d $(includedir)
install -d $(pkgconfigdir)
# install -m 755 darshan-parser $(bindir)
install -m 755 darshan-posix-parser $(bindir)
# install -m 755 darshan-posix-parser $(bindir)
# install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir)
# install -m 755 darshan-analyzer $(bindir)
......@@ -99,21 +105,21 @@ install:: all
ifeq ($(DARSHAN_ENABLE_SHARED),1)
install -m 755 libdarshan-util.so $(libdir)
endif
install -m 644 $(srcdir)/darshan-logutils.h $(includedir)
install -m 644 $(DARSHAN_LOG_FORMAT) $(includedir)
# install -m 644 $(srcdir)/darshan-logutils.h $(includedir)
# install -m 644 $(DARSHAN_LOG_FORMAT) $(includedir)
# install -m 755 darshan-job-summary/bin/darshan-job-summary.pl $(bindir)
install -d $(libdir)/TeX
install -m 644 $(srcdir)/darshan-job-summary/lib/TeX/Encode.pm $(libdir)/TeX/
install -d $(libdir)/Number
install -d $(libdir)/Number/Bytes
install -m 644 $(srcdir)/darshan-job-summary/lib/Number/Bytes/Human.pm $(libdir)/Number/Bytes
install -d $(datarootdir)
# install -d $(libdir)/TeX
# install -m 644 $(srcdir)/darshan-job-summary/lib/TeX/Encode.pm $(libdir)/TeX/
# install -d $(libdir)/Number
# install -d $(libdir)/Number/Bytes
# install -m 644 $(srcdir)/darshan-job-summary/lib/Number/Bytes/Human.pm $(libdir)/Number/Bytes
# install -d $(datarootdir)
# install -m 644 $(srcdir)/darshan-job-summary/share/* $(datarootdir)
install -m 644 maint/darshan-util.pc $(pkgconfigdir)
clean::
rm -f *.o *.a darshan-parser darshan-convert darshan-diff darshan-analyzer darshan-log-params
rm -f *.o *.a darshan-posix-parser
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
......
This diff is collapsed.
......@@ -6,10 +6,26 @@
#ifndef __DARSHAN_LOG_UTILS_H
#define __DARSHAN_LOG_UTILS_H
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif
#include "darshan-log-format.h"
#include "darshan-posix-log-format.h"
#include "uthash-1.9.2/src/uthash.h"
/* default to a compression buffer size of 4 MiB */
#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
struct darshan_fd_s
{
gzFile gzf;
int64_t pos;
char version[8];
int swap_flag;
struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
};
typedef struct darshan_fd_s* darshan_fd;
struct darshan_record_ref
......@@ -21,12 +37,12 @@ struct darshan_record_ref
darshan_fd darshan_log_open(const char *name, const char* mode);
int darshan_log_getheader(darshan_fd file, struct darshan_header *header);
int darshan_log_getjob(darshan_fd file, struct darshan_job *job);
int darshan_log_gethash(darshan_fd file, struct darshan_record_ref **hash);
int darshan_log_getexe(darshan_fd fd, char *buf);
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
char*** fs_types, int* count);
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
void **mod_buf, int *mod_buf_sz);
int darshan_log_gethash(darshan_fd file, struct darshan_record_ref **hash);
int darshan_log_get_moddat(darshan_fd fd, darshan_module_id mod_id,
void *moddat_buf, int moddat_buf_sz);
void darshan_log_close(darshan_fd file);
/* convenience macros for printing out counters */
......
......@@ -15,7 +15,7 @@
#include <getopt.h>
#include <assert.h>
#include "darshan-logutils.h"
#include "darshan-posix-logutils.h"
#include "uthash-1.9.2/src/uthash.h"
int main(int argc, char **argv)
......@@ -24,11 +24,11 @@ int main(int argc, char **argv)
int i;
char *filename;
char tmp_string[4096];
darshan_fd file;
darshan_fd fd;
struct darshan_header header;
struct darshan_job job;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
struct darshan_record_ref *ref;
int mount_count;
char** mnt_pts;
char** fs_types;
......@@ -36,8 +36,7 @@ int main(int argc, char **argv)
char *token;
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
struct darshan_posix_file *posix_mod_buf;
int posix_mod_buf_sz = 0;
struct darshan_posix_file next_file;
assert(argc == 2);
filename = argv[1];
......@@ -45,37 +44,37 @@ int main(int argc, char **argv)
struct stat sbuf;
stat(filename, &sbuf);
file = darshan_log_open(filename, "r");
if(!file)
fd = darshan_log_open(filename, "r");
if(!fd)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", filename);
return(-1);
}
/* read darshan log header */
ret = darshan_log_getheader(file, &header);
ret = darshan_log_getheader(fd, &header);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getheader() failed to read log header.\n");
darshan_log_close(file);
darshan_log_close(fd);
return(-1);
}
/* read darshan job info */
ret = darshan_log_getjob(file, &job);
ret = darshan_log_getjob(fd, &job);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getjob() failed to read job data.\n");
darshan_log_close(file);
darshan_log_close(fd);
return(-1);
}
/* get the original command line for this job */
ret = darshan_log_getexe(file, tmp_string);
ret = darshan_log_getexe(fd, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
darshan_log_close(file);
darshan_log_close(fd);
return(-1);
}
......@@ -117,11 +116,11 @@ int main(int argc, char **argv)
}
/* get the mount information for this log */
ret = darshan_log_getmounts(file, &mnt_pts, &fs_types, &mount_count);
ret = darshan_log_getmounts(fd, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmounts() failed to read mount information.\n");
darshan_log_close(file);
darshan_log_close(fd);
return(-1);
}
......@@ -133,63 +132,47 @@ int main(int argc, char **argv)
printf("# mount entry:\t%s\t%s\n", mnt_pts[i], fs_types[i]);
}
uint64_t job_size = header.rec_map.off - sizeof(struct darshan_header);
printf("\nDarshan log file module sizes:\n");
printf("\tHEADER size = %"PRIu64"\n\tJOB size = %"PRIu64"\n\tREC_HASH size = %"PRIu64"\n",
sizeof(struct darshan_header), job_size, header.rec_map.len);
printf("\t%s MODULE size = %"PRIu64"\n", darshan_module_names[DARSHAN_POSIX_MOD],
header.mod_map[DARSHAN_POSIX_MOD].len);
/* read hash of darshan records */
ret = darshan_log_gethash(file, &rec_hash);
ret = darshan_log_gethash(fd, &rec_hash);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmap() failed to read record map.\n");
darshan_log_close(file);
darshan_log_close(fd);
return(-1);
}
printf("\n*** FILE RECORD DATA ***\n");
/* get the log data for the POSIX module */
ret = darshan_log_getmod(file, DARSHAN_POSIX_MOD, (void**)&posix_mod_buf, &posix_mod_buf_sz);
if(ret < 0 || posix_mod_buf_sz % sizeof(struct darshan_posix_file))
ret = darshan_log_get_posix_file(fd, &next_file);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmod() failed to read POSIX module data.\n");
darshan_log_close(file);
fprintf(stderr, "darshan_log_get_posix_file() failed to read next record.\n");
darshan_log_close(fd);
return(-1);
}
if(posix_mod_buf_sz == 0)
if(ret == 0)
{
/* it looks like the app didn't open any files */
printf("# no files opened.\n");
darshan_log_close(file);
printf("# no files opened.");
darshan_log_close(fd);
return(0);
}
/* TODO: move this functionality somewhere else so it can be reused */
/* loop over the POSIX file records and print out counters */
for(i = 0; i < (posix_mod_buf_sz / sizeof(struct darshan_posix_file)); i++)
/* loop over each stored POSIX file record and print counters */
i = 1;
do
{
struct darshan_posix_file next_rec = posix_mod_buf[i];
DARSHAN_BSWAP64(&(next_rec.f_id));
DARSHAN_BSWAP64(&(next_rec.rank));
DARSHAN_BSWAP64(&(next_rec.counters[CP_POSIX_OPENS]));
DARSHAN_BSWAP64(&(next_rec.fcounters[CP_F_OPEN_TIMESTAMP]));
DARSHAN_BSWAP64(&(next_rec.fcounters[CP_F_CLOSE_TIMESTAMP]));
/* get the pathname for this record */
HASH_FIND(hlink, rec_hash, &next_rec.f_id, sizeof(darshan_record_id), ref);
HASH_FIND(hlink, rec_hash, &next_file.f_id, sizeof(darshan_record_id), ref);
assert(ref);
printf("\tRecord %d: id=%"PRIu64" (path=%s, rank=%"PRId64")\n",
i, next_rec.f_id, ref->rec.name, next_rec.rank);
i, next_file.f_id, ref->rec.name, next_file.rank);
printf("\t\tPOSIX_OPENS:\t%"PRIu64"\n\t\tF_OPEN_TIMESTAMP:\t%lf\n\t\tF_CLOSE_TIMESTAMP:\t%lf\n",
next_rec.counters[CP_POSIX_OPENS], next_rec.fcounters[CP_F_OPEN_TIMESTAMP],
next_rec.fcounters[CP_F_CLOSE_TIMESTAMP]);
}
next_file.counters[CP_POSIX_OPENS], next_file.fcounters[CP_F_OPEN_TIMESTAMP],
next_file.fcounters[CP_F_CLOSE_TIMESTAMP]);
i++;
} while((ret = darshan_log_get_posix_file(fd, &next_file)) == 1);
/* free mount info */
for(i=0; i<mount_count; i++)
......@@ -203,7 +186,7 @@ int main(int argc, char **argv)
free(fs_types);
}
darshan_log_close(file);
darshan_log_close(fd);
return(0);
}
......@@ -277,6 +277,7 @@ char* darshan_clean_file_path(
The `darshan_clean_file_path` function just cleans up the input path string, converting
relative paths to absolute paths and suppressing any potential noise within the string.
The address of the new string is returned and should be freed by the user.
* _path_ is the input path string to be cleaned up.
......@@ -346,7 +347,7 @@ to store the record structure in a hash table or some other structure.
==== Build modifications
Text
Text.
== Other resources
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment