GitLab maintenance scheduled for Today, 2019-12-05, from 17:00 to 18:00 CT - Services will be unavailable during this time.

Commit 7ec53306 authored by Shane Snyder's avatar Shane Snyder

Merge branch 'master' into 'master'

Merge Darshan eXtended Tracing (DXT) to Darshan Main stream

See merge request !2
parents 45c766b9 61272682
/*
* Copyright (C) 2016 Intel Corporation.
* See COPYRIGHT in top-level directory.
*/
#ifndef __DARSHAN_DXT_LOG_FORMAT_H
#define __DARSHAN_DXT_LOG_FORMAT_H
/* current DXT log format version */
#define DXT_POSIX_VER 1
#define DXT_MPIIO_VER 1
#define HOSTNAME_SIZE 64
/*
* DXT, the segment_info structure maintains detailed Segment IO tracing
* information
*/
typedef struct segment_info {
int64_t offset;
int64_t length;
double start_time;
double end_time;
} segment_info;
#define X(a) a,
#undef X
/* file record structure for DXT files. a record is created and stored for
* every DXT file opened by the original application. For the DXT module,
* the record includes:
* - a darshan_base_record structure, which contains the record id & rank
* - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/
struct dxt_file_record {
struct darshan_base_record base_rec;
int64_t shared_record; /* -1 means it is a shared file record */
char hostname[HOSTNAME_SIZE];
int64_t write_count;
int64_t read_count;
};
#endif /* __DARSHAN_DXT_LOG_FORMAT_H */
......@@ -117,6 +117,8 @@ struct darshan_base_record
#include "darshan-bgq-log-format.h"
#include "darshan-lustre-log-format.h"
#include "darshan-stdio-log-format.h"
/* DXT */
#include "darshan-dxt-log-format.h"
/* X-macro for keeping module ordering consistent */
/* NOTE: first val used to define module enum values,
......@@ -136,7 +138,10 @@ struct darshan_base_record
X(DARSHAN_PNETCDF_MOD, "PNETCDF", DARSHAN_PNETCDF_VER, &pnetcdf_logutils) \
X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils) \
X(DARSHAN_LUSTRE_MOD, "LUSTRE", DARSHAN_LUSTRE_VER, &lustre_logutils) \
X(DARSHAN_STDIO_MOD, "STDIO", DARSHAN_STDIO_VER, &stdio_logutils)
X(DARSHAN_STDIO_MOD, "STDIO", DARSHAN_STDIO_VER, &stdio_logutils) \
/* DXT */ \
X(DXT_POSIX_MOD, "X_POSIX", DXT_POSIX_VER, &dxt_posix_logutils) \
X(DXT_MPIIO_MOD, "X_MPIIO", DXT_MPIIO_VER, &dxt_mpiio_logutils)
/* unique identifiers to distinguish between available darshan modules */
/* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
......
......@@ -35,8 +35,8 @@ CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir)
LIBS = -lz @LIBBZ2@
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o lib/darshan-stdio.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po lib/darshan-stdio.po
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o lib/darshan-stdio.o lib/darshan-dxt.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po lib/darshan-stdio.po lib/darshan-dxt.po
ifdef DARSHAN_USE_BGQ
DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o
......@@ -127,6 +127,12 @@ lib/darshan-stdio.o: lib/darshan-stdio.c darshan.h darshan-common.h $(DARSHAN_LO
lib/darshan-stdio.po: lib/darshan-stdio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-stdio-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-dxt.o: lib/darshan-dxt.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-dxt.po: lib/darshan-dxt.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/lookup3.o: lib/lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
......
......@@ -607,6 +607,12 @@ void darshan_core_shutdown()
final_core->log_hdr_p->mod_map[i].len =
gz_fp - final_core->log_hdr_p->mod_map[i].off;
/* XXX: DXT manages its own module memory buffers, so we need to
* explicitly free them
*/
if(i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD)
free(mod_buf);
/* error out if the log append failed */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
......
This diff is collapsed.
......@@ -87,10 +87,17 @@ static void mpiio_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **mpiio_buf, int *mpiio_buf_sz);
/* extern DXT function defs */
extern void dxt_mpiio_write(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
extern void dxt_mpiio_read(darshan_record_id rec_id, int64_t length,
double start_time, double end_time);
static struct mpiio_runtime *mpiio_runtime = NULL;
static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
static int my_rank = -1;
static int enable_dxt_io_trace = 0;
#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
......@@ -98,7 +105,9 @@ static int my_rank = -1;
#define MPIIO_PRE_RECORD() do { \
MPIIO_LOCK(); \
if(!instrumentation_disabled) { \
if(!mpiio_runtime) mpiio_runtime_initialize(); \
if(!mpiio_runtime) { \
mpiio_runtime_initialize(); \
} \
if(mpiio_runtime) break; \
} \
MPIIO_UNLOCK(); \
......@@ -154,6 +163,10 @@ static int my_rank = -1;
if(!rec_ref) break; \
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_read(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -183,6 +196,10 @@ static int my_rank = -1;
if(!rec_ref) break; \
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_mpiio_write(rec_ref->file_rec->base_rec.id, size, __tm1, __tm2); \
} \
DARSHAN_BUCKET_INC(&(rec_ref->file_rec->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
darshan_common_val_counter(&rec_ref->access_root, &rec_ref->access_count, size, \
&(rec_ref->file_rec->counters[MPIIO_ACCESS1_ACCESS]), \
......@@ -844,6 +861,11 @@ static void mpiio_runtime_initialize()
}
memset(mpiio_runtime, 0, sizeof(*mpiio_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("ENABLE_DXT_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
return;
}
......
......@@ -157,11 +157,18 @@ static void posix_shutdown(
MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **posix_buf, int *posix_buf_sz);
/* extern DXT function defs */
extern void dxt_posix_write(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
extern void dxt_posix_read(darshan_record_id rec_id, int64_t offset,
int64_t length, double start_time, double end_time);
static struct posix_runtime *posix_runtime = NULL;
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
static int my_rank = -1;
static int darshan_mem_alignment = 1;
static int enable_dxt_io_trace = 0;
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
......@@ -169,7 +176,9 @@ static int darshan_mem_alignment = 1;
#define POSIX_PRE_RECORD() do { \
POSIX_LOCK(); \
if(!instrumentation_disabled) { \
if(!posix_runtime) posix_runtime_initialize(); \
if(!posix_runtime) { \
posix_runtime_initialize(); \
} \
if(posix_runtime) break; \
} \
POSIX_UNLOCK(); \
......@@ -228,6 +237,10 @@ static int darshan_mem_alignment = 1;
this_offset = __pread_offset; \
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed read tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_read(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
if(this_offset > rec_ref->last_byte_read) \
rec_ref->file_rec->counters[POSIX_SEQ_READS] += 1; \
if(this_offset == (rec_ref->last_byte_read + 1)) \
......@@ -282,6 +295,10 @@ static int darshan_mem_alignment = 1;
this_offset = __pwrite_offset; \
else \
this_offset = rec_ref->offset; \
/* DXT to record detailed write tracing information */ \
if(enable_dxt_io_trace) { \
dxt_posix_write(rec_ref->file_rec->base_rec.id, this_offset, __ret, __tm1, __tm2); \
} \
if(this_offset > rec_ref->last_byte_written) \
rec_ref->file_rec->counters[POSIX_SEQ_WRITES] += 1; \
if(this_offset == (rec_ref->last_byte_written + 1)) \
......@@ -1257,6 +1274,11 @@ static void posix_runtime_initialize()
}
memset(posix_runtime, 0, sizeof(*posix_runtime));
/* check if DXT (Darshan extended tracing) should be enabled */
if (getenv("ENABLE_DXT_IO_TRACE")) {
enable_dxt_io_trace = 1;
}
return;
}
......
all: libdarshan-util.a darshan-null-logutils.o darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
all: libdarshan-util.a darshan-null-logutils.o darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-dxt-parser darshan-merge jenkins-hash-gen
DESTDIR =
srcdir = @srcdir@
......@@ -13,10 +13,10 @@ libdir = $(DESTDIR)@libdir@
pkgconfigdir = $(DESTDIR)$(libdir)/pkgconfig
DARSHAN_LOG_FORMAT = $(srcdir)/../darshan-log-format.h
DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h $(srcdir)/../darshan-hdf5-log-format.h $(srcdir)/../darshan-pnetcdf-log-format.h $(srcdir)/../darshan-bgq-log-format.h $(srcdir)/../darshan-lustre-log-format.h $(srcdir)/../darshan-stdio-log-format.h
DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h darshan-bgq-logutils.h darshan-lustre-logutils.h darshan-stdio-logutils.h
DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o darshan-lustre-logutils.o darshan-stdio-logutils.o
DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po darshan-lustre-logutils.po darshan-stdio-logutils.po
DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h $(srcdir)/../darshan-hdf5-log-format.h $(srcdir)/../darshan-pnetcdf-log-format.h $(srcdir)/../darshan-bgq-log-format.h $(srcdir)/../darshan-lustre-log-format.h $(srcdir)/../darshan-stdio-log-format.h $(srcdir)/../darshan-dxt-log-format.h
DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h darshan-bgq-logutils.h darshan-lustre-logutils.h darshan-stdio-logutils.h darshan-dxt-logutils.h
DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o darshan-lustre-logutils.o darshan-stdio-logutils.o darshan-dxt-logutils.o
DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po darshan-lustre-logutils.po darshan-stdio-logutils.po darshan-dxt-logutils.po
DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
......@@ -91,6 +91,11 @@ darshan-stdio-logutils.o: darshan-stdio-logutils.c darshan-logutils.h darshan-st
darshan-stdio-logutils.po: darshan-stdio-logutils.c darshan-logutils.h darshan-stdio-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-stdio-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
darshan-dxt-logutils.o: darshan-dxt-logutils.c darshan-logutils.h darshan-dxt-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS) -c $< -o $@
darshan-dxt-logutils.po: darshan-dxt-logutils.c darshan-logutils.h darshan-dxt-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS)
ar rcs libdarshan-util.a $^
......@@ -115,6 +120,9 @@ darshan-diff: darshan-diff.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_
darshan-parser: darshan-parser.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $< libdarshan-util.a -o $@ $(LIBS)
darshan-dxt-parser: darshan-dxt-parser.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $< libdarshan-util.a -o $@ $(LIBS)
darshan-merge: darshan-merge.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $< libdarshan-util.a -o $@ $(LIBS)
......@@ -133,6 +141,7 @@ install:: all
install -m 755 darshan-convert $(bindir)
install -m 755 darshan-diff $(bindir)
install -m 755 darshan-parser $(bindir)
install -m 755 darshan-dxt-parser $(bindir)
install -m 755 darshan-merge $(bindir)
install -m 755 $(srcdir)/darshan-summary-per-file.sh $(bindir)
install -m 755 libdarshan-util.a $(libdir)
......@@ -148,6 +157,7 @@ endif
install -m 644 $(srcdir)/darshan-null-logutils.h $(includedir)
install -m 644 $(srcdir)/darshan-lustre-logutils.h $(includedir)
install -m 644 $(srcdir)/darshan-stdio-logutils.h $(includedir)
install -m 644 $(srcdir)/darshan-dxt-logutils.h $(includedir)
install -m 644 $(srcdir)/../darshan-null-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-posix-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-mpiio-log-format.h $(includedir)
......@@ -156,6 +166,7 @@ endif
install -m 644 $(srcdir)/../darshan-bgq-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-lustre-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-stdio-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-dxt-log-format.h $(includedir)
install -d $(includedir)/uthash-1.9.2
install -d $(includedir)/uthash-1.9.2/src
install -m 644 uthash-1.9.2/src/uthash.h $(includedir)/uthash-1.9.2/src/
......@@ -172,7 +183,7 @@ endif
clean::
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-dxt-parser darshan-merge jenkins-hash-gen
distclean:: clean
rm -f darshan-util-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile darshan-job-summary/bin/darshan-job-summary.pl maint/darshan-util.pc
......
This diff is collapsed.
/*
* Copyright (C) 2015 University of Chicago.
* Copyright (C) 2016 Intel Corporation.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_DXT_LOG_UTILS_H
#define __DARSHAN_DXT_LOG_UTILS_H
extern struct darshan_mod_logutil_funcs dxt_posix_logutils;
extern struct darshan_mod_logutil_funcs dxt_mpiio_logutils;
void dxt_log_print_posix_file(void *file_rec, char *file_name,
char *mnt_pt, char *fs_type, struct lustre_record_ref *rec_ref);
void dxt_log_print_mpiio_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
#endif
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <assert.h>
#include "uthash-1.9.2/src/uthash.h"
#include "darshan-logutils.h"
int usage (char *exename)
{
fprintf(stderr, "Usage: %s <filename>\n", exename);
exit(1);
}
int main(int argc, char **argv)
{
int ret;
int i, j;
char *filename;
char *comp_str;
char tmp_string[4096] = {0};
darshan_fd fd;
struct darshan_job job;
struct darshan_name_record_ref *name_hash = NULL;
struct darshan_name_record_ref *ref, *tmp_ref;
int mount_count;
struct darshan_mnt_info *mnt_data_array;
time_t tmp_time = 0;
int64_t run_time = 0;
char *token;
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
struct lustre_record_ref *lustre_rec_ref, *tmp_lustre_rec_ref;
struct lustre_record_ref *lustre_rec_hash = NULL;
char *mod_buf = NULL;
if (argc != 2)
usage(argv[0]);
filename = argv[1];
fd = darshan_log_open(filename);
if (!fd)
return(-1);
/* read darshan job info */
ret = darshan_log_get_job(fd, &job);
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
}
/* get the original command line for this job */
ret = darshan_log_get_exe(fd, tmp_string);
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
}
/* get the mount information for this log */
ret = darshan_log_get_mounts(fd, &mnt_data_array, &mount_count);
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
}
/* read hash of darshan records */
ret = darshan_log_get_namehash(fd, &name_hash);
if (ret < 0)
{
darshan_log_close(fd);
return(-1);
}
/* print any warnings related to this log file version */
darshan_log_print_version_warnings(fd->version);
if (fd->comp_type == DARSHAN_ZLIB_COMP)
comp_str = "ZLIB";
else if (fd->comp_type == DARSHAN_BZIP2_COMP)
comp_str = "BZIP2";
else if (fd->comp_type == DARSHAN_NO_COMP)
comp_str = "NONE";
else
comp_str = "UNKNOWN";
/* print job summary */
printf("# darshan log version: %s\n", fd->version);
printf("# compression method: %s\n", comp_str);
printf("# exe: %s\n", tmp_string);
printf("# uid: %" PRId64 "\n", job.uid);
printf("# jobid: %" PRId64 "\n", job.jobid);
printf("# start_time: %" PRId64 "\n", job.start_time);
tmp_time += job.start_time;
printf("# start_time_asci: %s", ctime(&tmp_time));
printf("# end_time: %" PRId64 "\n", job.end_time);
tmp_time = 0;
tmp_time += job.end_time;
printf("# end_time_asci: %s", ctime(&tmp_time));
printf("# nprocs: %" PRId64 "\n", job.nprocs);
if (job.end_time >= job.start_time)
run_time = job.end_time - job.start_time + 1;
printf("# run time: %" PRId64 "\n", run_time);
for (token = strtok_r(job.metadata, "\n", &save);
token != NULL;
token = strtok_r(NULL, "\n", &save))
{
char *key;
char *value;
/* NOTE: we intentionally only split on the first = character.
* There may be additional = characters in the value portion
* (for example, when storing mpi-io hints).
*/
strcpy(buffer, token);
key = buffer;
value = index(buffer, '=');
if(!value)
continue;
/* convert = to a null terminator to split key and value */
value[0] = '\0';
value++;
printf("# metadata: %s = %s\n", key, value);
}
/* print breakdown of each log file region's contribution to file size */
printf("\n# log file regions\n");
printf("# -------------------------------------------------------\n");
printf("# header: %zu bytes (uncompressed)\n", sizeof(struct darshan_header));
printf("# job data: %zu bytes (compressed)\n", fd->job_map.len);
printf("# record table: %zu bytes (compressed)\n", fd->name_map.len);
for (i = 0; i < DARSHAN_MAX_MODS; i++)
{
if (fd->mod_map[i].len)
{
printf("# %s module: %zu bytes (compressed), ver=%d\n",
darshan_module_names[i], fd->mod_map[i].len, fd->mod_ver[i]);
}
}
/* print table of mounted file systems */
printf("\n# mounted file systems (mount point and fs type)\n");
printf("# -------------------------------------------------------\n");
for (i = 0; i < mount_count; i++)
{
printf("# mount entry:\t%s\t%s\n", mnt_data_array[i].mnt_path,
mnt_data_array[i].mnt_type);
}
/* just exit if there is no DXT data in this log file */
if(fd->mod_map[DXT_POSIX_MOD].len == 0 && fd->mod_map[DXT_MPIIO_MOD].len == 0)
{
printf("\n# no DXT module data available for this Darshan log.\n");
goto cleanup;
}
for (i = 0; i < DARSHAN_MAX_MODS; i++)
{
struct darshan_base_record *base_rec;
/* check each module for any data */
if (fd->mod_map[i].len == 0)
continue;
/* skip modules with no logutil definitions */
else if (!mod_logutils[i])
{
fprintf(stderr, "Warning: no log utility handlers defined "
"for module %s, SKIPPING.\n", darshan_module_names[i]);
continue;
}
if (i == DXT_POSIX_MOD || i == DXT_MPIIO_MOD) {
printf("\n# ***************************************************\n");
printf("# %s module data\n", darshan_module_names[i]);
printf("# ***************************************************\n");
}
else if (i != DARSHAN_LUSTRE_MOD)
continue;
/* print warning if this module only stored partial data */
if(DARSHAN_MOD_FLAG_ISSET(fd->partial_flag, i))
printf("\n# *WARNING*: The %s module contains incomplete data!\n"
"# This happens when a module runs out of\n"
"# memory to store new record data.\n",
darshan_module_names[i]);
/* loop over each of this module's records and print them */
while(1)
{
char *mnt_pt = NULL;
char *fs_type = NULL;
char *rec_name = NULL;
if (i == DARSHAN_LUSTRE_MOD) {
lustre_rec_ref = malloc(sizeof(*lustre_rec_ref));
assert(lustre_rec_ref);
memset(lustre_rec_ref, 0, sizeof(*lustre_rec_ref));
ret = mod_logutils[i]->log_get_record(fd,
(void **)&(lustre_rec_ref->rec));
} else {
ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
}
if (ret < 1)
{
if (ret == -1)
{
fprintf(stderr, "Error: failed to parse %s module record.\n",
darshan_module_names[i]);
goto cleanup;
}
break;
}
if(i == DARSHAN_LUSTRE_MOD) {
HASH_ADD(hlink, lustre_rec_hash, rec->base_rec.id,
sizeof(darshan_record_id), lustre_rec_ref);
continue;
}
base_rec = (struct darshan_base_record *)mod_buf;
/* get the pathname for this record */
HASH_FIND(hlink, name_hash, &(base_rec->id),
sizeof(darshan_record_id), ref);
if (ref)
{
rec_name = ref->name_record->name;
/* get mount point and fs type associated with this record */
for (j = 0; j < mount_count; j++)
{
if(strncmp(mnt_data_array[j].mnt_path, rec_name,
strlen(mnt_data_array[j].mnt_path)) == 0)
{
mnt_pt = mnt_data_array[j].mnt_path;
fs_type = mnt_data_array[j].mnt_type;
break;
}
}
}
if (!mnt_pt)
mnt_pt = "UNKNOWN";
if (!fs_type)
fs_type = "UNKNOWN";
if (i == DXT_POSIX_MOD) {
/* look for corresponding lustre record and print DXT data */
HASH_FIND(hlink, lustre_rec_hash, &(base_rec->id),
sizeof(darshan_record_id), lustre_rec_ref);
dxt_log_print_posix_file(mod_buf, rec_name,
mnt_pt, fs_type, lustre_rec_ref);
} else if (i == DXT_MPIIO_MOD){
dxt_log_print_mpiio_file(mod_buf, rec_name,
mnt_pt, fs_type);
}
free(mod_buf);
mod_buf = NULL;
}
}
ret = 0;
cleanup:
darshan_log_close(fd);
/* free record hash data */
HASH_ITER(hlink, name_hash, ref, tmp_ref)
{
HASH_DELETE(hlink, name_hash, ref);
free(ref->name_record);
free(ref);
}
/* free lustre record data */
HASH_ITER(hlink, lustre_rec_hash, lustre_rec_ref, tmp_lustre_rec_ref)
{
HASH_DELETE(hlink, lustre_rec_hash, lustre_rec_ref);
free(lustre_rec_ref->rec);
free(lustre_rec_ref);
}
/* free mount info */
if (mount_count > 0)
{
free(mnt_data_array);
}
return(ret);
}
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
......@@ -20,7 +20,7 @@
/* Maximum size of a record - Lustre OST lists can get huge, but 81920 is enough
* for 10K OSTs
*/
#define DEF_MOD_BUF_SIZE 81920 /* 640 KiB */
#define DEF_MOD_BUF_SIZE 81920
struct darshan_fd_int_state;
......@@ -55,6 +55,13 @@ struct darshan_name_record_ref
UT_hash_handle hlink;
};
/* DXT */
struct lustre_record_ref
{
struct darshan_lustre_record *rec;
UT_hash_handle hlink;
};
struct darshan_mnt_info
{
char mnt_type[DARSHAN_EXE_LEN];
......@@ -128,6 +135,9 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[];
#include "darshan-lustre-logutils.h"
#include "darshan-stdio-logutils.h"
/* DXT */
#include "darshan-dxt-logutils.h"
darshan_fd darshan_log_open(const char *name);
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
int partial_flag);
......
......@@ -61,6 +61,16 @@ static int darshan_log_get_lustre_record(darshan_fd fd, void** lustre_buf_p)
else if(ret < sizeof(struct darshan_lustre_record))
return(0);
/* swap bytes if necessary */
if(fd->swap_flag)
{
DARSHAN_BSWAP64(&tmp_rec.base_rec.id);
DARSHAN_BSWAP64(&tmp_rec.base_rec.rank);
for(i=0; i<LUSTRE_NUM_INDICES; i++)
DARSHAN_BSWAP64(&tmp_rec.counters[i]);
DARSHAN_BSWAP64(&tmp_rec.ost_ids[0]);
}
if(*lustre_buf_p == NULL)
{
rec = malloc(LUSTRE_RECORD_SIZE(tmp_rec.counters[LUSTRE_STRIPE_WIDTH]));
......@@ -69,16 +79,6 @@ static int darshan_log_get_lustre_record(darshan_fd fd, void** lustre_buf_p)
}
memcpy(rec, &tmp_rec, sizeof(struct darshan_lustre_record));
/* swap bytes if necessary */
if(fd->swap_flag)
{
DARSHAN_BSWAP64(&rec->base_rec.id);
DARSHAN_BSWAP64(&rec->base_rec.rank);
for(i=0; i<LUSTRE_NUM_INDICES; i++)
DARSHAN_BSWAP64(&rec->counters[i]);
DARSHAN_BSWAP64(&(rec->ost_ids[0]));
}
/* now read the rest of the record */
if ( rec->counters[LUSTRE_STRIPE_WIDTH] > 1 ) {
ret = darshan_log_get_mod(
......@@ -87,16 +87,16 @@ static int darshan_log_get_lustre_record(darshan_fd fd, void** lustre_buf_p)
(void*)(&(rec->ost_ids[1])),
(rec->counters[LUSTRE_STRIPE_WIDTH] - 1)*sizeof(OST_ID)
);
if(ret < 0)
if(ret < (rec->counters[LUSTRE_STRIPE_WIDTH] - 1)*sizeof(OST_ID))
ret = -1;
else if(ret < (rec->counters[LUSTRE_STRIPE_WIDTH] - 1)*sizeof(OST_ID))
ret = 0;
else
{
ret = 1;
/* swap bytes if necessary */
if ( fd->swap_flag )
for (i = 1; i < rec->counters[LUSTRE_STRIPE_WIDTH]; i++ )
DARSHAN_BSWAP64(&(rec->ost_ids[i]));
/* swap bytes if necessary */
if ( fd->swap_flag )
for (i = 1; i < rec->counters[LUSTRE_STRIPE_WIDTH]; i++ )
DARSHAN_BSWAP64(&(rec->ost_ids[i]));
}
}