Commit f6769e56 authored by Shane Snyder's avatar Shane Snyder

logutil backwards compatibility for name records

parent f5ff61de
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#endif #endif
/* update this on file format changes */ /* update this on file format changes */
#define DARSHAN_LOG_VERSION "3.00" #define DARSHAN_LOG_VERSION "3.01"
/* magic number for validating output files and checking byte order */ /* magic number for validating output files and checking byte order */
#define DARSHAN_MAGIC_NR 6567223 #define DARSHAN_MAGIC_NR 6567223
......
...@@ -17,8 +17,8 @@ DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../d ...@@ -17,8 +17,8 @@ DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../d
DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h
DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o
DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po
DARSHAN_STATIC_COMPAT_OBJS = compat/darshan-logutils-compat.o
#XXX BGQ DARSHAN_DYNAMIC_COMPAT_OBJS = compat/darshan-logutils-compat.po
DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@ DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
...@@ -32,7 +32,7 @@ cp_zlib_include_flags = @__DARSHAN_ZLIB_INCLUDE_FLAGS@ ...@@ -32,7 +32,7 @@ cp_zlib_include_flags = @__DARSHAN_ZLIB_INCLUDE_FLAGS@
# deliberately avoid large file support for host side utilities to avoid # deliberately avoid large file support for host side utilities to avoid
# potentially buggy libz 64 bit offset support # potentially buggy libz 64 bit offset support
CFLAGS = -I . -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@ CFLAGS = -I . -I .. -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@
CFLAGS_SHARED = $(CFLAGS) -shared -fpic -DPIC CFLAGS_SHARED = $(CFLAGS) -shared -fpic -DPIC
LDFLAGS=@LDFLAGS@ LDFLAGS=@LDFLAGS@
...@@ -83,10 +83,15 @@ darshan-bgq-logutils.o: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-lo ...@@ -83,10 +83,15 @@ darshan-bgq-logutils.o: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-lo
darshan-bgq-logutils.po: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h | uthash-1.9.2 darshan-bgq-logutils.po: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@ $(CC) $(CFLAGS_SHARED) -c $< -o $@
libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS) compat/darshan-logutils-compat.o: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
$(CC) $(CFLAGS) -c $< -o $@
#compat/darshan-logutils-compat.po: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
# $(CC) $(CFLAGS_SHARED) -c $< -o $@
libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS) $(DARSHAN_STATIC_COMPAT_OBJS)
ar rcs libdarshan-util.a $^ ar rcs libdarshan-util.a $^
libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS) libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS)
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS) $(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
jenkins-hash-gen: jenkins-hash-gen.c lookup3.o jenkins-hash-gen: jenkins-hash-gen.c lookup3.o
...@@ -160,7 +165,7 @@ endif ...@@ -160,7 +165,7 @@ endif
clean:: clean::
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen rm -f *.o *.po compat/*.o compat/*.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
distclean:: clean distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
......
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#define _GNU_SOURCE
#include "darshan-util-config.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <inttypes.h>
#include <sys/types.h>
#include "darshan-logutils-compat.h"
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
int swap_flag, struct darshan_name_record_ref **hash)
{
struct darshan_name_record_ref *ref;
char *buf_ptr;
darshan_record_id *rec_id_ptr;
uint32_t *path_len_ptr;
char *path_ptr;
int rec_len;
int buf_processed = 0;
/* work through the name record buffer -- deserialize the mapping data and
* add to the output hash table
* NOTE: these mapping pairs are variable in length, so we have to be able
* to handle incomplete mappings temporarily here
*/
buf_ptr = name_rec_buf;
while(buf_len > (sizeof(darshan_record_id) + sizeof(uint32_t)))
{
/* see if we have enough buf space to read in the next full record */
path_len_ptr = (uint32_t *)(buf_ptr + sizeof(darshan_record_id));
if(swap_flag)
DARSHAN_BSWAP32(path_len_ptr);
rec_len = sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr;
/* we need to read more before we continue deserializing */
if(buf_len < rec_len)
break;
/* get pointers for each field of this darshan record */
/* NOTE: darshan record hash serialization method:
* ... darshan_record_id | (uint32_t) path_len | path ...
*/
rec_id_ptr = (darshan_record_id *)buf_ptr;
path_ptr = (char *)(buf_ptr + sizeof(darshan_record_id) + sizeof(uint32_t));
if(swap_flag)
/* we need to sort out endianness issues before deserializing */
DARSHAN_BSWAP64(rec_id_ptr);
HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
if(!ref)
{
ref = malloc(sizeof(*ref));
if(!ref)
return(-1);
ref->name_record = malloc(rec_len - sizeof(uint32_t) + 1);
if(!ref->name_record)
{
free(ref);
return(-1);
}
/* transform the serialized name record into the zero-length
* array structure darshan uses to track name records
*/
ref->name_record->id = *rec_id_ptr;
memcpy(ref->name_record->name, path_ptr, *path_len_ptr);
ref->name_record->name[*path_len_ptr] = '\0';
/* add this record to the hash */
HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
}
buf_ptr += rec_len;
buf_len -= rec_len;
buf_processed += rec_len;
}
return(buf_processed);
}
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_LOG_UTILS_COMPAT_H
#define __DARSHAN_LOG_UTILS_COMPAT_H
#include "darshan-logutils.h"
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
int swap_flag, struct darshan_name_record_ref **hash);
#endif
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <errno.h> #include <errno.h>
#include "darshan-logutils.h" #include "darshan-logutils.h"
#include "compat/darshan-logutils-compat.h"
/* default input buffer size for decompression algorithm */ /* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */ #define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */
...@@ -61,12 +62,18 @@ struct darshan_fd_int_state ...@@ -61,12 +62,18 @@ struct darshan_fd_int_state
char *exe_mnt_data; char *exe_mnt_data;
/* whether previous file operations have failed */ /* whether previous file operations have failed */
int err; int err;
/* log format version-specific function calls for getting
* data from the log file
*/
int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
/* compression/decompression stream read/write state */ /* compression/decompression stream read/write state */
struct darshan_dz_state dz; struct darshan_dz_state dz;
}; };
static int darshan_mnt_info_cmp(const void *a, const void *b); static int darshan_mnt_info_cmp(const void *a, const void *b);
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
int swap_flag, struct darshan_name_record_ref **hash);
static int darshan_log_get_header(darshan_fd fd); static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd); static int darshan_log_put_header(darshan_fd fd);
static int darshan_log_seek(darshan_fd fd, off_t offset); static int darshan_log_seek(darshan_fd fd, off_t offset);
...@@ -493,7 +500,8 @@ int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_arra ...@@ -493,7 +500,8 @@ int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_arra
/* darshan_log_get_namehash() /* darshan_log_get_namehash()
* *
* read the hash of name records from the darshan log file * read the set of name records from the darshan log file and add to the
* given hash table
* *
* returns 0 on success, -1 on failure * returns 0 on success, -1 on failure
*/ */
...@@ -501,14 +509,11 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has ...@@ -501,14 +509,11 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
{ {
struct darshan_fd_int_state *state = fd->state; struct darshan_fd_int_state *state = fd->state;
char *name_rec_buf; char *name_rec_buf;
char *tmp_p;
int name_rec_buf_sz; int name_rec_buf_sz;
int read; int read;
int read_req_sz; int read_req_sz;
struct darshan_name_record_ref *ref; int buf_len = 0;
struct darshan_name_record *name_rec; int buf_processed;
int buf_rem = 0;
int rec_len;
assert(state); assert(state);
...@@ -531,78 +536,29 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has ...@@ -531,78 +536,29 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
/* read chunks of the darshan record id -> name mapping from log file, /* read chunks of the darshan record id -> name mapping from log file,
* constructing a hash table in the process * constructing a hash table in the process
*/ */
read_req_sz = name_rec_buf_sz - buf_rem; read_req_sz = name_rec_buf_sz - buf_len;
read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID, read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
name_rec_buf + buf_rem, read_req_sz); name_rec_buf + buf_len, read_req_sz);
if(read < 0) if(read < 0)
{ {
fprintf(stderr, "Error: failed to read name hash from darshan log file.\n"); fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
free(name_rec_buf); free(name_rec_buf);
return(-1); return(-1);
} }
buf_rem += read; buf_len += read;
/* work through the name record buffer -- deserialize the mapping data and
* add to the output hash table
* NOTE: these mapping pairs are variable in length, so we have to be able
* to handle incomplete mappings temporarily here
*/
name_rec = (struct darshan_name_record *)name_rec_buf;
while(buf_rem > sizeof(darshan_record_id) + 1)
{
if(strnlen(name_rec->name, buf_rem - sizeof(darshan_record_id)) ==
(buf_rem - sizeof(darshan_record_id)))
{
/* if this record name's terminating null character is not
* present, we need to read more of the buffer before continuing
*/
break;
}
if(fd->swap_flag)
{
/* we need to sort out endianness issues before deserializing */
DARSHAN_BSWAP64(&(name_rec->id));
}
HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref); /* extract any name records in the buffer */
if(!ref) buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
{
rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
ref = malloc(sizeof(*ref));
if(!ref)
{
free(name_rec_buf);
return(-1);
}
ref->name_record = malloc(rec_len);
if(!ref->name_record)
{
free(ref);
free(name_rec_buf);
return(-1);
}
/* copy the name record over from the hash buffer */
memcpy(ref->name_record, name_rec, rec_len);
/* add this record to the hash */
HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
}
tmp_p = (char *)name_rec + rec_len;
name_rec = (struct darshan_name_record *)tmp_p;
buf_rem -= rec_len;
}
/* copy any leftover data to beginning of buffer to parse next */ /* copy any leftover data to beginning of buffer to parse next */
memcpy(name_rec_buf, name_rec, buf_rem); memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
buf_len -= buf_processed;
/* we keep reading until we get a short read informing us we have /* we keep reading until we get a short read informing us we have
* read all of the record hash * read all of the record hash
*/ */
} while(read == read_req_sz); } while(read == read_req_sz);
assert(buf_rem == 0); assert(buf_len == 0);
free(name_rec_buf); free(name_rec_buf);
return(0); return(0);
...@@ -813,6 +769,69 @@ static int darshan_mnt_info_cmp(const void *a, const void *b) ...@@ -813,6 +769,69 @@ static int darshan_mnt_info_cmp(const void *a, const void *b)
return(0); return(0);
} }
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
int swap_flag, struct darshan_name_record_ref **hash)
{
struct darshan_name_record_ref *ref;
struct darshan_name_record *name_rec;
char *tmp_p;
int buf_processed = 0;
int rec_len;
/* work through the name record buffer -- deserialize the record data
* and add to the output hash table
* NOTE: these mapping pairs are variable in length, so we have to be able
* to handle incomplete mappings temporarily here
*/
name_rec = (struct darshan_name_record *)name_rec_buf;
while(buf_len > sizeof(darshan_record_id) + 1)
{
if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
(buf_len - sizeof(darshan_record_id)))
{
/* if this record name's terminating null character is not
* present, we need to read more of the buffer before continuing
*/
break;
}
rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
if(swap_flag)
{
/* we need to sort out endianness issues before deserializing */
DARSHAN_BSWAP64(&(name_rec->id));
}
HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
if(!ref)
{
ref = malloc(sizeof(*ref));
if(!ref)
return(-1);
ref->name_record = malloc(rec_len);
if(!ref->name_record)
{
free(ref);
return(-1);
}
/* copy the name record over from the hash buffer */
memcpy(ref->name_record, name_rec, rec_len);
/* add this record to the hash */
HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
}
tmp_p = (char *)name_rec + rec_len;
name_rec = (struct darshan_name_record *)tmp_p;
buf_len -= rec_len;
buf_processed += rec_len;
}
return(buf_processed);
}
/* read the header of the darshan log and set internal fd data structures /* read the header of the darshan log and set internal fd data structures
* NOTE: this is the only portion of the darshan log that is uncompressed * NOTE: this is the only portion of the darshan log that is uncompressed
* *
...@@ -840,10 +859,19 @@ static int darshan_log_get_header(darshan_fd fd) ...@@ -840,10 +859,19 @@ static int darshan_log_get_header(darshan_fd fd)
} }
/* other log file versions can be detected and handled here */ /* other log file versions can be detected and handled here */
if(strcmp(fd->version, "3.00")) if(strcmp(fd->version, "3.00") == 0)
{
fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
}
else if(strcmp(fd->version, "3.01") == 0)
{
fd->state->get_namerecs = darshan_log_get_namerecs;
}
else
{ {
fprintf(stderr, "Error: incompatible darshan file.\n"); fprintf(stderr, "Error: incompatible darshan file.\n");
fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION); fprintf(stderr, "Error: expected version %s, but got %s\n",
DARSHAN_LOG_VERSION, fd->version);
return(-1); return(-1);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment