logutil backwards compatibility for name records

darshan-log-format.h

@@ -20,7 +20,7 @@
 #endif
 
 /* update this on file format changes */
-#define DARSHAN_LOG_VERSION "3.00"
+#define DARSHAN_LOG_VERSION "3.01"
 
 /* magic number for validating output files and checking byte order */
 #define DARSHAN_MAGIC_NR 6567223

darshan-util/Makefile.in

@@ -17,8 +17,8 @@ DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../d
 DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h
 DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o
 DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po
-
-#XXX BGQ
+DARSHAN_STATIC_COMPAT_OBJS = compat/darshan-logutils-compat.o
+DARSHAN_DYNAMIC_COMPAT_OBJS = compat/darshan-logutils-compat.po
 
 DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
 
@@ -32,7 +32,7 @@ cp_zlib_include_flags = @__DARSHAN_ZLIB_INCLUDE_FLAGS@
 
 # deliberately avoid large file support for host side utilities to avoid
 # potentially buggy libz 64 bit offset support
-CFLAGS = -I . -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@ 
+CFLAGS = -I . -I .. -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@ 
 CFLAGS_SHARED = $(CFLAGS) -shared -fpic -DPIC 
 LDFLAGS=@LDFLAGS@
 
@@ -83,10 +83,15 @@ darshan-bgq-logutils.o: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-lo
 darshan-bgq-logutils.po: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h | uthash-1.9.2
 	$(CC) $(CFLAGS_SHARED) -c  $< -o $@
 
-libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS)
+compat/darshan-logutils-compat.o: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+	$(CC) $(CFLAGS) -c  $< -o $@
+#compat/darshan-logutils-compat.po: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+#	$(CC) $(CFLAGS_SHARED) -c  $< -o $@
+
+libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS) $(DARSHAN_STATIC_COMPAT_OBJS)
 	ar rcs libdarshan-util.a $^
 
-libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS)
+libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS) 
 	$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
 	
 jenkins-hash-gen: jenkins-hash-gen.c lookup3.o
@@ -160,7 +165,7 @@ endif
 
 
 clean::
-	rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
+	rm -f *.o *.po compat/*.o compat/*.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
 
 distclean:: clean
 	rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl

darshan-util/compat/darshan-logutils-compat.c

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _GNU_SOURCE
+#include "darshan-util-config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include "darshan-logutils-compat.h"
+
+int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
+    int swap_flag, struct darshan_name_record_ref **hash)
+{
+    struct darshan_name_record_ref *ref;
+    char *buf_ptr;
+    darshan_record_id *rec_id_ptr;
+    uint32_t *path_len_ptr;
+    char *path_ptr;
+    int rec_len;
+    int buf_processed = 0;
+
+    /* work through the name record buffer -- deserialize the mapping data and
+     * add to the output hash table
+     * NOTE: these mapping pairs are variable in length, so we have to be able
+     * to handle incomplete mappings temporarily here
+     */
+    buf_ptr = name_rec_buf;
+    while(buf_len > (sizeof(darshan_record_id) + sizeof(uint32_t)))
+    {
+        /* see if we have enough buf space to read in the next full record */
+        path_len_ptr = (uint32_t *)(buf_ptr + sizeof(darshan_record_id));
+        if(swap_flag)
+            DARSHAN_BSWAP32(path_len_ptr);
+        rec_len = sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr;
+
+        /* we need to read more before we continue deserializing */
+        if(buf_len < rec_len)
+            break;
+
+        /* get pointers for each field of this darshan record */
+        /* NOTE: darshan record hash serialization method: 
+         *          ... darshan_record_id | (uint32_t) path_len | path ...
+         */
+        rec_id_ptr = (darshan_record_id *)buf_ptr;
+        path_ptr = (char *)(buf_ptr + sizeof(darshan_record_id) + sizeof(uint32_t));
+
+        if(swap_flag)
+            /* we need to sort out endianness issues before deserializing */
+            DARSHAN_BSWAP64(rec_id_ptr);
+
+        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
+        if(!ref)
+        {
+            ref = malloc(sizeof(*ref));
+            if(!ref)
+                return(-1);
+
+            ref->name_record = malloc(rec_len - sizeof(uint32_t) + 1);
+            if(!ref->name_record)
+            {
+                free(ref);
+                return(-1);
+            }
+
+            /* transform the serialized name record into the zero-length
+             * array structure darshan uses to track name records
+             */
+            ref->name_record->id = *rec_id_ptr;
+            memcpy(ref->name_record->name, path_ptr, *path_len_ptr);
+            ref->name_record->name[*path_len_ptr] = '\0';
+
+            /* add this record to the hash */
+            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
+        }
+
+        buf_ptr += rec_len;
+        buf_len -= rec_len;
+        buf_processed += rec_len;
+    }
+
+    return(buf_processed);
+}
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */

darshan-util/compat/darshan-logutils-compat.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_LOG_UTILS_COMPAT_H
+#define __DARSHAN_LOG_UTILS_COMPAT_H
+
+#include "darshan-logutils.h"
+
+int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
+    int swap_flag, struct darshan_name_record_ref **hash);
+
+#endif

darshan-util/darshan-logutils.c

@@ -18,6 +18,7 @@
 #include <errno.h>
 
 #include "darshan-logutils.h"
+#include "compat/darshan-logutils-compat.h"
 
 /* default input buffer size for decompression algorithm */
 #define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */
@@ -61,12 +62,18 @@ struct darshan_fd_int_state
     char *exe_mnt_data;
     /* whether previous file operations have failed */
     int err;
+    /* log format version-specific function calls for getting
+     * data from the log file
+     */
+    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
 
     /* compression/decompression stream read/write state */
     struct darshan_dz_state dz;
 };
 
 static int darshan_mnt_info_cmp(const void *a, const void *b);
+static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
+    int swap_flag, struct darshan_name_record_ref **hash);
 static int darshan_log_get_header(darshan_fd fd);
 static int darshan_log_put_header(darshan_fd fd);
 static int darshan_log_seek(darshan_fd fd, off_t offset);
@@ -493,7 +500,8 @@ int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_arra
 
 /* darshan_log_get_namehash()
  *
- * read the hash of name records from the darshan log file
+ * read the set of name records from the darshan log file and add to the
+ * given hash table
  *
  * returns 0 on success, -1 on failure
  */
@@ -501,14 +509,11 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
 {
     struct darshan_fd_int_state *state = fd->state;
     char *name_rec_buf;
-    char *tmp_p;
     int name_rec_buf_sz;
     int read;
     int read_req_sz;
-    struct darshan_name_record_ref *ref;
-    struct darshan_name_record *name_rec;
-    int buf_rem = 0;
-    int rec_len;
+    int buf_len = 0;
+    int buf_processed;
 
     assert(state);
 
@@ -531,78 +536,29 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
         /* read chunks of the darshan record id -> name mapping from log file,
          * constructing a hash table in the process
          */
-        read_req_sz = name_rec_buf_sz - buf_rem;
+        read_req_sz = name_rec_buf_sz - buf_len;
         read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
-            name_rec_buf + buf_rem, read_req_sz);
+            name_rec_buf + buf_len, read_req_sz);
         if(read < 0)
         {
             fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
             free(name_rec_buf);
             return(-1);
         }
-        buf_rem += read;
-
-        /* work through the name record buffer -- deserialize the mapping data and
-         * add to the output hash table
-         * NOTE: these mapping pairs are variable in length, so we have to be able
-         * to handle incomplete mappings temporarily here
-         */
-        name_rec = (struct darshan_name_record *)name_rec_buf;
-        while(buf_rem > sizeof(darshan_record_id) + 1)
-        {
-            if(strnlen(name_rec->name, buf_rem - sizeof(darshan_record_id)) ==
-                (buf_rem - sizeof(darshan_record_id)))
-            {
-                /* if this record name's terminating null character is not
-                 * present, we need to read more of the buffer before continuing
-                 */
-                break;
-            }
-
-            if(fd->swap_flag)
-            {
-                /* we need to sort out endianness issues before deserializing */
-                DARSHAN_BSWAP64(&(name_rec->id));
-            }
+        buf_len += read;
 
-            HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
-            if(!ref)
-            {
-                rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
-                ref = malloc(sizeof(*ref));
-                if(!ref)
-                {
-                    free(name_rec_buf);
-                    return(-1);
-                }
-                ref->name_record = malloc(rec_len);
-                if(!ref->name_record)
-                {
-                    free(ref);
-                    free(name_rec_buf);
-                    return(-1);
-                }
-
-                /* copy the name record over from the hash buffer */
-                memcpy(ref->name_record, name_rec, rec_len);
-
-                /* add this record to the hash */
-                HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
-            }
-
-            tmp_p = (char *)name_rec + rec_len;
-            name_rec = (struct darshan_name_record *)tmp_p;
-            buf_rem -= rec_len;
-        }
+        /* extract any name records in the buffer */
+        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
 
         /* copy any leftover data to beginning of buffer to parse next */
-        memcpy(name_rec_buf, name_rec, buf_rem);
+        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
+        buf_len -= buf_processed;
 
         /* we keep reading until we get a short read informing us we have
          * read all of the record hash
          */
     } while(read == read_req_sz);
-    assert(buf_rem == 0);
+    assert(buf_len == 0);
 
     free(name_rec_buf);
     return(0);
@@ -813,6 +769,69 @@ static int darshan_mnt_info_cmp(const void *a, const void *b)
         return(0);
 }
 
+static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
+    int swap_flag, struct darshan_name_record_ref **hash)
+{
+    struct darshan_name_record_ref *ref;
+    struct darshan_name_record *name_rec;
+    char *tmp_p;
+    int buf_processed = 0;
+    int rec_len;
+
+    /* work through the name record buffer -- deserialize the record data
+     * and add to the output hash table
+     * NOTE: these mapping pairs are variable in length, so we have to be able
+     * to handle incomplete mappings temporarily here
+     */
+    name_rec = (struct darshan_name_record *)name_rec_buf;
+    while(buf_len > sizeof(darshan_record_id) + 1)
+    {
+        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
+            (buf_len - sizeof(darshan_record_id)))
+        {
+            /* if this record name's terminating null character is not
+             * present, we need to read more of the buffer before continuing
+             */
+            break;
+        }
+        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
+
+        if(swap_flag)
+        {
+            /* we need to sort out endianness issues before deserializing */
+            DARSHAN_BSWAP64(&(name_rec->id));
+        }
+
+        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
+        if(!ref)
+        {
+            ref = malloc(sizeof(*ref));
+            if(!ref)
+                return(-1);
+
+            ref->name_record = malloc(rec_len);
+            if(!ref->name_record)
+            {
+                free(ref);
+                return(-1);
+            }
+
+            /* copy the name record over from the hash buffer */
+            memcpy(ref->name_record, name_rec, rec_len);
+
+            /* add this record to the hash */
+            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
+        }
+
+        tmp_p = (char *)name_rec + rec_len;
+        name_rec = (struct darshan_name_record *)tmp_p;
+        buf_len -= rec_len;
+        buf_processed += rec_len;
+    }
+
+    return(buf_processed);
+}
+
 /* read the header of the darshan log and set internal fd data structures
  * NOTE: this is the only portion of the darshan log that is uncompressed
  *
@@ -840,10 +859,19 @@ static int darshan_log_get_header(darshan_fd fd)
     }
 
     /* other log file versions can be detected and handled here */
-    if(strcmp(fd->version, "3.00"))
+    if(strcmp(fd->version, "3.00") == 0)
+    {
+        fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
+    }
+    else if(strcmp(fd->version, "3.01") == 0)
+    {
+        fd->state->get_namerecs = darshan_log_get_namerecs;
+    }
+    else
     {
         fprintf(stderr, "Error: incompatible darshan file.\n");
-        fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
+        fprintf(stderr, "Error: expected version %s, but got %s\n",
+            DARSHAN_LOG_VERSION, fd->version);
         return(-1);
     }