fixes #130

The uncompressed header size for darshan logs has been increased from 1 KiB
to 4 KiB, which allows room for more metadata annotations


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@858 3b7491f3-a168-0410-bf4b-c445ed680a29

ChangeLog

@@ -25,6 +25,8 @@ darshan-2.2.9-pre1
   - Ability to disable shared-file reduction by setting the
     DARSHAN_DISABLE_SHARED_REDUCTION environment variable.
   - More thorough output from darshan-parser --perf, suggested by Huong Luu.
+  - Increased uncompressed log header size from 1 KiB to 4 KiB to support
+    additional annotations.
 
 darshan-2.2.8
 =============

darshan-log-format.h

@@ -19,13 +19,13 @@
 #endif
 
 /* update this on file format changes */
-#define CP_VERSION "2.03"
+#define CP_VERSION "2.04"
 
 /* magic number for validating output files and checking byte order */
 #define CP_MAGIC_NR 6567223
 
 /* size (in bytes) of job record */
-#define CP_JOB_RECORD_SIZE 1024
+#define CP_JOB_RECORD_SIZE 4096
 
 /* max length of exe string within job record (not counting '\0') */
 #define CP_EXE_LEN (CP_JOB_RECORD_SIZE - sizeof(struct darshan_job) - 1)

darshan-util/darshan-logutils.c

@@ -34,6 +34,7 @@ struct darshan_fd_s
     char version[10];
     int job_struct_size;
     char* name;
+    int COMPAT_CP_EXE_LEN;
 };
 
 /* isn't there a clever c way to avoid this? */
@@ -219,10 +220,14 @@ int (*getfile_internal)(darshan_fd fd,
     struct darshan_file *file);
 #define JOB_SIZE_124 28
 #define JOB_SIZE_200 56
+#define CP_JOB_RECORD_SIZE_200 1024
+#define CP_JOB_RECORD_SIZE_1x 1024
 
 /* internal routines for parsing different file versions */
 static int getjob_internal_201(darshan_fd file, struct darshan_job *job);
 static int getjob_internal_200(darshan_fd file, struct darshan_job *job);
+static int getfile_internal_204(darshan_fd fd, struct darshan_job *job, 
+    struct darshan_file *file);
 static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
     struct darshan_file *file);
 static int getjob_internal_124(darshan_fd file, struct darshan_job *job);
@@ -369,35 +374,47 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
         return(-1);
     }
 
-    if(strcmp(file->version, "2.03") == 0)
+    if(strcmp(file->version, "2.04") == 0)
+    {
+        getjob_internal = getjob_internal_201;
+        getfile_internal = getfile_internal_204;
+        file->job_struct_size = sizeof(*job);
+        file->COMPAT_CP_EXE_LEN = CP_EXE_LEN;
+    }
+    else if(strcmp(file->version, "2.03") == 0)
     {
         getjob_internal = getjob_internal_201;
         getfile_internal = getfile_internal_200;
         file->job_struct_size = sizeof(*job);
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "2.02") == 0)
     {
         getjob_internal = getjob_internal_201;
         getfile_internal = getfile_internal_200;
         file->job_struct_size = sizeof(*job);
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "2.01") == 0)
     {
         getjob_internal = getjob_internal_201;
         getfile_internal = getfile_internal_200;
         file->job_struct_size = sizeof(*job);
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "2.00") == 0)
     {
         getjob_internal = getjob_internal_200;
         getfile_internal = getfile_internal_200;
         file->job_struct_size = JOB_SIZE_200;
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "1.24") == 0)
     {
         getjob_internal = getjob_internal_124;
         getfile_internal = getfile_internal_124;
         file->job_struct_size = JOB_SIZE_124;
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "1.23") == 0)
     {
@@ -405,18 +422,21 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
         getjob_internal = getjob_internal_124;
         getfile_internal = getfile_internal_124;
         file->job_struct_size = JOB_SIZE_124;
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "1.22") == 0)
     {
         getjob_internal = getjob_internal_124;
         getfile_internal = getfile_internal_122;
         file->job_struct_size = JOB_SIZE_124;
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
     }
     else if(strcmp(file->version, "1.21") == 0)
     {
         getjob_internal = getjob_internal_124;
         getfile_internal = getfile_internal_121;
         file->job_struct_size = JOB_SIZE_124;
+        file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
     }
     else
     {
@@ -564,14 +584,14 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char**
     int ret;
     char* pos;
     int array_index = 0;
-    char buf[CP_EXE_LEN+1];
+    char buf[fd->COMPAT_CP_EXE_LEN+1];
 
     ret = darshan_log_seek(fd, fd->job_struct_size);
     if(ret < 0)
         return(ret);
 
-    ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1));
-    if (ret < (CP_EXE_LEN + 1))
+    ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
+    if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
     {
         perror("darshan_log_read");
         return(-1);
@@ -606,9 +626,9 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char**
     while((pos = strrchr(buf, '\n')) != NULL)
     {
         /* overestimate string lengths */
-        (*mnt_pts)[array_index] = malloc(CP_EXE_LEN);
+        (*mnt_pts)[array_index] = malloc(fd->COMPAT_CP_EXE_LEN);
         assert((*mnt_pts)[array_index]);
-        (*fs_types)[array_index] = malloc(CP_EXE_LEN);
+        (*fs_types)[array_index] = malloc(fd->COMPAT_CP_EXE_LEN);
         assert((*fs_types)[array_index]);
         
         ret = sscanf(++pos, "%" PRId64 "\t%s\t%s", &(*devs)[array_index],
@@ -670,8 +690,8 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
     if(ret < 0)
         return(ret);
 
-    ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1));
-    if (ret < (CP_EXE_LEN + 1))
+    ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
+    if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
     {
         perror("darshan_log_read");
         return(-1);
@@ -735,12 +755,20 @@ void darshan_log_close(darshan_fd file)
  */
 void darshan_log_print_version_warnings(struct darshan_job *job)
 {
-    if(strcmp(job->version_string, "2.03") == 0)
+    if(strcmp(job->version_string, "2.04") == 0)
     {
         /* current version */
         return;
     }
 
+    if(strcmp(job->version_string, "2.03") == 0)
+    {
+        /* no meaningful change to interpretation of log file, 2.04 just
+         * increased the header space available for annotations.
+         */
+        return;
+    }
+
     if(strcmp(job->version_string, "2.02") == 0)
     {
         printf("# WARNING: version 2.01 log format has the following limitations:\n");
@@ -1199,7 +1227,7 @@ static int getjob_internal_200(darshan_fd file, struct darshan_job *job)
     return(-1);
 }
 
-static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
+static int getfile_internal_204(darshan_fd fd, struct darshan_job *job, 
     struct darshan_file *file)
 {
     int ret;
@@ -1255,6 +1283,62 @@ static int getfile_internal_200(darshan_fd fd, struct darshan_job *job,
     return(-1);
 }
 
+static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
+    struct darshan_file *file)
+{
+    int ret;
+    const char* err_string;
+    int i;
+
+    if(fd->pos < CP_JOB_RECORD_SIZE_200)
+    {
+        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_200);
+        if(ret < 0)
+            return(ret);
+    }
+
+    /* reset file record, so that diff compares against a zero'd out record
+     * if file is missing
+     */
+    memset(file, 0, sizeof(&file));
+
+    ret = darshan_log_read(fd, file, sizeof(*file));
+    if(ret == sizeof(*file))
+    {
+        /* got exactly one, correct size record */
+        if(fd->swap_flag)
+        {
+            /* swap bytes if necessary */
+            DARSHAN_BSWAP64(&file->hash);
+            DARSHAN_BSWAP64(&file->rank);
+            for(i=0; i<CP_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&file->counters[i]);
+            for(i=0; i<CP_F_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&file->fcounters[i]);
+        }
+        return(1);
+    }
+
+
+    if(ret > 0)
+    {
+        /* got a short read */
+        fprintf(stderr, "Error: invalid file record (too small)\n");
+        return(-1);
+    }
+
+    if(ret == 0)
+    {
+        /* hit end of file */
+        return(0);
+    }
+
+    /* all other errors */
+    err_string = darshan_log_error(fd, &ret);
+    fprintf(stderr, "Error: %s\n", err_string);
+    return(-1);
+}
+
 /* If we see version 1.24, assume that it is stored in big endian 32 bit
  * format.  Convert up to current format.
  */
@@ -1386,9 +1470,9 @@ static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job,
     /* set file pointer if this is the first file record; otherwise pick up
      * where we left off last time
      */
-    if(fd->pos < CP_JOB_RECORD_SIZE)
+    if(fd->pos < CP_JOB_RECORD_SIZE_1x)
     {
-        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
+        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_1x);
         if(ret < 0)
             return(ret);
     }