change util record ref structure def

darshan-util/darshan-convert.c

@@ -148,13 +148,13 @@ void obfuscate_filenames(int key, struct darshan_record_ref *rec_hash)
 
     HASH_ITER(hlink, rec_hash, ref, tmp)
     {
-        hashed = darshan_hashlittle(ref->rec.name, strlen(ref->rec.name), key);
+        hashed = darshan_hashlittle(ref->name, strlen(ref->name), key);
         sprintf(tmp_string, "%u", hashed);
-        free(ref->rec.name);
-        ref->rec.name = malloc(strlen(tmp_string) + 1);
-        assert(ref->rec.name);
-        memcpy(ref->rec.name, tmp_string, strlen(tmp_string));
-        ref->rec.name[strlen(tmp_string)] = '\0';
+        free(ref->name);
+        ref->name = malloc(strlen(tmp_string) + 1);
+        assert(ref->name);
+        memcpy(ref->name, tmp_string, strlen(tmp_string));
+        ref->name[strlen(tmp_string)] = '\0';
     }
 
     return;
@@ -208,10 +208,10 @@ static void remove_hash_recs(struct darshan_record_ref **rec_hash, darshan_recor
 
     HASH_ITER(hlink, *rec_hash, ref, tmp)
     {
-        if(ref->rec.id != hash)
+        if(ref->id != hash)
         {
             HASH_DELETE(hlink, *rec_hash, ref);
-            free(ref->rec.name);
+            free(ref->name);
             free(ref);
         }
     }
@@ -403,7 +403,7 @@ int main(int argc, char **argv)
     HASH_ITER(hlink, rec_hash, ref, tmp)
     {
         HASH_DELETE(hlink, rec_hash, ref);
-        free(ref->rec.name);
+        free(ref->name);
         free(ref);
     }
 

darshan-util/darshan-logutils.c

@@ -589,8 +589,8 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
                     free(hash_buf);
                     return(-1);
                 }
-                ref->rec.name = malloc(strlen(path_ptr) + 1);
-                if(!ref->rec.name)
+                ref->name = malloc(strlen(path_ptr) + 1);
+                if(!ref->name)
                 {
                     free(ref);
                     free(hash_buf);
@@ -598,11 +598,11 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
                 }
 
                 /* set the fields for this record */
-                ref->rec.id = *rec_id_ptr;
-                strcpy(ref->rec.name, path_ptr);
+                ref->id = *rec_id_ptr;
+                strcpy(ref->name, path_ptr);
 
                 /* add this record to the hash */
-                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
+                HASH_ADD(hlink, *hash, id, sizeof(darshan_record_id), ref);
             }
 
             buf_ptr += strlen(path_ptr) + 1;
@@ -658,10 +658,10 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
          * NOTE: darshan record hash serialization method: 
          *          ... darshan_record_id | path '\0' ...
          */
-        *((darshan_record_id *)buf_ptr) = ref->rec.id;
+        *((darshan_record_id *)buf_ptr) = ref->id;
         buf_ptr += sizeof(darshan_record_id);
-        strcpy(buf_ptr, ref->rec.name);
-        buf_ptr += strlen(ref->rec.name) + 1;
+        strcpy(buf_ptr, ref->name);
+        buf_ptr += strlen(ref->name) + 1;
 
         /* write this hash entry to log file */
         wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
@@ -1146,10 +1146,8 @@ static void darshan_log_dzdestroy(darshan_fd fd)
             break;
 #endif
         case DARSHAN_NO_COMP:
-        {
             /* do nothing */
             break;
-        }
         default:
             fprintf(stderr, "Error: invalid compression type.\n");
     }
@@ -1192,10 +1190,8 @@ static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
             break;
 #endif
         case DARSHAN_NO_COMP:
-        {
             ret = darshan_log_noz_read(fd, map, buf, len, reset_strm_flag);
             break;
-        }
         default:
             fprintf(stderr, "Error: invalid compression type.\n");
             return(-1);

darshan-util/darshan-logutils.h

@@ -43,7 +43,8 @@ typedef struct darshan_fd_s* darshan_fd;
 
 struct darshan_record_ref
 {
-    struct darshan_record rec;
+    char *name;
+    darshan_record_id id;
     UT_hash_handle hlink;
 };
 

darshan-util/darshan-parser.c

@@ -412,7 +412,7 @@ int main(int argc, char **argv)
             /* get mount point and fs type associated with this record */
             for(j=0; j<mount_count; j++)
             {
-                if(strncmp(mnt_pts[j], ref->rec.name, strlen(mnt_pts[j])) == 0)
+                if(strncmp(mnt_pts[j], ref->name, strlen(mnt_pts[j])) == 0)
                 {
                     mnt_pt = mnt_pts[j];
                     fs_type = fs_types[j];
@@ -427,7 +427,7 @@ int main(int argc, char **argv)
             if(mask & OPTION_BASE)
             {
                 /* print the corresponding module data for this record */
-                mod_logutils[i]->log_print_record(mod_buf, ref->rec.name,
+                mod_logutils[i]->log_print_record(mod_buf, ref->name,
                     mnt_pt, fs_type);
             }
 
@@ -623,7 +623,7 @@ cleanup:
     HASH_ITER(hlink, rec_hash, ref, tmp_ref)
     {
         HASH_DELETE(hlink, rec_hash, ref);
-        free(ref->rec.name);
+        free(ref->name);
         free(ref);
     }
 
@@ -1461,7 +1461,7 @@ void posix_file_list(hash_entry_t *file_hash,
 
         printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
             curr->rec_id,
-            ref->rec.name,
+            ref->name,
             curr->procs,
             curr->slowest_time,
             curr->cumul_time/(double)curr->procs);
@@ -1555,7 +1555,7 @@ void mpiio_file_list(hash_entry_t *file_hash,
 
         printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
             curr->rec_id,
-            ref->rec.name,
+            ref->name,
             curr->procs,
             curr->slowest_time,
             curr->cumul_time/(double)curr->procs);

darshan-util/darshan-stitch-logs.c

-    #include <stdio.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <fcntl.h>
-#include <glob.h>
 #include <string.h>
+#include <getopt.h>
+#include <glob.h>
 
 #include "darshan-logutils.h"
 
 #define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
 
+/* TODO: are there any checks we should do to ensure tmp logs belong to the same job */
+/* we can't specifically check the job id, since the pid is used if no job scheduler */
+
+/* TODO: how do we set the output logfile name to be unique, and have necessary semantic info contained */
+
+void usage(char *exename)
+{
+    fprintf(stderr, "Usage: %s [options] <tmp_dir> <job_id>\n", exename);
+    fprintf(stderr, "       TODO: description.\n");
+    fprintf(stderr, "       --shared-redux Reduce globally shared records into a single record.\n");
+
+    exit(1);
+}
+
+void parse_args(int argc, char **argv, char **tmplog_dir, int *tmplog_jobid,
+    int *shared_redux)
+{
+    int index;
+    static struct option long_opts[] =
+    {
+        {"shared-redux", no_argument, NULL, 's'},
+        {0, 0, 0, 0}
+    };
+
+    *shared_redux = 0;
+
+    while(1)
+    {
+        int c = getopt_long(argc, argv, "", long_opts, &index);
+
+        if(c == -1) break;
+
+        switch(c)
+        {
+            case 's':
+                *shared_redux = 1;
+                break;
+            case '?':
+            default:
+                usage(argv[0]);
+                break;
+        }
+    }
+
+    if(optind + 2 == argc)
+    {
+        *tmplog_dir = argv[optind];
+        *tmplog_jobid = atoi(argv[optind+1]);
+    }
+    else
+    {
+        usage(argv[0]);
+    }
+
+    return;
+}
+
 int logfile_path_comp(const void *a, const void *b)
 {
     char *pathA = *(char **)a;
@@ -31,6 +89,7 @@ int logfile_path_comp(const void *a, const void *b)
 
 int main(int argc, char *argv[])
 {
+    int shared_redux;
     char *tmplog_dir;
     int job_id;
     glob_t globbuf;
@@ -50,22 +109,8 @@ int main(int argc, char *argv[])
     int i, j;
     int ret;
 
-    /* TODO: are there any checks we should do to ensure tmp logs belong to the same job */
-    /* we can't specifically check the job id, since the pid is used if no job scheduler */
-
-    /* TODO: how do we set the output logfile name to be unique, and have necessary semantic info contained */
-
-    if(argc != 3)
-    {
-        fprintf(stderr, "Usage: ./darshan-stitch-tmplogs <tmp_dir> <job_id>\n"
-            "\t<tmp_dir> is the directory containing the temporary Darshan logs\n"
-            "\t<job_id> is the job id of the logs we are trying to stitch\n");
-        return(0);
-    }
-
     /* grab command line arguments */
-    tmplog_dir = argv[1];
-    job_id = atoi(argv[2]);
+    parse_args(argc, argv, &tmplog_dir, &job_id, &shared_redux);
 
     /* construct the list of input log files to stitch together */
     snprintf(glob_pstr, 512, "%s/darshan_job%d*", tmplog_dir, job_id);
@@ -169,14 +214,12 @@ int main(int argc, char *argv[])
          */
         HASH_ITER(hlink, in_hash, ref, tmp)
         {
-            HASH_FIND(hlink, stitch_hash, &(ref->rec.id),
-                sizeof(darshan_record_id), found);
+            HASH_FIND(hlink, stitch_hash, &(ref->id), sizeof(darshan_record_id), found);
             if(!found)
             {
-                HASH_ADD(hlink, stitch_hash, rec.id,
-                    sizeof(darshan_record_id), ref);
+                HASH_ADD(hlink, stitch_hash, id, sizeof(darshan_record_id), ref);
             }
-            else if(strcmp(ref->rec.name, found->rec.name))
+            else if(strcmp(ref->name, found->name))
             {
                 fprintf(stderr,
                     "Error: invalid Darshan record table entry.\n");
@@ -250,25 +293,30 @@ int main(int argc, char *argv[])
     }
     memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
 
+    /* iterate over active darshan modules and gather module data to write
+     * to the stitched together output log
+     */
     for(i = 0; i < DARSHAN_MPIIO_MOD; i++)
     {
         if(!mod_logutils[i]) continue;
 
-#if 0
-        /* XXX first build shared record list? */
-        for(j = 0; j < globbuf.gl_pathc; j++)
+        if(shared_redux)
         {
+            /* copy all root's file records into an array */
 
-        }
+            /* compare and updated shared records? */
+            for(j = 1; j < globbuf.gl_pathc; j++)
+            {
+            }
 
-        /* XXX second aggregate shared records ? */
-        for(j = 0; j < globbuf.gl_pathc; j++)
-        {
+            /* XXX aggregate shared records? */
+            for(j = 0; j < globbuf.gl_pathc; j++)
+            {
 
+            }
         }
-#endif
 
-        /* XXX third write each rank's blobs, with rank 0 writing the shared ones ? */
+        /* XXX third write each rank's blobs, with rank 0 writing the shared ones? */
         for(j = 0; j < globbuf.gl_pathc; j++)
         {
             in_fd = darshan_log_open(globbuf.gl_pathv[j]);