update logutils to use new compression format

darshan-util/darshan-hdf5-logutils.c

@@ -30,49 +30,48 @@ char *hdf5_f_counter_names[] = {
 };
 #undef X
 
+static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id);
+static void darshan_log_print_hdf5_file(void *file_rec,
+    char *file_name, char *mnt_pt, char *fs_type);
+
 struct darshan_mod_logutil_funcs hdf5_logutils =
 {
     .log_get_record = &darshan_log_get_hdf5_file,
     .log_print_record = &darshan_log_print_hdf5_file,
 };
 
-int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id)
+static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id)
 {
     int i;
-    int ret;
-    struct darshan_hdf5_file *file = NULL;
-    *rec_id = 0;
+    struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)
+        (*hdf5_buf_p);
 
-    file = malloc(sizeof(*file));
-    if(!file)
+    if(*bytes_left < sizeof(struct darshan_hdf5_file))
         return(-1);
-    memset(file, 0, sizeof(*file));
 
-    ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD,
-        (void *)file, sizeof(*file));
-    if(ret == 1)
+    if(fd->swap_flag)
     {
-        if(fd->swap_flag)
-        {
-            /* swap bytes if necessary */
-            DARSHAN_BSWAP64(&file->f_id);
-            DARSHAN_BSWAP64(&file->rank);
-            for(i=0; i<HDF5_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->counters[i]);
-            for(i=0; i<HDF5_F_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->fcounters[i]);
-        }
-
-        /* pass the file record back */
-        *file_rec = (void *)file;
-        *rec_id = file->f_id;
+        /* swap bytes if necessary */
+        DARSHAN_BSWAP64(&file->f_id);
+        DARSHAN_BSWAP64(&file->rank);
+        for(i=0; i<HDF5_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->counters[i]);
+        for(i=0; i<HDF5_F_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->fcounters[i]);
     }
 
-    return(ret);
+    /* update/set output variables */
+    *file_rec = (void *)file;
+    *rec_id = file->f_id;
+    *hdf5_buf_p = (file + 1); /* increment input buf by size of file record */
+    *bytes_left -= sizeof(struct darshan_hdf5_file);
+
+    return(0);
 }
 
-void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
+static void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     int i;

darshan-util/darshan-hdf5-logutils.h

@@ -15,9 +15,4 @@ extern char *hdf5_f_counter_names[];
 
 extern struct darshan_mod_logutil_funcs hdf5_logutils;
 
-int darshan_log_get_hdf5_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id);
-void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
-    char *mnt_pt, char *fs_type);
-
 #endif

darshan-util/darshan-logutils.h

@@ -16,15 +16,21 @@
 
 #include "darshan-log-format.h"
 
-/* default to a decompression buffer size of 4 MiB */
-#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
+/* default to a buffer size of 4 MiB for compression/decompression */
+#define DARSHAN_DEF_COMP_BUF_SZ (4*1024*1024)
 
+/* TODO: can we refactor this def out of header? modules currently poke at swap_flag
+ * directly, but other than that there's no reason for another module to know this
+ * definition.
+ */
 struct darshan_fd_s
 {
-    gzFile gzf;
+    int fildes;
     int64_t pos;
     char version[8];
     int swap_flag;
+    char *exe_mnt_data;
+    struct darshan_log_map job_map;
     struct darshan_log_map rec_map;
     struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
 };
@@ -40,8 +46,10 @@ struct darshan_mod_logutil_funcs
 {
     int (*log_get_record)(
         darshan_fd fd,
-        void **rec_dat,
-        darshan_record_id *rec_id
+        void** mod_buf_p,
+        int* mod_bytes_left,
+        void** file_rec,
+        darshan_record_id* rec_id
     );
     void (*log_print_record)(
         void *file_rec,
@@ -72,7 +80,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts,
 int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash);
 int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash);
 int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
-    void *mod_buf, int mod_buf_sz);
+    void *mod_buf, int *mod_buf_sz);
 int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
     void *mod_buf, int mod_buf_sz);
 void darshan_log_close(darshan_fd file);

darshan-util/darshan-mpiio-logutils.c

@@ -30,49 +30,48 @@ char *mpiio_f_counter_names[] = {
 };
 #undef X
 
+static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id);
+static void darshan_log_print_mpiio_file(void *file_rec,
+    char *file_name, char *mnt_pt, char *fs_type);
+
 struct darshan_mod_logutil_funcs mpiio_logutils =
 {
     .log_get_record = &darshan_log_get_mpiio_file,
     .log_print_record = &darshan_log_print_mpiio_file,
 };
 
-int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id)
+static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id)
 {
     int i;
-    int ret;
-    struct darshan_mpiio_file *file = NULL;
-    *rec_id = 0;
+    struct darshan_mpiio_file *file = (struct darshan_mpiio_file *)
+        (*mpiio_buf_p);
 
-    file = malloc(sizeof(*file));
-    if(!file)
+    if(*bytes_left < sizeof(struct darshan_mpiio_file))
         return(-1);
-    memset(file, 0, sizeof(*file));
 
-    ret = darshan_log_getmod(fd, DARSHAN_MPIIO_MOD,
-        (void *)file, sizeof(*file));
-    if(ret == 1)
+    if(fd->swap_flag)
     {
-        if(fd->swap_flag)
-        {
-            /* swap bytes if necessary */
-            DARSHAN_BSWAP64(&file->f_id);
-            DARSHAN_BSWAP64(&file->rank);
-            for(i=0; i<MPIIO_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->counters[i]);
-            for(i=0; i<MPIIO_F_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->fcounters[i]);
-        }
-
-        /* pass the file record back */
-        *file_rec = (void *)file;
-        *rec_id = file->f_id;
+        /* swap bytes if necessary */
+        DARSHAN_BSWAP64(&file->f_id);
+        DARSHAN_BSWAP64(&file->rank);
+        for(i=0; i<MPIIO_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->counters[i]);
+        for(i=0; i<MPIIO_F_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->fcounters[i]);
     }
 
-    return(ret);
+    /* update/set output variables */
+    *file_rec = (void *)file;
+    *rec_id = file->f_id;
+    *mpiio_buf_p = (file + 1); /* increment input buf by size of file record */
+    *bytes_left -= sizeof(struct darshan_mpiio_file);
+
+    return(0);
 }
 
-void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
+static void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     int i;

darshan-util/darshan-mpiio-logutils.h

@@ -15,9 +15,4 @@ extern char *mpiio_f_counter_names[];
 
 extern struct darshan_mod_logutil_funcs mpiio_logutils;
 
-int darshan_log_get_mpiio_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id);
-void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
-    char *mnt_pt, char *fs_type);
-
 #endif

darshan-util/darshan-parser.c

@@ -25,12 +25,12 @@ int main(int argc, char **argv)
     int ret;
     int i, j;
     char *filename;
-    char tmp_string[4096];
+    char tmp_string[4096] = {0};
     darshan_fd fd;
     struct darshan_header header;
     struct darshan_job job;
     struct darshan_record_ref *rec_hash = NULL;
-    struct darshan_record_ref *ref;
+    struct darshan_record_ref *ref, *tmp;
     int mount_count;
     char** mnt_pts;
     char** fs_types;
@@ -39,6 +39,8 @@ int main(int argc, char **argv)
     char *save;
     char buffer[DARSHAN_JOB_METADATA_LEN];
     int empty_mods = 0;
+    char *mod_buf;
+    int mod_buf_sz;
 
     /* TODO: argument parsing */
     assert(argc == 2);
@@ -132,10 +134,10 @@ int main(int argc, char **argv)
     }
 
     /* print breakdown of each log file component's contribution to file size */
-    printf("\n# log file component sizes (decompressed)\n");
+    printf("\n# log file component sizes (compressed)\n");
     printf("# -------------------------------------------------------\n");
-    printf("# header: %zu bytes\n", sizeof(struct darshan_header));
-    printf("# job data: %zu bytes\n", sizeof(struct darshan_job));
+    printf("# header: %zu bytes (uncompressed)\n", sizeof(struct darshan_header));
+    printf("# job data: %zu bytes\n", header.rec_map.off - sizeof(struct darshan_header));
     printf("# record table: %zu bytes\n", header.rec_map.len);
     for(i=0; i<DARSHAN_MAX_MODS; i++)
     {
@@ -154,20 +156,37 @@ int main(int argc, char **argv)
         printf("# mount entry:\t%s\t%s\n", mnt_pts[i], fs_types[i]);
     }
 
+    mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
+    if(!mod_buf)
+        return(-1);
+
     DARSHAN_PRINT_HEADER();
+    /* TODO: does each module print header of what each counter means??? */
 
     for(i=0; i<DARSHAN_MAX_MODS; i++)
     {
+        int mod_bytes_left;
+        void *mod_buf_p;
         void *rec_p = NULL;
         darshan_record_id rec_id;
 
-        /* skip modules not present in log file */
-        if(header.mod_map[i].len == 0)
+        memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
+        mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
+
+        /* check each module for any data */
+        ret = darshan_log_getmod(fd, i, mod_buf, &mod_buf_sz);
+        if(ret < 0)
         {
+            fprintf(stderr, "Error: failed to get module %s data\n",
+                darshan_module_names[i]);
+            fflush(stderr);
+            darshan_log_close(fd);
+            return(-1);
+        }
+        else if(ret == 0)
+        {
+            /* skip modules not present in log file */
             empty_mods++;
-            if(empty_mods == DARSHAN_MAX_MODS)
-                printf("# no module data available.\n");
-
             continue;
         }
 
@@ -180,15 +199,26 @@ int main(int argc, char **argv)
         }
 
         /* this module has data to be parsed and printed */
-
-        /* TODO: do modules print header of what each counter means??? */
+        mod_buf_p = mod_buf;
+        mod_bytes_left = mod_buf_sz;
 
         /* loop over each of this module's records and print them */
-        while ((ret = mod_logutils[i]->log_get_record(fd, &rec_p, &rec_id)) == 1)
+        while (mod_bytes_left > 0)
         {
             char *mnt_pt = NULL;
             char *fs_type = NULL;
 
+            ret = mod_logutils[i]->log_get_record(fd, &mod_buf_p, &mod_bytes_left,
+                &rec_p, &rec_id);
+            if(ret < 0)
+            {
+                fprintf(stderr, "Error: failed to parse module %s data record\n",
+                    darshan_module_names[i]);
+                fflush(stderr);
+                darshan_log_close(fd);
+                return(-1);
+            }
+
             /* get the pathname for this record */
             HASH_FIND(hlink, rec_hash, &rec_id, sizeof(darshan_record_id), ref);
             assert(ref);
@@ -211,19 +241,17 @@ int main(int argc, char **argv)
             /* print the corresponding module data for this record */
             mod_logutils[i]->log_print_record(rec_p, ref->rec.name,
                 mnt_pt, fs_type);
-
-            /* NOTE: we have to free the given file record data */
-            free(rec_p);
         }
+    }
+    if(empty_mods == DARSHAN_MAX_MODS)
+        printf("# no module data available.\n");
 
-        if(ret < 0)
-        {
-            fprintf(stderr, "Error: failed to parse module %s data\n",
-                darshan_module_names[i]);
-            fflush(stderr);
-            darshan_log_close(fd);
-            return(-1);
-        }
+    /* free record hash data */
+    HASH_ITER(hlink, rec_hash, ref, tmp)
+    {
+        HASH_DELETE(hlink, rec_hash, ref);
+        free(ref->rec.name);
+        free(ref);
     }
 
     /* free mount info */
@@ -238,6 +266,7 @@ int main(int argc, char **argv)
         free(fs_types);
     }
 
+    free(mod_buf);
     darshan_log_close(fd);
 
     return(0);

darshan-util/darshan-pnetcdf-logutils.c

@@ -30,49 +30,48 @@ char *pnetcdf_f_counter_names[] = {
 };
 #undef X
 
+static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id);
+static void darshan_log_print_pnetcdf_file(void *file_rec,
+    char *file_name, char *mnt_pt, char *fs_type);
+
 struct darshan_mod_logutil_funcs pnetcdf_logutils =
 {
     .log_get_record = &darshan_log_get_pnetcdf_file,
     .log_print_record = &darshan_log_print_pnetcdf_file,
 };
 
-int darshan_log_get_pnetcdf_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id)
+static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id)
 {
     int i;
-    int ret;
-    struct darshan_pnetcdf_file *file = NULL;
-    *rec_id = 0;
+    struct darshan_pnetcdf_file *file = (struct darshan_pnetcdf_file *)
+        (*pnetcdf_buf_p);
 
-    file = malloc(sizeof(*file));
-    if(!file)
+    if(*bytes_left < sizeof(struct darshan_pnetcdf_file))
         return(-1);
-    memset(file, 0, sizeof(*file));
 
-    ret = darshan_log_getmod(fd, DARSHAN_PNETCDF_MOD,
-        (void *)file, sizeof(*file));
-    if(ret == 1)
+    if(fd->swap_flag)
     {
-        if(fd->swap_flag)
-        {
-            /* swap bytes if necessary */
-            DARSHAN_BSWAP64(&file->f_id);
-            DARSHAN_BSWAP64(&file->rank);
-            for(i=0; i<PNETCDF_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->counters[i]);
-            for(i=0; i<PNETCDF_F_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->fcounters[i]);
-        }
-
-        /* pass the file record back */
-        *file_rec = (void *)file;
-        *rec_id = file->f_id;
+        /* swap bytes if necessary */
+        DARSHAN_BSWAP64(&file->f_id);
+        DARSHAN_BSWAP64(&file->rank);
+        for(i=0; i<PNETCDF_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->counters[i]);
+        for(i=0; i<PNETCDF_F_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->fcounters[i]);
     }
 
-    return(ret);
+    /* update/set output variables */
+    *file_rec = (void *)file;
+    *rec_id = file->f_id;
+    *pnetcdf_buf_p = (file + 1); /* increment input buf by size of file record */
+    *bytes_left -= sizeof(struct darshan_pnetcdf_file);
+
+    return(0);
 }
 
-void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
+static void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     int i;

darshan-util/darshan-pnetcdf-logutils.h

@@ -15,9 +15,4 @@ extern char *pnetcdf_f_counter_names[];
 
 extern struct darshan_mod_logutil_funcs pnetcdf_logutils;
 
-int darshan_log_get_pnetcdf_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id);
-void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
-    char *mnt_pt, char *fs_type);
-
 #endif

darshan-util/darshan-posix-logutils.c

@@ -30,49 +30,49 @@ char *posix_f_counter_names[] = {
 };
 #undef X
 
+static int darshan_log_get_posix_file(darshan_fd fd, void** psx_buf_p,
+    int* bytes_left, void** file_rec, darshan_record_id* rec_id);
+static void darshan_log_print_posix_file(void *file_rec,
+    char *file_name, char *mnt_pt, char *fs_type);
+
 struct darshan_mod_logutil_funcs posix_logutils =
 {
     .log_get_record = &darshan_log_get_posix_file,
     .log_print_record = &darshan_log_print_posix_file,
 };
 
-int darshan_log_get_posix_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id)
+static int darshan_log_get_posix_file(darshan_fd fd, void** psx_buf_p,
+    int* bytes_left, void **file_rec, darshan_record_id* rec_id)
 {
     int i;
-    int ret;
-    struct darshan_posix_file *file = NULL;
-    *rec_id = 0;
+    struct darshan_posix_file *file = (struct darshan_posix_file *)
+        (*psx_buf_p);
 
-    file = malloc(sizeof(*file));
-    if(!file)
+    if(*bytes_left < sizeof(struct darshan_posix_file))
         return(-1);
-    memset(file, 0, sizeof(*file));
 
-    ret = darshan_log_getmod(fd, DARSHAN_POSIX_MOD,
-        (void *)file, sizeof(*file));
-    if(ret == 1)
+    if(fd->swap_flag)
     {
-        if(fd->swap_flag)
-        {
-            /* swap bytes if necessary */
-            DARSHAN_BSWAP64(&file->f_id);
-            DARSHAN_BSWAP64(&file->rank);
-            for(i=0; i<POSIX_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->counters[i]);
-            for(i=0; i<POSIX_F_NUM_INDICES; i++)
-                DARSHAN_BSWAP64(&file->fcounters[i]);
-        }
-
-        /* pass the file record back */
-        *file_rec = (void *)file;
-        *rec_id = file->f_id;
+        /* swap bytes if necessary */
+        DARSHAN_BSWAP64(&file->f_id);
+        DARSHAN_BSWAP64(&file->rank);
+        for(i=0; i<POSIX_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->counters[i]);
+        for(i=0; i<POSIX_F_NUM_INDICES; i++)
+            DARSHAN_BSWAP64(&file->fcounters[i]);
     }
 
-    return(ret);
+    /* update/set output variables */
+    *file_rec = (void *)file;
+    *rec_id = file->f_id;
+    *psx_buf_p = (file + 1); /* increment input buf by size of file record */
+    //((struct darshan_posix_file *)(*psx_buf_p))++;
+    *bytes_left -= sizeof(struct darshan_posix_file);
+
+    return(0);
 }
 
-void darshan_log_print_posix_file(void *file_rec, char *file_name,
+static void darshan_log_print_posix_file(void *file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     int i;

darshan-util/darshan-posix-logutils.h

@@ -15,9 +15,4 @@ extern char *posix_f_counter_names[];
 
 extern struct darshan_mod_logutil_funcs posix_logutils;
 
-int darshan_log_get_posix_file(darshan_fd fd, void **file_rec,
-    darshan_record_id *rec_id);
-void darshan_log_print_posix_file(void *file_rec, char *file_name,
-    char *mnt_pt, char *fs_type);
-
 #endif