new implementation of darshan-logutils

darshan-util/darshan-analyzer.c

@@ -35,18 +35,16 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
 {
     int ret;
     darshan_fd file;
-    struct darshan_header header;
     struct darshan_job job;
     struct darshan_mod_logutil_funcs *psx_mod = mod_logutils[DARSHAN_POSIX_MOD];
-    struct darshan_posix_file *psx_buf, *psx_buf_p;
-    int psx_buf_sz, psx_buf_bytes_left;
-    struct darshan_posix_file *psx_rec;
+    struct darshan_posix_file psx_rec;
     darshan_record_id rec_id;
     int f_count;
     double total_io_time;
     double total_job_time;
 
     assert(psx_mod);
+    memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
 
     file = darshan_log_open(fname);
     if (file == NULL)
@@ -55,14 +53,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
         return -1;
     }
 
-    ret = darshan_log_getheader(file, &header);
-    if (ret < 0)
-    {
-        fprintf(stderr, "darshan_log_getheader() failed on file %s.\n", fname);
-        darshan_log_close(file);
-        return -1;
-    }
-
     ret = darshan_log_getjob(file, &job);
     if (ret < 0)
     {
@@ -71,49 +61,36 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
         return -1;
     }
 
-    psx_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
-    psx_buf = malloc(psx_buf_sz);
-    if (!psx_buf)
-    {
-        darshan_log_close(file);
-        return -1;
-    }
-
-    ret = darshan_log_getmod(file, DARSHAN_POSIX_MOD, (void *)psx_buf, &psx_buf_sz);
-    if (ret < 0)
-    {
-        fprintf(stderr, "darshan_log_getmod() failed on file %s.\n", fname);
-        darshan_log_close(file);
-        return -1;
-    }
-
     f_count = 0;
     total_io_time = 0.0;
 
-    psx_buf_bytes_left = psx_buf_sz;
-    psx_buf_p = psx_buf;
-    while(psx_buf_bytes_left)
+    while((ret = psx_mod->log_get_record(file, &psx_rec, &rec_id)) == 1)
     {
-        ret = psx_mod->log_get_record((void **)&psx_buf_p, &psx_buf_bytes_left,
-            (void **)&psx_rec, &rec_id, file->swap_flag);
-
         f_count   += 1;
 
-        if (psx_rec->rank == -1)
+        if (psx_rec.rank == -1)
             *used_shared = 1;
         else
             *used_fpp = 1;
 
-        total_io_time += (psx_rec->fcounters[POSIX_F_READ_TIME] +
-                         psx_rec->fcounters[POSIX_F_WRITE_TIME] +
-                         psx_rec->fcounters[POSIX_F_META_TIME]);
+        total_io_time += (psx_rec.fcounters[POSIX_F_READ_TIME] +
+                         psx_rec.fcounters[POSIX_F_WRITE_TIME] +
+                         psx_rec.fcounters[POSIX_F_META_TIME]);
+
+        memset(&psx_rec, 0, sizeof(struct darshan_posix_file));
+    }
+    if (ret < 0)
+    {
+        fprintf(stderr, "Error: unable to read posix file record in log file %s.\n", fname);
+        darshan_log_close(file);
+        return -1;
     }
 
-    if (header.mod_map[DARSHAN_MPIIO_MOD].len > 0)
+    if (file->mod_map[DARSHAN_MPIIO_MOD].len > 0)
         *used_mpio += 1;
-    if (header.mod_map[DARSHAN_HDF5_MOD].len > 0)
+    if (file->mod_map[DARSHAN_HDF5_MOD].len > 0)
         *used_hdf5 += 1;
-    if (header.mod_map[DARSHAN_PNETCDF_MOD].len > 0)
+    if (file->mod_map[DARSHAN_PNETCDF_MOD].len > 0)
         *used_pnet += 1;
 
     total_job_time = (double)job.end_time - (double)job.start_time;
@@ -131,7 +108,6 @@ int process_log(const char *fname, double *io_ratio, int *used_mpio, int *used_p
         *io_ratio = 0.0;
     }
 
-    free(psx_buf);
     darshan_log_close(file);
 
     return 0;

darshan-util/darshan-bgq-logutils.c

@@ -30,48 +30,65 @@ char *bgq_f_counter_names[] = {
 };
 #undef X
 
-static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left,
-    void** file_rec, darshan_record_id* rec_id, int byte_swap_flag);
-static void darshan_log_print_bgq_file(void *file_rec,
+static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
+    darshan_record_id* rec_id);
+static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf);
+static void darshan_log_print_bgq_rec(void *file_rec,
     char *file_name, char *mnt_pt, char *fs_type);
 
 struct darshan_mod_logutil_funcs bgq_logutils =
 {
-    .log_get_record = &darshan_log_get_bgq_file,
-    .log_print_record = &darshan_log_print_bgq_file,
+    .log_get_record = &darshan_log_get_bgq_rec,
+    .log_put_record = &darshan_log_put_bgq_rec,
+    .log_print_record = &darshan_log_print_bgq_rec,
 };
 
-static int darshan_log_get_bgq_file(void** psx_buf_p, int* bytes_left,
-    void** file_rec, darshan_record_id* rec_id, int byte_swap_flag)
+static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf,
+    darshan_record_id* rec_id)
 {
+    struct darshan_bgq_record *rec;
     int i;
-    struct darshan_bgq_record *file = (struct darshan_bgq_record *)
-        (*psx_buf_p);
+    int ret;
 
-    if(*bytes_left < sizeof(struct darshan_bgq_record))
+    ret = darshan_log_getmod(fd, DARSHAN_BGQ_MOD, bgq_buf,
+        sizeof(struct darshan_bgq_record));
+    if(ret < 0)
         return(-1);
-
-    if(byte_swap_flag)
+    else if(ret < sizeof(struct darshan_bgq_record))
+        return(0);
+    else
     {
-        /* swap bytes if necessary */
-        DARSHAN_BSWAP64(&file->f_id);
-        DARSHAN_BSWAP64(&file->rank);
-        for(i=0; i<BGQ_NUM_INDICES; i++)
-            DARSHAN_BSWAP64(&file->counters[i]);
-        for(i=0; i<BGQ_F_NUM_INDICES; i++)
-            DARSHAN_BSWAP64(&file->fcounters[i]);
+        rec = (struct darshan_bgq_record *)bgq_buf;
+        if(fd->swap_flag)
+        {
+            /* swap bytes if necessary */
+            DARSHAN_BSWAP64(&rec->f_id);
+            DARSHAN_BSWAP64(&rec->rank);
+            for(i=0; i<BGQ_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&rec->counters[i]);
+            for(i=0; i<BGQ_F_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&rec->fcounters[i]);
+        }
+
+        *rec_id = rec->f_id;
+        return(1);
     }
+}
 
-    /* update/set output variables */
-    *file_rec = (void *)file;
-    *rec_id = file->f_id;
-    *psx_buf_p = (file + 1); /* increment input buf by size of file record */
-    *bytes_left -= sizeof(struct darshan_bgq_record);
+static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf)
+{
+    struct darshan_bgq_record *rec = (struct darshan_bgq_record *)bgq_buf;
+    int ret;
+
+    ret = darshan_log_putmod(fd, DARSHAN_BGQ_MOD, rec,
+        sizeof(struct darshan_bgq_record));
+    if(ret < 0)
+        return(-1);
 
     return(0);
 }
 
-static void darshan_log_print_bgq_file(void *file_rec, char *file_name,
+static void darshan_log_print_bgq_rec(void *file_rec, char *file_name,
     char *mnt_pt, char *fs_type)
 {
     int i;

darshan-util/darshan-convert.c

@@ -19,6 +19,8 @@
 
 #include "darshan-logutils.h"
 
+#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
+
 extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
 
 int usage (char *exename)
@@ -221,7 +223,6 @@ int main(int argc, char **argv)
     int ret;
     char *infile_name;
     char *outfile_name;
-    struct darshan_header header;
     struct darshan_job job;
     char tmp_string[4096] = {0};
     darshan_fd infile;
@@ -232,8 +233,7 @@ int main(int argc, char **argv)
     char** fs_types;
     struct darshan_record_ref *rec_hash = NULL;
     struct darshan_record_ref *ref, *tmp;
-    char *mod_buf;
-    int mod_buf_sz;
+    char mod_buf[DEF_MOD_BUF_SIZE];
     enum darshan_comp_type comp_type;
     int bzip2;
     int obfuscate;
@@ -247,42 +247,23 @@ int main(int argc, char **argv)
 
     infile = darshan_log_open(infile_name);
     if(!infile)
-    {
-        fprintf(stderr, "darshan_log_open() failed to open %s\n.", infile_name);
         return(-1);
-    }
  
     comp_type = bzip2 ? comp_type = DARSHAN_BZIP2_COMP : DARSHAN_ZLIB_COMP;
     outfile = darshan_log_create(outfile_name, comp_type);
     if(!outfile)
     {
-        fprintf(stderr, "darshan_log_create() failed to create %s\n.", outfile_name);
         darshan_log_close(infile);
         return(-1);
     }
 
-    /* read header from input file */
-    ret = darshan_log_getheader(infile, &header);
-    if(ret < 0)
-    {
-        fprintf(stderr, "Error: unable to read header from input log file %s.\n", infile_name);
-        darshan_log_close(infile);
-        darshan_log_close(outfile);
-        return(-1);
-    }
-
-    /* NOTE: we do not write the header to the output file until the end, as
-     * the mapping data stored in this structure may change in the conversion
-     * process (particularly, if we are converting between libz/bz2 compression)
-     */
-
     /* read job info */
     ret = darshan_log_getjob(infile, &job);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to read job information from log file.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
+        unlink(outfile_name);
         return(-1);
     }
 
@@ -293,7 +274,6 @@ int main(int argc, char **argv)
     ret = darshan_log_putjob(outfile, &job);
     if (ret < 0)
     {
-        fprintf(stderr, "Error: unable to write job information to log file.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
         return(-1);
@@ -302,9 +282,9 @@ int main(int argc, char **argv)
     ret = darshan_log_getexe(infile, tmp_string);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to read trailing job information.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
+        unlink(outfile_name);
         return(-1);
     }
 
@@ -313,7 +293,6 @@ int main(int argc, char **argv)
     ret = darshan_log_putexe(outfile, tmp_string);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to write trailing job information.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
         return(-1);
@@ -322,16 +301,15 @@ int main(int argc, char **argv)
     ret = darshan_log_getmounts(infile, &mnt_pts, &fs_types, &mount_count);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to read trailing job information.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
+        unlink(outfile_name);
         return(-1);
     }
 
     ret = darshan_log_putmounts(outfile, mnt_pts, fs_types, mount_count);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to write mount information.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
         return(-1);
@@ -340,9 +318,9 @@ int main(int argc, char **argv)
     ret = darshan_log_gethash(infile, &rec_hash);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to read darshan record hash.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
+        unlink(outfile_name);
         return(-1);
     }
 
@@ -355,104 +333,56 @@ int main(int argc, char **argv)
     ret = darshan_log_puthash(outfile, rec_hash);
     if(ret < 0)
     {
-        fprintf(stderr, "Error: unable to write darshan record hash.\n");
         darshan_log_close(infile);
         darshan_log_close(outfile);
         return(-1);
     }
 
-    mod_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
-    if(!mod_buf)
-        return(-1);
-
+    /* loop over each module and convert it's data to the new format */
     for(i=0; i<DARSHAN_MAX_MODS; i++)
     {
-        int mod_bytes_left;
-        int mod_bytes_left_save;
-        void *mod_buf_p;
-        void *rec_p = NULL;
         darshan_record_id rec_id;
 
-        memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
-        mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
-
         /* check each module for any data */
-        ret = darshan_log_getmod(infile, i, mod_buf, &mod_buf_sz);
-        if(ret < 0)
-        {
-            fprintf(stderr, "Error: failed to get module %s data.\n",
-                darshan_module_names[i]);
-            darshan_log_close(infile);
-            darshan_log_close(outfile);
-            return(-1);
-        }
-        else if(ret == 0)
-        {
-            /* skip modules not present in log file */
+        if(infile->mod_map[i].len == 0)
             continue;
-        }
-
-        /* skip modules with no defined logutil handlers */
-        if(!mod_logutils[i])
+        else if(!mod_logutils[i])
         {
             fprintf(stderr, "Warning: no log utility handlers defined "
-                "for module %s, SKIPPING\n", darshan_module_names[i]);
+                "for module %s, SKIPPING.\n", darshan_module_names[i]);
             continue;
         }
 
         /* we have module data to convert */
-        /* NOTE: it is necessary to iterate through each module's
-         * records to correct any endianness issues before writing
-         * this data back to file
-         */
-        mod_bytes_left = mod_buf_sz;
-        mod_buf_p = mod_buf;
-        while(mod_bytes_left > 0)
-        {
-            mod_bytes_left_save = mod_bytes_left;
-            ret = mod_logutils[i]->log_get_record(&mod_buf_p, &mod_bytes_left,
-                &rec_p, &rec_id, infile->swap_flag);
-            if(ret < 0)
-            {
-                fprintf(stderr, "Error: failed to parse module %s data record\n",
-                    darshan_module_names[i]);
-                darshan_log_close(infile);
-                darshan_log_close(outfile);
-                return(-1);
-            }
+        memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
 
-            if(hash == rec_id)
-            {
-                mod_buf_p = rec_p;
-                mod_buf_sz = mod_bytes_left_save - mod_bytes_left;
-                break;
-            }
-            else if(mod_bytes_left == 0)
-            {
-                mod_buf_p = mod_buf;
-            }
-        }
-
-        ret = darshan_log_putmod(outfile, i, mod_buf_p, mod_buf_sz);
-        if(ret < 0)
+        ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id);
+        if(ret != 1)
         {
-            fprintf(stderr, "Error: failed to put module %s data.\n",
+            fprintf(stderr, "Error: failed to parse the first %s module record.\n",
                 darshan_module_names[i]);
             darshan_log_close(infile);
             darshan_log_close(outfile);
+            unlink(outfile_name);
             return(-1);
         }
-    }
-    free(mod_buf);
 
-    /* write header to output file */
-    ret = darshan_log_putheader(outfile);
-    if(ret < 0)
-    {
-        fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
-        darshan_log_close(infile);
-        darshan_log_close(outfile);
-        return(-1);
+        /* loop over each of the module's records and convert */
+        do
+        {
+            if(!hash || hash == rec_id)
+            {
+                ret = mod_logutils[i]->log_put_record(outfile, mod_buf);
+                if(ret < 0)
+                {
+                    darshan_log_close(infile);
+                    darshan_log_close(outfile);
+                    return(-1);
+                }
+
+                memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
+            }
+        } while((ret = mod_logutils[i]->log_get_record(infile, mod_buf, &rec_id)) == 1);
     }
 
     darshan_log_close(infile);

darshan-util/darshan-hdf5-logutils.c

@@ -30,43 +30,60 @@ char *hdf5_f_counter_names[] = {
 };
 #undef X
 
-static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left,
-    void** file_rec, darshan_record_id* rec_id, int byte_swap_flag);
+static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
+    darshan_record_id* rec_id);
+static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
 static void darshan_log_print_hdf5_file(void *file_rec,
     char *file_name, char *mnt_pt, char *fs_type);
 
 struct darshan_mod_logutil_funcs hdf5_logutils =
 {
     .log_get_record = &darshan_log_get_hdf5_file,
+    .log_put_record = &darshan_log_put_hdf5_file,
     .log_print_record = &darshan_log_print_hdf5_file,
 };
 
-static int darshan_log_get_hdf5_file(void** hdf5_buf_p, int* bytes_left,
-    void** file_rec, darshan_record_id* rec_id, int byte_swap_flag)
+static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf,
+    darshan_record_id* rec_id)
 {
+    struct darshan_hdf5_file *file;
     int i;
-    struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)
-        (*hdf5_buf_p);
+    int ret;
 
-    if(*bytes_left < sizeof(struct darshan_hdf5_file))
+    ret = darshan_log_getmod(fd, DARSHAN_HDF5_MOD, hdf5_buf,
+        sizeof(struct darshan_hdf5_file));
+    if(ret < 0)
         return(-1);
-
-    if(byte_swap_flag)
+    else if(ret < sizeof(struct darshan_hdf5_file))
+        return(0);
+    else
     {
-        /* swap bytes if necessary */
-        DARSHAN_BSWAP64(&file->f_id);
-        DARSHAN_BSWAP64(&file->rank);
-        for(i=0; i<HDF5_NUM_INDICES; i++)
-            DARSHAN_BSWAP64(&file->counters[i]);
-        for(i=0; i<HDF5_F_NUM_INDICES; i++)
-            DARSHAN_BSWAP64(&file->fcounters[i]);
+        file = (struct darshan_hdf5_file *)hdf5_buf;
+        if(fd->swap_flag)
+        {
+            /* swap bytes if necessary */
+            DARSHAN_BSWAP64(&file->f_id);
+            DARSHAN_BSWAP64(&file->rank);
+            for(i=0; i<HDF5_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&file->counters[i]);
+            for(i=0; i<HDF5_F_NUM_INDICES; i++)
+                DARSHAN_BSWAP64(&file->fcounters[i]);
+        }
+
+        *rec_id = file->f_id;
+        return(1);
     }
+}
 
-    /* update/set output variables */
-    *file_rec = (void *)file;
-    *rec_id = file->f_id;
-    *hdf5_buf_p = (file + 1); /* increment input buf by size of file record */
-    *bytes_left -= sizeof(struct darshan_hdf5_file);
+static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf)
+{
+    struct darshan_hdf5_file *file = (struct darshan_hdf5_file *)hdf5_buf;
+    int ret;
+
+    ret = darshan_log_putmod(fd, DARSHAN_HDF5_MOD, file,
+        sizeof(struct darshan_hdf5_file));
+    if(ret < 0)
+        return(-1);
 
     return(0);
 }

darshan-util/darshan-logutils.h

@@ -16,9 +16,6 @@
 
 #include "darshan-log-format.h"
 
-/* default to a buffer size of 4 MiB for compression/decompression */
-#define DARSHAN_DEF_COMP_BUF_SZ (4*1024*1024)
-
 /* TODO: can we refactor this def out of header? modules currently poke at swap_flag
  * directly, but other than that there's no reason for another module to know this
  * definition.
@@ -26,14 +23,24 @@
 struct darshan_fd_s
 {
     int fildes;
+    int o_flags;
     int64_t pos;
     enum darshan_comp_type comp_type;
+    char logfile_path[PATH_MAX];
     char version[8];
     int swap_flag;
     char *exe_mnt_data;
     struct darshan_log_map job_map;
     struct darshan_log_map rec_map;
     struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
+    int err;
+
+    /* XXX */
+    void *dz_strm;
+    int dz_size;
+    unsigned char *dz_buf;
+    int dz_eor;
+    int dz_prev_reg_id;
 };
 typedef struct darshan_fd_s* darshan_fd;
 
@@ -46,11 +53,13 @@ struct darshan_record_ref
 struct darshan_mod_logutil_funcs
 {
     int (*log_get_record)(
-        void** mod_buf_p,
-        int* mod_bytes_left,
-        void** file_rec,
-        darshan_record_id* rec_id,
-        int byte_swap_flag
+        darshan_fd fd,
+        void* buf,
+        darshan_record_id* rec_id
+    );
+    int (*log_put_record)(
+        darshan_fd fd,
+        void *buf
     );
     void (*log_print_record)(
         void *file_rec,
@@ -70,8 +79,6 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS];
 
 darshan_fd darshan_log_open(const char *name);
 darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type);
-int darshan_log_getheader(darshan_fd fd, struct darshan_header *header);
-int darshan_log_putheader(darshan_fd fd);
 int darshan_log_getjob(darshan_fd fd, struct darshan_job *job);
 int darshan_log_putjob(darshan_fd fd, struct darshan_job *job);
 int darshan_log_getexe(darshan_fd fd, char *buf);
@@ -83,7 +90,7 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts,
 int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash);
 int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash);
 int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
-    void *mod_buf, int *mod_buf_sz);
+    void *buf, int len);
 int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
     void *mod_buf, int mod_buf_sz);
 void darshan_log_close(darshan_fd file);

darshan-util/darshan-mpiio-logutils.c

@@ -30,43 +30,60 @@ char *mpiio_f_counter_names[] = {
 };
 #undef X