darshan-logutils.c 43.1 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7
#include "darshan-util-config.h"
8
9
#include <stdio.h>
#include <string.h>
10
#include <assert.h>
11
#include <stdlib.h>
12
#include <unistd.h>
13
#include <inttypes.h>
14
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#include "darshan-logutils.h"

/* isn't there a clever c way to avoid this? */
char *darshan_names[] = {
    "CP_INDEP_OPENS",
    "CP_COLL_OPENS",               /* count of MPI collective opens */
    "CP_INDEP_READS",              /* count of independent MPI reads */
    "CP_INDEP_WRITES",             /* count of independent MPI writes */
    "CP_COLL_READS",               /* count of collective MPI reads */
    "CP_COLL_WRITES",              /* count of collective MPI writes */
    "CP_SPLIT_READS",              /* count of split collective MPI reads */
    "CP_SPLIT_WRITES",             /* count of split collective MPI writes */
    "CP_NB_READS",                 /* count of nonblocking MPI reads */
    "CP_NB_WRITES",                /* count of nonblocking MPI writes */
    "CP_SYNCS",                    /* count of MPI_File_sync */
    "CP_POSIX_READS",              /* count of posix reads */
    "CP_POSIX_WRITES",             /* count of posix writes */
    "CP_POSIX_OPENS",              /* count of posix opens */
    "CP_POSIX_SEEKS",              /* count of posix seeks */
    "CP_POSIX_STATS",              /* count of posix stat/lstat/fstats */
    "CP_POSIX_MMAPS",              /* count of posix mmaps */
    "CP_POSIX_FREADS",
    "CP_POSIX_FWRITES",
    "CP_POSIX_FOPENS",
    "CP_POSIX_FSEEKS",
    "CP_POSIX_FSYNCS",
    "CP_POSIX_FDSYNCS",
45
46
47
    "CP_INDEP_NC_OPENS",
    "CP_COLL_NC_OPENS",
    "CP_HDF5_OPENS",
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
    "CP_COMBINER_NAMED",           /* count of each MPI datatype category */
    "CP_COMBINER_DUP",
    "CP_COMBINER_CONTIGUOUS",
    "CP_COMBINER_VECTOR",
    "CP_COMBINER_HVECTOR_INTEGER",
    "CP_COMBINER_HVECTOR",
    "CP_COMBINER_INDEXED",
    "CP_COMBINER_HINDEXED_INTEGER",
    "CP_COMBINER_HINDEXED",
    "CP_COMBINER_INDEXED_BLOCK",
    "CP_COMBINER_STRUCT_INTEGER",
    "CP_COMBINER_STRUCT",
    "CP_COMBINER_SUBARRAY",
    "CP_COMBINER_DARRAY",
    "CP_COMBINER_F90_REAL",
    "CP_COMBINER_F90_COMPLEX",
    "CP_COMBINER_F90_INTEGER",
    "CP_COMBINER_RESIZED",
    "CP_HINTS",                     /* count of MPI hints used */
    "CP_VIEWS",                     /* count of MPI set view calls */
    "CP_MODE",                      /* mode of file */
    "CP_BYTES_READ",                /* total bytes read */
    "CP_BYTES_WRITTEN",             /* total bytes written */
    "CP_MAX_BYTE_READ",             /* highest offset byte read */
    "CP_MAX_BYTE_WRITTEN",          /* highest offset byte written */
    "CP_CONSEC_READS",              /* count of consecutive reads */
    "CP_CONSEC_WRITES",             /* count of consecutive writes */
    "CP_SEQ_READS",                 /* count of sequential reads */
    "CP_SEQ_WRITES",                /* count of sequential writes */
    "CP_RW_SWITCHES",
    "CP_MEM_NOT_ALIGNED",           /* count of accesses not mem aligned */
    "CP_MEM_ALIGNMENT",             /* mem alignment in bytes */
    "CP_FILE_NOT_ALIGNED",          /* count of accesses not file aligned */
    "CP_FILE_ALIGNMENT",            /* file alignment in bytes */
82
83
    "CP_MAX_READ_TIME_SIZE",
    "CP_MAX_WRITE_TIME_SIZE",
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
    "CP_SIZE_READ_0_100",           /* count of posix read size ranges */
    "CP_SIZE_READ_100_1K",
    "CP_SIZE_READ_1K_10K",
    "CP_SIZE_READ_10K_100K",
    "CP_SIZE_READ_100K_1M",
    "CP_SIZE_READ_1M_4M",
    "CP_SIZE_READ_4M_10M",
    "CP_SIZE_READ_10M_100M",
    "CP_SIZE_READ_100M_1G",
    "CP_SIZE_READ_1G_PLUS",
    "CP_SIZE_WRITE_0_100",          /* count of posix write size ranges */
    "CP_SIZE_WRITE_100_1K",
    "CP_SIZE_WRITE_1K_10K",
    "CP_SIZE_WRITE_10K_100K",
    "CP_SIZE_WRITE_100K_1M",
    "CP_SIZE_WRITE_1M_4M",
    "CP_SIZE_WRITE_4M_10M",
    "CP_SIZE_WRITE_10M_100M",
    "CP_SIZE_WRITE_100M_1G",
    "CP_SIZE_WRITE_1G_PLUS",
    "CP_SIZE_READ_AGG_0_100",       /* count of MPI read size ranges */
    "CP_SIZE_READ_AGG_100_1K",
    "CP_SIZE_READ_AGG_1K_10K",
    "CP_SIZE_READ_AGG_10K_100K",
    "CP_SIZE_READ_AGG_100K_1M",
    "CP_SIZE_READ_AGG_1M_4M",
    "CP_SIZE_READ_AGG_4M_10M",
    "CP_SIZE_READ_AGG_10M_100M",
    "CP_SIZE_READ_AGG_100M_1G",
    "CP_SIZE_READ_AGG_1G_PLUS",
    "CP_SIZE_WRITE_AGG_0_100",      /* count of MPI write size ranges */
    "CP_SIZE_WRITE_AGG_100_1K",
    "CP_SIZE_WRITE_AGG_1K_10K",
    "CP_SIZE_WRITE_AGG_10K_100K",
    "CP_SIZE_WRITE_AGG_100K_1M",
    "CP_SIZE_WRITE_AGG_1M_4M",
    "CP_SIZE_WRITE_AGG_4M_10M",
    "CP_SIZE_WRITE_AGG_10M_100M",
    "CP_SIZE_WRITE_AGG_100M_1G",
    "CP_SIZE_WRITE_AGG_1G_PLUS",
    "CP_EXTENT_READ_0_100",          /* count of MPI read extent ranges */
    "CP_EXTENT_READ_100_1K",
    "CP_EXTENT_READ_1K_10K",
    "CP_EXTENT_READ_10K_100K",
    "CP_EXTENT_READ_100K_1M",
    "CP_EXTENT_READ_1M_4M",
    "CP_EXTENT_READ_4M_10M",
    "CP_EXTENT_READ_10M_100M",
    "CP_EXTENT_READ_100M_1G",
    "CP_EXTENT_READ_1G_PLUS",
    "CP_EXTENT_WRITE_0_100",         /* count of MPI write extent ranges */
    "CP_EXTENT_WRITE_100_1K",
    "CP_EXTENT_WRITE_1K_10K",
    "CP_EXTENT_WRITE_10K_100K",
    "CP_EXTENT_WRITE_100K_1M",
    "CP_EXTENT_WRITE_1M_4M",
    "CP_EXTENT_WRITE_4M_10M",
    "CP_EXTENT_WRITE_10M_100M",
    "CP_EXTENT_WRITE_100M_1G",
    "CP_EXTENT_WRITE_1G_PLUS",
    "CP_STRIDE1_STRIDE",             /* the four most frequently appearing strides */
    "CP_STRIDE2_STRIDE",
    "CP_STRIDE3_STRIDE",
    "CP_STRIDE4_STRIDE",
    "CP_STRIDE1_COUNT",              /* count of each of the most frequent strides */
    "CP_STRIDE2_COUNT",
    "CP_STRIDE3_COUNT",
    "CP_STRIDE4_COUNT",
    "CP_ACCESS1_ACCESS",
    "CP_ACCESS2_ACCESS",
    "CP_ACCESS3_ACCESS",
    "CP_ACCESS4_ACCESS",
    "CP_ACCESS1_COUNT",
    "CP_ACCESS2_COUNT",
    "CP_ACCESS3_COUNT",
    "CP_ACCESS4_COUNT",
160
    "CP_DEVICE",
161
    "CP_SIZE_AT_OPEN",
162
163
164
165
    "CP_FASTEST_RANK",
    "CP_FASTEST_RANK_BYTES",
    "CP_SLOWEST_RANK",
    "CP_SLOWEST_RANK_BYTES",
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

    "CP_NUM_INDICES"
};

/* isn't there a clever c way to avoid this? */
char *darshan_f_names[] = {
    "CP_F_OPEN_TIMESTAMP",        /* timestamp of first open */
    "CP_F_READ_START_TIMESTAMP",  /* timestamp of first read */
    "CP_F_WRITE_START_TIMESTAMP", /* timestamp of first write */
    "CP_F_CLOSE_TIMESTAMP",       /* timestamp of last close */
    "CP_F_READ_END_TIMESTAMP",    /* timestamp of last read */
    "CP_F_WRITE_END_TIMESTAMP",   /* timestamp of last write */
    "CP_F_POSIX_READ_TIME",       /* cumulative posix read time */
    "CP_F_POSIX_WRITE_TIME",      /* cumulative posix write time */
    "CP_F_POSIX_META_TIME",       /* cumulative posix meta time */
    "CP_F_MPI_META_TIME",         /* cumulative mpi-io metadata time */
    "CP_F_MPI_READ_TIME",         /* cumulative mpi-io read time */
    "CP_F_MPI_WRITE_TIME",        /* cumulative mpi-io write time */
184
185
    "CP_F_MAX_READ_TIME",
    "CP_F_MAX_WRITE_TIME",
186
187
188
189
190
191
    "CP_F_FASTEST_RANK_TIME",
    "CP_F_SLOWEST_RANK_TIME",
    "CP_F_VARIANCE_RANK_TIME",
    "CP_F_VARIANCE_RANK_BYTES",

    "CP_F_NUM_INDICES"
192
193
};

194
195
196
197
198
199
200
/* function pointers so that we can switch functions depending on what file
 * version is detected
 */
int (*getjob_internal)(darshan_fd file, struct darshan_job *job);
int (*getfile_internal)(darshan_fd fd, 
    struct darshan_job *job, 
    struct darshan_file *file);
201
#define JOB_SIZE_124 28
202
#define JOB_SIZE_200 56
203

204
/* internal routines for parsing different file versions */
205
static int getjob_internal_201(darshan_fd file, struct darshan_job *job);
206
207
208
209
210
211
212
213
214
215
static int getjob_internal_200(darshan_fd file, struct darshan_job *job);
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file);
static int getjob_internal_124(darshan_fd file, struct darshan_job *job);
static int getfile_internal_124(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file);
static int getfile_internal_122(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file);
static int getfile_internal_121(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file);
216
217
static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file, int n_counters, int n_fcounters);
218
static void shift_missing_1_24(struct darshan_file* file);
219
220
static void shift_missing_1_22(struct darshan_file* file);
static void shift_missing_1_21(struct darshan_file* file);
221

222
223
static int darshan_log_seek(darshan_fd fd, int64_t offset);
static int darshan_log_read(darshan_fd fd, void* buf, int len);
224
static int darshan_log_write(darshan_fd fd, void* buf, int len);
225
226
static const char* darshan_log_error(darshan_fd fd, int* errnum);

227
/* a rather crude API for accessing raw binary darshan files */
228
darshan_fd darshan_log_open(const char *name, const char* mode)
229
{
230
231
232
233
234
235
    int test_fd;
    uint8_t magic[2];
    int ret;
    int try_bz2 = 1;
    int len = strlen(name);

236
237
238
239
    /* we only allows "w" or "r" modes, nothing fancy */
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');

240
241
242
243
244
    darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

245
246
247
248
249
250
251
252
253
    tmp_fd->mode[0] = mode[0];
    tmp_fd->mode[1] = mode[1];
    tmp_fd->name  = strdup(name);
    if(!tmp_fd->name)
    {
        free(tmp_fd);
        return(NULL);
    }

254
255
256
257
258
259
260
261
    if(strcmp(mode, "r") == 0)
    {
        /* Try to detect if existing file is a bzip2 file or not.  Both 
         * libbz2 and libz will fall back to normal I/O (without compression) 
         * automatically, so we need to do some detection manually up front 
         * in order to get a chance to try both compression formats.
         */
        test_fd = open(name, O_RDONLY);
262
        if(test_fd < 0)
263
264
        {
            perror("open");
265
            free(tmp_fd->name);
266
267
268
269
270
271
272
273
            free(tmp_fd);
            return(NULL);
        }
        ret = read(test_fd, &magic, 2);
        if(ret != 2)
        {
            fprintf(stderr, "Error: failed to read any data from %s.\n", 
                name);
274
            free(tmp_fd->name);
275
276
277
278
279
280
281
282
283
284
285
            free(tmp_fd);
            close(test_fd);
            return(NULL);
        }
        /* header magic for bz2 */
        if(magic[0] != 0x42 && magic[1] != 0x5A)
        {
            try_bz2 = 0;
        }
        close(test_fd);
    }
286

287
288
289
290
291
292
    if(strcmp(mode, "w") == 0)
    {
        /* TODO: is this the behavior that we want? */
        /* if we are writing a new file, go by the file extension to tell
         * whether to use bz2 or not?
         */
293
        if(len >= 3 && name[len-3] == 'b' && name[len-2] == 'z' && name[len-1] == '2')
294
295
296
297
298
            try_bz2 = 1;
        else
            try_bz2 = 0;
    }

299
#ifdef HAVE_LIBBZ2
300
301
302
    if(try_bz2)
    {
        tmp_fd->bzf = BZ2_bzopen(name, mode);
303
        if(!tmp_fd->bzf)
304
        {
305
306
307
            free(tmp_fd->name);
            free(tmp_fd);
            return(NULL);
308
        }
309
        return(tmp_fd);
310
    }
311
312
313
314
315
316
317
#else
    if(try_bz2)
    {
        fprintf(stderr, "Error: this Darshan build does not support bz2 files.\n");
        fprintf(stderr, "Error: please install libbz2-dev and reconfigure.\n");
        return(NULL);
    }
318
319
#endif

320
    tmp_fd->gzf = gzopen(name, mode);
321
322
    if(!tmp_fd->gzf)
    {
323
        free(tmp_fd->name);
324
325
326
327
        free(tmp_fd);
        tmp_fd = NULL;
    }
    return tmp_fd;
328
329
}

330
/* darshan_log_getjob()
331
332
333
 *
 * returns 0 on success, -1 on failure
 */
334
int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
335
336
{
    int ret;
337
    char buffer[DARSHAN_JOB_METADATA_LEN];
338

339
340
341
    ret = darshan_log_seek(file, 0);
    if(ret < 0)
        return(ret);
342

343
344
345
    /* read version number first so we know how to digest the rest of the
     * file
     */
346
    ret = darshan_log_read(file, file->version, 10);
347
    if(ret < 10)
348
    {
349
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
350
351
352
        return(-1);
    }

Philip Carns's avatar
Philip Carns committed
353
354
355
356
357
358
359
    if(strcmp(file->version, "2.02") == 0)
    {
        getjob_internal = getjob_internal_201;
        getfile_internal = getfile_internal_200;
        file->job_struct_size = sizeof(*job);
    }
    else if(strcmp(file->version, "2.01") == 0)
360
    {
361
        getjob_internal = getjob_internal_201;
362
        getfile_internal = getfile_internal_200;
363
        file->job_struct_size = sizeof(*job);
364
    }
365
366
367
368
369
370
    else if(strcmp(file->version, "2.00") == 0)
    {
        getjob_internal = getjob_internal_200;
        getfile_internal = getfile_internal_200;
        file->job_struct_size = JOB_SIZE_200;
    }
371
    else if(strcmp(file->version, "1.24") == 0)
372
    {
373
374
        getjob_internal = getjob_internal_124;
        getfile_internal = getfile_internal_124;
375
        file->job_struct_size = JOB_SIZE_124;
376
    }
377
    else if(strcmp(file->version, "1.23") == 0)
378
    {
379
380
381
        /* same as 1.24, except that mnt points may be incorrect */
        getjob_internal = getjob_internal_124;
        getfile_internal = getfile_internal_124;
382
        file->job_struct_size = JOB_SIZE_124;
383
    }
384
    else if(strcmp(file->version, "1.22") == 0)
385
    {
386
387
        getjob_internal = getjob_internal_124;
        getfile_internal = getfile_internal_122;
388
        file->job_struct_size = JOB_SIZE_124;
389
    }
390
    else if(strcmp(file->version, "1.21") == 0)
391
    {
392
        getjob_internal = getjob_internal_124;
Philip Carns's avatar
Philip Carns committed
393
        getfile_internal = getfile_internal_121;
394
        file->job_struct_size = JOB_SIZE_124;
395
396
    }
    else
397
    {
398
399
400
        fprintf(stderr, "Error: incompatible darshan file.\n");
        fprintf(stderr, "Error: expected version %s, but got %s\n", 
                CP_VERSION, file->version);
401
        return(-1);
402
403
    }

404
    ret = getjob_internal(file, job);
405
406
407

    if (ret == 0)
    {
408
#ifdef HAVE_STRNDUP
409
        char *metadata = strndup(job->metadata, sizeof(job->metadata));
410
411
412
#else
        char *metadata = strndup(job->metadata);
#endif
413
414
        char *kv;
        char *key;
415
        char *value;
416
417
        char *save;

418
419
420
        for(kv=strtok_r(metadata, "\n", &save);
            kv != NULL;
            kv=strtok_r(NULL, "\n", &save))
421
        {
422
423
424
425
426
427
428
429
430
431
432
433
            /* NOTE: we intentionally only split on the first = character.
             * There may be additional = characters in the value portion
             * (for example, when storing mpi-io hints).
             */
            strcpy(buffer, kv);
            key = buffer;
            value = index(buffer, '=');
            if(!value)
                continue;
            /* convert = to a null terminator to split key and value */
            value[0] = '\0';
            value++;
434
            if (strcmp(key, "prev_ver") == 0)
435
            {
436
                strncpy(job->version_string, value, sizeof(job->version_string));
437
            }
438
439
        }
        free(metadata);
440
441
    }

442
443
    return(ret);
}
444

445
446
447
448
449
450
451
452
/* darshan_putjob()
 * write job header in gzfile
 *
 * return 0 on success, -1 on failure.
 */
int darshan_log_putjob(darshan_fd file, struct darshan_job *job)
{
    struct darshan_job job_copy;
453
    char    pv_str[64];
454
455
    int     ret;

456
457
    ret = darshan_log_seek(file, 0);
    if(ret < 0)
458
459
        return(ret);

460
461
    memset(&job_copy, 0, sizeof(job_copy));
    memcpy(&job_copy, job, sizeof(job_copy));
462
    sprintf(pv_str, "prev_ver=%s\n", job->version_string);
463
464
    sprintf(job_copy.version_string, "%s", CP_VERSION);
    strncat(job_copy.metadata, pv_str, strlen(pv_str));
465
466
    job_copy.magic_nr = CP_MAGIC_NR;

467
    ret = darshan_log_write(file, &job_copy, sizeof(job_copy));
468
469
470
471
472
473
474
475
476
    if (ret != sizeof(job_copy))
    {
        fprintf(stderr, "Error: failed to write job header: %d\n", ret);
        return(-1);
    }

    return(0);
}

477
478
479
480
481
482
483
/* darshan_log_getfile()
 *
 * return 1 if file record found, 0 on eof, and -1 on error
 */
int darshan_log_getfile(darshan_fd fd, struct darshan_job *job, struct darshan_file *file)
{
    int ret;
484

485
486
487
    ret = getfile_internal(fd, job, file);

    return(ret);
488
489
}

490
491
492
493
494
495
496
497
/* darshan_log_putfile()
 *
 * return 0 if file record written, -1 on error.
 */
int darshan_log_putfile(darshan_fd fd, struct darshan_job *job, struct darshan_file *file)
{
    int     ret;

498
499
500
501
502
503
    if(fd->pos < CP_JOB_RECORD_SIZE)
    {
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
        if(ret < 0)
            return(ret);
    }
504

505
    ret = darshan_log_write(fd, file, sizeof(*file));
506
507
508
509
510
511
512
513
514
    if (ret != sizeof(*file))
    {
        fprintf(stderr, "Error: writing file record failed: %d\n", ret);
        return(-1);
    }

    return(0);
}

515
516
517
518
519
520
/* darshan_log_getmounts()
 * 
 * retrieves mount table information from the log.  Note that devs, mnt_pts,
 * and fs_types are arrays that will be allocated by the function and must
 * be freed by the caller.  count will indicate the size of the arrays
 */
521
int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char***
522
    fs_types, int* count)
523
524
525
526
527
528
{
    int ret;
    char* pos;
    int array_index = 0;
    char buf[CP_EXE_LEN+1];

529
530
531
    ret = darshan_log_seek(fd, fd->job_struct_size);
    if(ret < 0)
        return(ret);
532

533
    ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1));
534
535
    if (ret < (CP_EXE_LEN + 1))
    {
536
        perror("darshan_log_read");
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
        return(-1);
    }

    /* count entries */
    *count = 0;
    pos = buf;
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
556
    *devs = malloc((*count)*sizeof(int64_t));
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
    assert(*devs);
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);
    
    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
    while((pos = strrchr(buf, '\n')) != NULL)
    {
        /* overestimate string lengths */
        (*mnt_pts)[array_index] = malloc(CP_EXE_LEN);
        assert((*mnt_pts)[array_index]);
        (*fs_types)[array_index] = malloc(CP_EXE_LEN);
        assert((*fs_types)[array_index]);
        
Philip Carns's avatar
Philip Carns committed
574
        ret = sscanf(++pos, "%" PRId64 "\t%s\t%s", &(*devs)[array_index],
575
576
            (*fs_types)[array_index], (*mnt_pts)[array_index]);

577
578
579
580
581
582
583
584
585
        if(ret != 3)
        {
            fprintf(stderr, "Error: poorly formatted mount table in log file.\n");
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }
586

587
588
589
    return (0);
}

590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
/* darshan_log_putmounts
 *
 * encode mount information back into mtab format.
 *
 * returns 0 on success, -1 on failure.
 */
int darshan_log_putmounts(darshan_fd fd, int64_t* devs, char** mnt_pts, char** fs_types, int count)
{
    int     ret;
    char    line[1024];
    int     i;

    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%" PRId64 "\t%s\t%s",
                devs[i], fs_types[i], mnt_pts[i]);
606
        ret = darshan_log_write(fd, line, strlen(line));
607
608
609
610
611
612
        if (ret != strlen(line))
        {
            fprintf(stderr, "Error: failed to write mount entry: %d\n", ret);
            return(-1);
        }
    }
613
614
615
616
617
618
619
620

    /* seek ahead to end of exe region, will be zero filled */
    ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
    if (ret)
    {
        fprintf(stderr, "Error: forward seek failed: %d\n", CP_JOB_RECORD_SIZE);
    }

621
622
    return(0);
}
623

624
int darshan_log_getexe(darshan_fd fd, char *buf)
625
626
{
    int ret;
627
    char* newline;
628

629
630
631
    ret = darshan_log_seek(fd, fd->job_struct_size);
    if(ret < 0)
        return(ret);
632

633
    ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1));
634
635
    if (ret < (CP_EXE_LEN + 1))
    {
636
        perror("darshan_log_read");
637
638
        return(-1);
    }
639

640
641
642
643
644
645
646
647
    /* this call is only supposed to return the exe string, but starting in
     * log format 1.23 there could be a table of mount entry information
     * after the exe.  Look for newline character and truncate there.
     */
    newline = strchr(buf, '\n');
    if(newline)
        *newline = '\0';

648
649
650
    return (0);
}

651
652
653
654
655
656
657
658
659
660
661
/* darshan_log_putexe()
 *
 * Write the exe string to the log.
 *
 * return 0 on success, -1 on failure.
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int     ret;
    int     len;

662
663
    ret = darshan_log_seek(fd, sizeof(struct darshan_job));
    if(ret < 0)
664
665
666
667
        return(ret);

    len = strlen(buf);

668
    ret = darshan_log_write(fd, buf, len);
669
670
671
672
673
674
675
676
677
    if (ret != len)
    {
        fprintf(stderr, "Error: failed to write exe info: %d\n", ret);
        ret = -1;
    }

    return(ret);
}

678
void darshan_log_close(darshan_fd file)
679
{
680
681
682
683
684
685
686
687
#ifdef HAVE_LIBBZ2
    if(file->bzf)
        BZ2_bzclose(file->bzf);
#endif

    if(file->gzf)
        gzclose(file->gzf);

688
    free(file->name);
689
    free(file);
690
691
}

692
693
694
695
696
697
/* darshan_log_print_version_warnings()
 *
 * Print summary of any problems with the detected log format
 */
void darshan_log_print_version_warnings(struct darshan_job *job)
{
Philip Carns's avatar
Philip Carns committed
698
    if(strcmp(job->version_string, "2.02") == 0)
699
700
701
702
    {
        /* current version */
        return;
    }
703

Philip Carns's avatar
Philip Carns committed
704
705
706
707
708
709
710
    if(strcmp(job->version_string, "2.01") == 0)
    {
        printf("# WARNING: version 2.01 log format has the following limitations:\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
        return;
    }

711
712
    if(strcmp(job->version_string, "2.00") == 0)
    {
Philip Carns's avatar
Philip Carns committed
713
714
        printf("# WARNING: version 2.00 log format has the following limitations:\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
715
716
        return;
    }
717
 
718
    if(strcmp(job->version_string, "1.24") == 0)
719
    {
720
721
722
723
724
725
726
727
728
        printf("# WARNING: version 1.24 log format does not support the following parameters:\n");
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
729
730
731
        printf("# WARNING: version 1.24 log format has the following limitations:\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
732
733
        return;
    }
734
    
735
736
    if(strcmp(job->version_string, "1.23") == 0)
    {
737
738
739
740
741
742
743
744
745
        printf("# WARNING: version 1.23 log format does not support the following parameters:\n");
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
746
747
748
749
        printf("# WARNING: version 1.23 log format has the following limitations:\n");
        printf("# - may have incorrect mount point mappings for files with rank > 0.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
750
        return;
751
752
    }

753
754
755
756
    if(strcmp(job->version_string, "1.22") == 0)
    {
        printf("# WARNING: version 1.22 log format does not support the following parameters:\n");
        printf("#   CP_DEVICE\n");
757
        printf("#   CP_SIZE_AT_OPEN\n");
758
759
760
761
762
763
764
765
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
766
767
768
769
770
        printf("# WARNING: version 1.22 log format has the following limitations:\n");
        printf("# - does not record mounted file systems, mount points, or fs types.\n");
        printf("# - attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
771
772
        return;
    }
773
774
775
776
777
778
779
780
781

    if(strcmp(job->version_string, "1.21") == 0)
    {
        printf("# WARNING: version 1.21 log format does not support the following parameters:\n");
        printf("#   CP_INDEP_NC_OPENS\n");
        printf("#   CP_COLL_NC_OPENS\n");
        printf("#   CP_HDF5_OPENS\n");
        printf("#   CP_MAX_READ_TIME_SIZE\n");
        printf("#   CP_MAX_WRITE_TIME_SIZE\n");
782
        printf("#   CP_DEVICE\n");
783
        printf("#   CP_SIZE_AT_OPEN\n");
784
785
        printf("#   CP_F_MAX_READ_TIME\n");
        printf("#   CP_F_MAX_WRITE_TIME\n");
786
787
788
789
790
791
792
793
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
794
795
796
797
798
        printf("# WARNING: version 1.21 log format has the following limitations:\n");
        printf("# - does not record mounted file systems, mount points, or fs types.\n");
        printf("# - attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
799
800
801
802
803
804
805
806
        return;
    }

    fprintf(stderr, "Error: version %s not supported by parser.\n",
        job->version_string);
    return;
}

807
808
809
810
/* shift_missing_1_21()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.21
811
 */
812
/*******************************
Philip Carns's avatar
Philip Carns committed
813
 * version 1.21 to 2.00 differences 
814
815
816
817
818
819
820
821
 * - added:
 *   - CP_INDEP_NC_OPENS
 *   - CP_COLL_NC_OPENS
 *   - CP_HDF5_OPENS
 *   - CP_MAX_READ_TIME_SIZE
 *   - CP_MAX_WRITE_TIME_SIZE
 *   - CP_DEVICE
 *   - CP_SIZE_AT_OPEN
822
823
824
825
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
826
827
 *   - CP_F_MAX_READ_TIME
 *   - CP_F_MAX_WRITE_TIME
828
829
830
831
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
832
 * - changed params:
833
834
835
 *   - CP_FILE_RECORD_SIZE: 1184 to 1328
 *   - CP_NUM_INDICES: 133 to 144
 *   - CP_F_NUM_INDICES: 12 to 18
836
 */
837
838
839
840
841
842
843
844
845
static void shift_missing_1_21(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_INDEP_NC_OPENS,
        CP_COLL_NC_OPENS,
        CP_HDF5_OPENS,
        CP_MAX_READ_TIME_SIZE,
        CP_MAX_WRITE_TIME_SIZE,
846
        CP_DEVICE,
847
        CP_SIZE_AT_OPEN,
848
849
850
851
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
852
853
854
855
        -1};
    int missing_f_counters[] = {
        CP_F_MAX_READ_TIME,
        CP_F_MAX_WRITE_TIME,
856
857
858
859
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

    return;
}
896
897
898
899
900
901

/* shift_missing_1_22()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.22
 */
902
/*******************************
Philip Carns's avatar
Philip Carns committed
903
 * version 1.22 to 2.00 differences
904
905
906
907
 *
 * - added:
 *   - CP_DEVICE
 *   - CP_SIZE_AT_OPEN
908
909
910
911
912
913
914
915
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
916
 * - changed params:
917
918
919
 *   - CP_FILE_RECORD_SIZE: 1240 to 1328
 *   - CP_NUM_INDICES: 138 to 144
 *   - CP_F_NUM_INDICES: 14 to 18
920
 */
921
922
923
924
925
static void shift_missing_1_22(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_DEVICE,
926
        CP_SIZE_AT_OPEN,
927
928
929
930
931
932
933
934
935
936
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
        -1};
    int missing_f_counters[] = {
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

971
972
    return;
}
973

974
975
976
977
978
979
980
981
982
/* shift_missing_1_24()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.24
 */
/*******************************
 * version 1.24 to 2.00 differences
 *
 * - added:
983
984
985
986
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
 * - changed params:
 *   - CP_FILE_RECORD_SIZE: ? to 1328
 *   - CP_NUM_INDICES: 140 to 144
 *   - CP_F_NUM_INDICES: 14 to 18
 */
static void shift_missing_1_24(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
        -1};
    int missing_f_counters[] = {
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

    return;
}

1047
static int getjob_internal_201(darshan_fd file, struct darshan_job *job)
1048
1049
1050
{
    int ret;

1051
1052
1053
    ret = darshan_log_seek(file, 0);
    if(ret < 0)
        return(ret);
1054

1055
    ret = darshan_log_read(file, job, sizeof(*job));
1056
1057
    if (ret < sizeof(*job))
    {
1058
        fprintf(stderr, "Error: invalid log file (too short).\n");
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
        return(-1);
    }

    if(job->magic_nr == CP_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        file->swap_flag = 0;
        return(0);
    }

    /* try byte swapping */
    DARSHAN_BSWAP64(&job->magic_nr);
    if(job->magic_nr == CP_MAGIC_NR)
    {
        file->swap_flag = 1;
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
1078
        DARSHAN_BSWAP64(&job->jobid);
1079
1080
1081
1082
1083
1084
1085
1086
        return(0);
    }

    /* otherwise this file is just broken */
    fprintf(stderr, "Error: bad magic number in darshan file.\n");
    return(-1);
}

1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104

static int getjob_internal_200(darshan_fd file, struct darshan_job *job)
{
    int ret;
    struct darshan_job_200
    {
        char version_string[8];
        int64_t magic_nr;
        int64_t uid;
        int64_t start_time;
        int64_t end_time;
        int64_t nprocs;
        int64_t jobid;
    } job_200;

    memset(job, 0, sizeof(job_200));
    memset(job, 0, sizeof(*job));

1105
1106
1107
    ret = darshan_log_seek(file, 0);
    if(ret < 0)
        return(ret);
1108

1109
    ret = darshan_log_read(file, &job_200, sizeof(job_200));
1110
1111
    if (ret < sizeof(job_200))
    {
1112
        fprintf(stderr, "Error: invalid log file (too short).\n");
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
        return(-1);
    }

    memcpy(job->version_string, job_200.version_string, 8);
    job->magic_nr   = job_200.magic_nr;
    job->uid        = job_200.uid;
    job->start_time = job_200.start_time;
    job->end_time   = job_200.end_time;
    job->nprocs     = job_200.nprocs;
    job->jobid      = job_200.jobid;

    if(job->magic_nr == CP_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        file->swap_flag = 0;
        return(0);
    }

    /* try byte swapping */
    DARSHAN_BSWAP64(&job->magic_nr);
    if(job->magic_nr == CP_MAGIC_NR)
    {
        file->swap_flag = 1;
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
        return(0);
    }

    /* otherwise this file is just broken */
    fprintf(stderr, "Error: bad magic number in darshan file.\n");
    return(-1);
}

1149
1150
1151
1152
1153
1154
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;
    const char* err_string;
    int i;
1155

1156
1157
1158
1159
1160
1161
    if(fd->pos < CP_JOB_RECORD_SIZE)
    {
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
        if(ret < 0)
            return(ret);
    }
1162
1163
1164
1165
1166
1167

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
    memset(file, 0, sizeof(&file));

1168
    ret = darshan_log_read(fd, file, sizeof(*file));
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
            DARSHAN_BSWAP64(&file->hash);
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

1193
    if(ret == 0)
1194
1195
1196
1197
1198
1199
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
1200
    err_string = darshan_log_error(fd, &ret);
1201
1202
1203
1204
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

1205
1206
1207
/* If we see version 1.24, assume that it is stored in big endian 32 bit
 * format.  Convert up to current format.
 */
1208
1209
static int getjob_internal_124(darshan_fd fd, struct darshan_job *job)
{
1210
1211
1212
1213
1214
1215
1216
    char* buffer;
    int ret;
    uint32_t uid;
    int32_t start_time;
    int32_t end_time;
    int32_t nprocs;

1217
1218
1219
1220
1221
1222
#ifdef WORDS_BIGENDIAN
    fd->swap_flag = 0;
#else
    fd->swap_flag = 1;
#endif

1223
1224
1225
1226
1227
1228
1229
1230
    memset(job, 0, sizeof(*job));

    buffer = (char*)malloc(JOB_SIZE_124);
    if(!buffer)
    {
        return(-1);
    }

1231
1232
1233
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
        return(ret);
1234

1235
    ret = darshan_log_read(fd, buffer, JOB_SIZE_124);
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
    if (ret < JOB_SIZE_124)
    {
        fprintf(stderr, "Error: invalid log file (could not read file record).\n");
        free(buffer);
        return(-1);
    }

    /* pull job header information out of specific bytes in case struct
     * padding is off
     */
    strncpy(job->version_string, buffer, 8);
    uid = *((uint32_t*)&buffer[12]);
    start_time = *((int32_t*)&buffer[16]);
    end_time = *((int32_t*)&buffer[20]);
    nprocs = *((int32_t*)&buffer[24]);

    free(buffer);

1254
1255
1256
1257
1258
1259
1260
1261
    if(fd->swap_flag)
    {
        /* byte swap */
        DARSHAN_BSWAP32(&uid);
        DARSHAN_BSWAP32(&start_time);
        DARSHAN_BSWAP32(&end_time);
        DARSHAN_BSWAP32(&nprocs);
    }
1262
1263
1264
1265
1266

    job->uid += uid;
    job->start_time += start_time;
    job->end_time += end_time;
    job->nprocs += nprocs;
1267
    job->jobid = 0; /* old log versions did not have this field */
1268
1269
1270
1271
1272
    
    /* set magic number */
    job->magic_nr = CP_MAGIC_NR;

    return(0);
1273
1274
1275
1276
}

static int getfile_internal_124(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
1277
{
1278
1279
    int ret;

1280
    ret = getfile_internal_1x(fd, job, file, 140, 14);
1281
1282
1283
1284
1285
1286
    if(ret <= 0)
        return(ret);

    shift_missing_1_24(file);

    return(1);
1287
1288
1289
1290
1291
1292
1293
}

static int getfile_internal_122(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;

1294
1295
    ret = getfile_internal_1x(fd, job, file, 138, 14);
    if(ret <= 0)
1296
1297
1298
1299
        return(ret);

    shift_missing_1_22(file);

1300
    return(1);
1301
1302
1303
1304
1305
1306
1307
}

static int getfile_internal_121(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;

1308
1309
    ret = getfile_internal_1x(fd, job, file, 133, 12);
    if(ret <= 0)
1310
1311
1312
1313
        return(ret);

    shift_missing_1_21(file);

1314
    return(1);
1315
1316
1317
1318
}

static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file, int n_counters, int n_fcounters)
1319
{
1320
1321
1322
1323
1324
1325
1326
1327
1328
    char* buffer;
    int ret;
    const char* err_string;
    int i;
    uint64_t hash;
    int32_t rank;
    int64_t* counters;
    double* fcounters;
    char* name_suffix;
1329
    int FILE_SIZE_1x = (32 + n_counters*8 + n_fcounters*8);
1330

1331
1332
1333
1334
1335
    memset(file, 0, sizeof(*file));

    /* set file pointer if this is the first file record; otherwise pick up
     * where we left off last time
     */
1336
1337
1338
1339
1340
1341
1342
    if(fd->pos < CP_JOB_RECORD_SIZE)
    {
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
        if(ret < 0)
            return(ret);
    }
    
1343
    /* space for file struct, int64 array, and double array */
1344
    buffer = (char*)malloc(FILE_SIZE_1x);
1345
1346
1347
1348
1349
    if(!buffer)
    {
        return(-1);
    }

1350
    ret = darshan_log_read(fd, buffer, FILE_SIZE_1x);
1351

1352
    if(ret > 0 && ret < FILE_SIZE_1x)
1353
1354
1355
1356
1357
1358
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        free(buffer);
        return(-1);
    }
1359
    else if(ret == 0)
1360
1361
1362
1363
1364
1365
1366
1367
    {
        /* hit end of file */
        free(buffer);
        return(0);
    }
    else if(ret <= 0)
    {
        /* all other errors */
1368
        err_string = darshan_log_error(fd, &ret);
1369
1370
1371
1372
1373
1374
1375
1376
1377
        fprintf(stderr, "Error: %s\n", err_string);
        free(buffer);
        return(-1);
    }

    /* got exactly one, correct size record */
    hash = *((int64_t*)&buffer[0]);
    rank = *((int32_t*)&buffer[8]);
    counters = ((int64_t*)&buffer[16]);
1378
1379
    fcounters = ((double*)&buffer[16 + n_counters*8]);
    name_suffix = &buffer[16 + n_counters*8 + n_fcounters*8];
1380
1381
1382
1383
1384
1385
1386


    if(fd->swap_flag)
    {
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&hash);
        DARSHAN_BSWAP32(&rank);
1387
        for(i=0; i<n_counters; i++)
1388
            DARSHAN_BSWAP64(&counters[i]);
1389
        for(i=0; i<n_fcounters; i++)
1390
1391
1392
1393
1394
1395
            DARSHAN_BSWAP64(&fcounters[i]);
    }

    /* assign into new format */
    file->hash = hash;
    file->rank += rank;
1396
1397
    memcpy(file->counters, counters, n_counters*8);
    memcpy(file->fcounters, fcounters, n_fcounters*8);
1398
1399
1400
1401
    memcpy(file->name_suffix, name_suffix, 12);

    free(buffer);
    return(1);
1402
1403
}

1404
1405
1406
1407
1408
1409
1410
1411
/* return amount written on success, -1 on failure.
 */
static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
    int ret;

    if(fd->gzf)
    {
1412
        ret = gzwrite(fd->gzf, buf, len);
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

#ifdef HAVE_LIBBZ2
    if(fd->bzf)
    {
        ret = BZ2_bzwrite(fd->bzf, buf, len);
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }
#endif

    return(-1);
}


1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466