darshan-logutils.c 27.5 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7
#include "darshan-util-config.h"
8
9
#include <stdio.h>
#include <string.h>
10
#include <assert.h>
11
#include <stdlib.h>
12
#include <unistd.h>
13
#include <inttypes.h>
14
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

18
19
20
21
22
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif

23
24
#include "darshan-logutils.h"

25
26
struct darshan_fd_s
{
27
    int pf;
28
29
30
    int64_t pos;
    char mode[2];
    int swap_flag;
31
    char version[8];
32
    char* name;
33
    /* TODO: ultimately store indices here */
34
35
};

36
37
38
39
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
//static const char* darshan_log_error(darshan_fd fd, int* errnum);
40

41
/* a rather crude API for accessing raw binary darshan files */
42
darshan_fd darshan_log_open(const char *name, const char *mode)
43
{
44
    int o_flags;
45

46
47
48
    /* we only allows "w" or "r" modes, nothing fancy */
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');
49
50
51
52
    if(mode[0] == 'r')
        o_flags = O_RDONLY;
    else
        o_flags = O_WRONLY;
53

54
55
56
57
58
    darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

59
60
    /* TODO: why is mode needed??? */
    /* TODO: why is name needed??? */
61
62
63
64
65
66
67
68
69
    tmp_fd->mode[0] = mode[0];
    tmp_fd->mode[1] = mode[1];
    tmp_fd->name  = strdup(name);
    if(!tmp_fd->name)
    {
        free(tmp_fd);
        return(NULL);
    }

70
71
    tmp_fd->pf = open(name, o_flags);
    if(tmp_fd->pf < 0)
72
    {
73
        free(tmp_fd->name);
74
        free(tmp_fd);
75
        return(NULL);
76
    }
77
78

    return(tmp_fd);
79
80
}

81
int darshan_log_getheader(darshan_fd file, struct darshan_header *header)
82
83
{
    int ret;
84

85
86
    ret = darshan_log_seek(file, 0);
    if(ret < 0)
87
88
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
89
        return(ret);
90
    }
91

92
93
94
    /* read header from log file */
    ret = darshan_log_read(file, header, sizeof(*header));
    if(ret < sizeof(*header))
95
    {
96
        fprintf(stderr, "Error: invalid darshan log file (failed to read header).\n");
97
98
99
        return(-1);
    }

100
101
102
103
    /* save the version string -- this can be used to support multiple log versions */
    strncpy(file->version, header->version_string, 8);

    if(header->magic_nr == CP_MAGIC_NR)
104
    {
105
106
107
        /* no byte swapping needed, this file is in host format already */
        file->swap_flag = 0;
        return(0);
108
109
    }

110
111
112
    /* try byte swapping */
    DARSHAN_BSWAP64(&header->magic_nr);
    if(header->magic_nr == CP_MAGIC_NR)
113
    {
114
115
        file->swap_flag = 1;
        return(0);
116
117
    }

118
119
120
    /* otherwise this file is just broken */
    fprintf(stderr, "Error: bad magic number in darshan log file.\n");
    return(-1);
121
}
122

123
/* darshan_log_getjob()
124
 *
125
 * returns 0 on success, -1 on failure
126
 */
127
int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
128
{
129
130
    int ret;
    char buffer[DARSHAN_JOB_METADATA_LEN];
131

132
    ret = darshan_log_seek(file, sizeof(struct darshan_header));
133
    if(ret < 0)
134
    {
135
136
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(ret);
137
138
    }

139
140
141
    /* read the job data from the log file */
    ret = darshan_log_read(file, job, sizeof(*job));
    if(ret < sizeof(*job))
142
    {
143
        fprintf(stderr, "Error: invalid darshan log file (failed to read job data).\n");
144
145
146
        return(-1);
    }

147
    if(file->swap_flag)
148
    {
149
150
151
152
153
154
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
155
156
157
158
159
    }

    return(0);
}

160
int darshan_log_getmap(darshan_fd file, unsigned char **map_buf)
161
162
{
    int ret;
163
164
    struct stat sbuf;
    int map_buf_size;
165

166
    ret = darshan_log_seek(file, sizeof(struct darshan_header) + CP_JOB_RECORD_SIZE);
167
    if(ret < 0)
168
169
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
170
        return(ret);
171
    }
172

173
174
175
176
177
178
179
    /* TODO: use indices map rather than stat to determine offsets */
    /* allocate a buffer to store the (serialized) darshan record map */
    /* NOTE: caller's responsibility to free this allocated map buffer */
    fstat(file->pf, &sbuf);
    map_buf_size = sbuf.st_size - (sizeof(struct darshan_header) + CP_JOB_RECORD_SIZE);
    *map_buf = malloc(map_buf_size);
    if(!(*map_buf))
180
181
        return(-1);

182
183
184
    /* read the record map from the log file */
    ret = darshan_log_read(file, *map_buf, map_buf_size);
    if(ret < map_buf_size)
185
    {
186
187
        fprintf(stderr, "Error: invalid darshan log file (failed to read record map).\n");
        return(-1);
188
189
    }

190
    if(file->swap_flag)
191
    {
192
193
194
195
196
197
198
        /* we need to sort out endianness issues before passing back the serialized buffer */
        /* NOTE: darshan record map serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        unsigned char *buf_ptr = *map_buf;
        darshan_record_id *rec_id_ptr;
        uint32_t *path_len_ptr;
199

200
        while(buf_ptr < (*map_buf + map_buf_size))
201
        {
202
203
204
205
206
207
208
209
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            buf_ptr += *path_len_ptr;

            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
210
211
        }
    }
212

213
    return(0);
214
215
}

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
/* TODO: implement */
/* TODO: could this could be used in darshan-runtime? do we refactor so we aren't maintaining in 2 spots? */
int darshan_log_build_map(unsigned char *map_buf, int map_buf_size, some_struct *rec_hash)
{
    unsigned char *buf_ptr;

    return(0);
}

/* TODO: implement */
/* TODO: could this could be used in darshan-runtime? do we refactor so we aren't maintaining in 2 spots? */
int darshan_log_destroy_map()
{
    return(0);
}

#if 0
/* darshan_log_getfile()
234
 *
235
 * return 1 if file record found, 0 on eof, and -1 on error
236
 */
237
int darshan_log_getfile(darshan_fd fd, struct darshan_job *job, struct darshan_file *file)
238
{
239
    int ret;
240

241
    ret = getfile_internal(fd, job, file);
242

243
    return(ret);
244
}
245

246
int darshan_log_getexe(darshan_fd fd, char *buf)
247
248
{
    int ret;
249
    char* newline;
250

251
252
253
    ret = darshan_log_seek(fd, fd->job_struct_size);
    if(ret < 0)
        return(ret);
254

Philip Carns's avatar
Philip Carns committed
255
256
    ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
    if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
257
    {
258
        perror("darshan_log_read");
259
260
        return(-1);
    }
261

262
263
264
265
266
267
268
269
    /* this call is only supposed to return the exe string, but starting in
     * log format 1.23 there could be a table of mount entry information
     * after the exe.  Look for newline character and truncate there.
     */
    newline = strchr(buf, '\n');
    if(newline)
        *newline = '\0';

270
271
    return (0);
}
272
#endif
273

274
void darshan_log_close(darshan_fd file)
275
{
276
277
    if(file->pf)
        close(file->pf);
278

279
    free(file->name);
280
    free(file);
281
282
}

283
#if 0
284
285
286
287
288
289
/* darshan_log_print_version_warnings()
 *
 * Print summary of any problems with the detected log format
 */
void darshan_log_print_version_warnings(struct darshan_job *job)
{
Philip Carns's avatar
Philip Carns committed
290
    if(strcmp(job->version_string, "2.05") == 0)
291
    {
292
        /* current version */
293
294
295
        return;
    }

Philip Carns's avatar
Philip Carns committed
296
297
298
299
300
301
302
    if(strcmp(job->version_string, "2.04") == 0)
    {
        printf("# WARNING: version 2.04 log format has the following limitations:\n");
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
        return;
    }

Philip Carns's avatar
Philip Carns committed
303
304
    if(strcmp(job->version_string, "2.03") == 0)
    {
Philip Carns's avatar
Philip Carns committed
305
        /* no meaningful change to interpretation of log file, 2.03 just
Philip Carns's avatar
Philip Carns committed
306
307
         * increased the header space available for annotations.
         */
Philip Carns's avatar
Philip Carns committed
308
309
        printf("# WARNING: version 2.03 log format has the following limitations:\n");
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
Philip Carns's avatar
Philip Carns committed
310
311
312
        return;
    }

313
314
315
316
    if(strcmp(job->version_string, "2.02") == 0)
    {
        printf("# WARNING: version 2.01 log format has the following limitations:\n");
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
317
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
318
319
320
        return;
    }

Philip Carns's avatar
Philip Carns committed
321
322
323
    if(strcmp(job->version_string, "2.01") == 0)
    {
        printf("# WARNING: version 2.01 log format has the following limitations:\n");
324
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
325
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
326
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
Philip Carns's avatar
Philip Carns committed
327
328
329
        return;
    }

330
331
    if(strcmp(job->version_string, "2.00") == 0)
    {
Philip Carns's avatar
Philip Carns committed
332
        printf("# WARNING: version 2.00 log format has the following limitations:\n");
333
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
334
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
335
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
336
337
        return;
    }
338
 
339
    if(strcmp(job->version_string, "1.24") == 0)
340
    {
341
342
343
344
345
346
347
348
349
        printf("# WARNING: version 1.24 log format does not support the following parameters:\n");
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
350
        printf("# WARNING: version 1.24 log format has the following limitations:\n");
351
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
352
353
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
354
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
355
356
        return;
    }
357
    
358
359
    if(strcmp(job->version_string, "1.23") == 0)
    {
360
361
362
363
364
365
366
367
368
        printf("# WARNING: version 1.23 log format does not support the following parameters:\n");
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
369
        printf("# WARNING: version 1.23 log format has the following limitations:\n");
370
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
371
372
373
        printf("# - may have incorrect mount point mappings for files with rank > 0.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
374
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
375
        return;
376
377
    }

378
379
380
381
    if(strcmp(job->version_string, "1.22") == 0)
    {
        printf("# WARNING: version 1.22 log format does not support the following parameters:\n");
        printf("#   CP_DEVICE\n");
382
        printf("#   CP_SIZE_AT_OPEN\n");
383
384
385
386
387
388
389
390
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
391
        printf("# WARNING: version 1.22 log format has the following limitations:\n");
392
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
393
394
395
396
        printf("# - does not record mounted file systems, mount points, or fs types.\n");
        printf("# - attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
397
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
398
399
        return;
    }
400
401
402
403
404
405
406
407
408

    if(strcmp(job->version_string, "1.21") == 0)
    {
        printf("# WARNING: version 1.21 log format does not support the following parameters:\n");
        printf("#   CP_INDEP_NC_OPENS\n");
        printf("#   CP_COLL_NC_OPENS\n");
        printf("#   CP_HDF5_OPENS\n");
        printf("#   CP_MAX_READ_TIME_SIZE\n");
        printf("#   CP_MAX_WRITE_TIME_SIZE\n");
409
        printf("#   CP_DEVICE\n");
410
        printf("#   CP_SIZE_AT_OPEN\n");
411
412
        printf("#   CP_F_MAX_READ_TIME\n");
        printf("#   CP_F_MAX_WRITE_TIME\n");
413
414
415
416
417
418
419
420
        printf("#   CP_FASTEST_RANK\n");
        printf("#   CP_FASTEST_RANK_BYTES\n");
        printf("#   CP_SLOWEST_RANK\n");
        printf("#   CP_SLOWEST_RANK_BYTES\n");
        printf("#   CP_F_FASTEST_RANK_TIME\n");
        printf("#   CP_F_SLOWEST_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_TIME\n");
        printf("#   CP_F_VARIANCE_RANK_BYTES\n");
Philip Carns's avatar
Philip Carns committed
421
        printf("# WARNING: version 1.21 log format has the following limitations:\n");
422
        printf("# - *_TIMESTAMP fields are not normalized relative to MPI_Init() time.\n");
Philip Carns's avatar
Philip Carns committed
423
424
425
426
        printf("# - does not record mounted file systems, mount points, or fs types.\n");
        printf("# - attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
        printf("# - does not store the job id in the file.\n");
        printf("# - inaccurate statistics in some multi-threaded cases.\n");
Philip Carns's avatar
Philip Carns committed
427
        printf("# - CP_F_SLOWEST_RANK_TIME and CP_F_FASTEST_RANK_TIME only report elapsed time at the POSIX level.\n");
428
429
430
431
432
433
434
435
        return;
    }

    fprintf(stderr, "Error: version %s not supported by parser.\n",
        job->version_string);
    return;
}

436
437
438
439
/* shift_missing_1_21()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.21
440
 */
441
/*******************************
Philip Carns's avatar
Philip Carns committed
442
 * version 1.21 to 2.00 differences 
443
444
445
446
447
448
449
450
 * - added:
 *   - CP_INDEP_NC_OPENS
 *   - CP_COLL_NC_OPENS
 *   - CP_HDF5_OPENS
 *   - CP_MAX_READ_TIME_SIZE
 *   - CP_MAX_WRITE_TIME_SIZE
 *   - CP_DEVICE
 *   - CP_SIZE_AT_OPEN
451
452
453
454
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
455
456
 *   - CP_F_MAX_READ_TIME
 *   - CP_F_MAX_WRITE_TIME
457
458
459
460
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
461
 * - changed params:
462
463
464
 *   - CP_FILE_RECORD_SIZE: 1184 to 1328
 *   - CP_NUM_INDICES: 133 to 144
 *   - CP_F_NUM_INDICES: 12 to 18
465
 */
466
467
468
469
470
471
472
473
474
static void shift_missing_1_21(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_INDEP_NC_OPENS,
        CP_COLL_NC_OPENS,
        CP_HDF5_OPENS,
        CP_MAX_READ_TIME_SIZE,
        CP_MAX_WRITE_TIME_SIZE,
475
        CP_DEVICE,
476
        CP_SIZE_AT_OPEN,
477
478
479
480
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
481
482
483
484
        -1};
    int missing_f_counters[] = {
        CP_F_MAX_READ_TIME,
        CP_F_MAX_WRITE_TIME,
485
486
487
488
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

    return;
}
525
526
527
528
529
530

/* shift_missing_1_22()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.22
 */
531
/*******************************
Philip Carns's avatar
Philip Carns committed
532
 * version 1.22 to 2.00 differences
533
534
535
536
 *
 * - added:
 *   - CP_DEVICE
 *   - CP_SIZE_AT_OPEN
537
538
539
540
541
542
543
544
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
545
 * - changed params:
546
547
548
 *   - CP_FILE_RECORD_SIZE: 1240 to 1328
 *   - CP_NUM_INDICES: 138 to 144
 *   - CP_F_NUM_INDICES: 14 to 18
549
 */
550
551
552
553
554
static void shift_missing_1_22(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_DEVICE,
555
        CP_SIZE_AT_OPEN,
556
557
558
559
560
561
562
563
564
565
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
        -1};
    int missing_f_counters[] = {
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

600
601
    return;
}
602

603
604
605
606
607
608
609
610
611
/* shift_missing_1_24()
 *
 * translates indices to account for counters that weren't present in log
 * format 1.24
 */
/*******************************
 * version 1.24 to 2.00 differences
 *
 * - added:
612
613
614
615
 *   - CP_FASTEST_RANK
 *   - CP_FASTEST_RANK_BYTES
 *   - CP_SLOWEST_RANK
 *   - CP_SLOWEST_RANK_BYTES
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
 *   - CP_F_FASTEST_RANK_TIME
 *   - CP_F_SLOWEST_RANK_TIME
 *   - CP_F_VARIANCE_RANK_TIME
 *   - CP_F_VARIANCE_RANK_BYTES
 * - changed params:
 *   - CP_FILE_RECORD_SIZE: ? to 1328
 *   - CP_NUM_INDICES: 140 to 144
 *   - CP_F_NUM_INDICES: 14 to 18
 */
static void shift_missing_1_24(struct darshan_file* file)
{
    int c_index = 0;
    int missing_counters[] = {
        CP_FASTEST_RANK,
        CP_FASTEST_RANK_BYTES,
        CP_SLOWEST_RANK,
        CP_SLOWEST_RANK_BYTES,
        -1};
    int missing_f_counters[] = {
        CP_F_FASTEST_RANK_TIME,
        CP_F_SLOWEST_RANK_TIME,
        CP_F_VARIANCE_RANK_TIME,
        CP_F_VARIANCE_RANK_BYTES,
        -1};

    c_index = 0;
    while(missing_counters[c_index] != -1)
    {
        int missing_counter = missing_counters[c_index];
        c_index++;
        if(missing_counter < (CP_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->counters[missing_counter+1],
                &file->counters[missing_counter],
                (CP_NUM_INDICES-missing_counter-1)*sizeof(int64_t));
        }
        /* zero out missing counter */
        file->counters[missing_counter] = 0;
    }

    c_index = 0;
    while(missing_f_counters[c_index] != -1)
    {
        int missing_counter = missing_f_counters[c_index];
        c_index++;
        if(missing_counter < (CP_F_NUM_INDICES - 1))
        {
            /* shift down */
            memmove(&file->fcounters[missing_counter+1],
                &file->fcounters[missing_counter],
                (CP_F_NUM_INDICES-missing_counter-1)*sizeof(double));
        }
        /* zero out missing counter */
        file->fcounters[missing_counter] = 0;
    }

    return;
}

Philip Carns's avatar
Philip Carns committed
676
static int getfile_internal_204(darshan_fd fd, struct darshan_job *job, 
677
678
679
680
681
    struct darshan_file *file)
{
    int ret;
    const char* err_string;
    int i;
682

683
684
685
686
687
688
    if(fd->pos < CP_JOB_RECORD_SIZE)
    {
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE);
        if(ret < 0)
            return(ret);
    }
689
690
691
692
693
694

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
    memset(file, 0, sizeof(&file));

695
    ret = darshan_log_read(fd, file, sizeof(*file));
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
            DARSHAN_BSWAP64(&file->hash);
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

720
    if(ret == 0)
721
722
723
724
725
726
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
727
    err_string = darshan_log_error(fd, &ret);
728
729
730
731
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

Philip Carns's avatar
Philip Carns committed
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;
    const char* err_string;
    int i;

    if(fd->pos < CP_JOB_RECORD_SIZE_200)
    {
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_200);
        if(ret < 0)
            return(ret);
    }

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
    memset(file, 0, sizeof(&file));

    ret = darshan_log_read(fd, file, sizeof(*file));
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
            DARSHAN_BSWAP64(&file->hash);
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

    if(ret == 0)
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
    err_string = darshan_log_error(fd, &ret);
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

788
789
static int getfile_internal_124(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
790
{
791
792
    int ret;

793
    ret = getfile_internal_1x(fd, job, file, 140, 14);
794
795
796
797
798
799
    if(ret <= 0)
        return(ret);

    shift_missing_1_24(file);

    return(1);
800
801
802
803
804
805
806
}

static int getfile_internal_122(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;

807
808
    ret = getfile_internal_1x(fd, job, file, 138, 14);
    if(ret <= 0)
809
810
811
812
        return(ret);

    shift_missing_1_22(file);

813
    return(1);
814
815
816
817
818
819
820
}

static int getfile_internal_121(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file)
{
    int ret;

821
822
    ret = getfile_internal_1x(fd, job, file, 133, 12);
    if(ret <= 0)
823
824
825
826
        return(ret);

    shift_missing_1_21(file);

827
    return(1);
828
829
830
831
}

static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job, 
    struct darshan_file *file, int n_counters, int n_fcounters)
832
{
833
834
835
836
837
838
839
840
841
    char* buffer;
    int ret;
    const char* err_string;
    int i;
    uint64_t hash;
    int32_t rank;
    int64_t* counters;
    double* fcounters;
    char* name_suffix;
842
    int FILE_SIZE_1x = (32 + n_counters*8 + n_fcounters*8);
843

844
845
846
847
848
    memset(file, 0, sizeof(*file));

    /* set file pointer if this is the first file record; otherwise pick up
     * where we left off last time
     */
Philip Carns's avatar
Philip Carns committed
849
    if(fd->pos < CP_JOB_RECORD_SIZE_1x)
850
    {
Philip Carns's avatar
Philip Carns committed
851
        ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_1x);
852
853
854
855
        if(ret < 0)
            return(ret);
    }
    
856
    /* space for file struct, int64 array, and double array */
857
    buffer = (char*)malloc(FILE_SIZE_1x);
858
859
860
861
862
    if(!buffer)
    {
        return(-1);
    }

863
    ret = darshan_log_read(fd, buffer, FILE_SIZE_1x);
864

865
    if(ret > 0 && ret < FILE_SIZE_1x)
866
867
868
869
870
871
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        free(buffer);
        return(-1);
    }
872
    else if(ret == 0)
873
874
875
876
877
878
879
880
    {
        /* hit end of file */
        free(buffer);
        return(0);
    }
    else if(ret <= 0)
    {
        /* all other errors */
881
        err_string = darshan_log_error(fd, &ret);
882
883
884
885
886
887
888
889
890
        fprintf(stderr, "Error: %s\n", err_string);
        free(buffer);
        return(-1);
    }

    /* got exactly one, correct size record */
    hash = *((int64_t*)&buffer[0]);
    rank = *((int32_t*)&buffer[8]);
    counters = ((int64_t*)&buffer[16]);
891
892
    fcounters = ((double*)&buffer[16 + n_counters*8]);
    name_suffix = &buffer[16 + n_counters*8 + n_fcounters*8];
893
894
895
896
897
898
899


    if(fd->swap_flag)
    {
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&hash);
        DARSHAN_BSWAP32(&rank);
900
        for(i=0; i<n_counters; i++)
901
            DARSHAN_BSWAP64(&counters[i]);
902
        for(i=0; i<n_fcounters; i++)
903
904
905
906
907
908
            DARSHAN_BSWAP64(&fcounters[i]);
    }

    /* assign into new format */
    file->hash = hash;
    file->rank += rank;
909
910
    memcpy(file->counters, counters, n_counters*8);
    memcpy(file->fcounters, fcounters, n_fcounters*8);
911
912
913
914
    memcpy(file->name_suffix, name_suffix, 12);

    free(buffer);
    return(1);
915
}
916
#endif
917

918
/* ** SDS ** */
919
920
921
922
923
924
/* return amount written on success, -1 on failure.
 */
static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
    int ret;

925
    if(fd->pf)
926
    {
927
        ret = write(fd->pf, buf, len);
928
929
930
931
932
933
934
935
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

936
/* ** SDS ** */
937
938
939
940
941
942
/* return amount read on success, 0 on EOF, -1 on failure.
 */
static int darshan_log_read(darshan_fd fd, void* buf, int len)
{
    int ret;

943
    if(fd->pf)
944
    {
945
        ret = read(fd->pf, buf, len);
946
947
948
949
950
951
952
953
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

954
#if 0
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
static const char* darshan_log_error(darshan_fd fd, int* errnum)
{
    if(fd->gzf)
    {
        return(gzerror(fd->gzf, errnum));
    }

#ifdef HAVE_LIBBZ2
    if(fd->bzf)
    {
        return(BZ2_bzerror(fd->bzf, errnum));
    }
#endif

    *errnum = 0;
    return(NULL);
}
972
#endif
973

974
/* ** SDS ** */
975
976
/* return 0 on successful seek to offset, -1 on failure.
 */
977
static int darshan_log_seek(darshan_fd fd, off_t offset)
978
{
979
980
981
    off_t ret_off;

    /* TODO: need to look at each use case here -- do I have everything right? */
982
983
984
985

    if(fd->pos == offset)
        return(0);

986
987
    ret_off = lseek(fd->pf, offset, SEEK_SET);
    if(ret_off == offset)
988
    {
989
        fd->pos = offset;
990
991
992
993
994
        return(0);
    }

    return(-1);
}