darshan-logutils.c 62.2 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32
33
struct darshan_dz_state
{
34
35
36
37
38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39
40
41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43
44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64
65
66
67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70
71
72
    struct darshan_dz_state dz;
};

73
74
75
76
77
78
79
80
81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83
84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85
86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87
88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90
91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92
93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94
95
96
97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100
101
102
103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104
105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106
107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108
109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111
112
113
114
115

/* filtered namerecs test */
static int darshan_log_get_filtered_namerecs(void *name_rec_buf, int buf_len, int swap_flag, struct darshan_name_record_ref **hash, darshan_record_id *whitelist, int whitelist_count);


116
117
118
119
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);

120
static char *darshan_util_lib_ver = PACKAGE_VERSION;
121
122
123
124

/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
125
126
127

/* darshan_log_open()
 *
128
 * open an existing darshan log file for reading only
129
 *
130
 * returns file descriptor on success, NULL on failure
131
 */
132
darshan_fd darshan_log_open(const char *name)
133
{
134
    darshan_fd tmp_fd;
135
    int ret;
136

137
    /* allocate a darshan file descriptor */
138
139
140
141
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
142
143
144
145
146
147
148
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
149

150
    /* open the log file in read mode */
151
152
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
153
    {
154
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
155
        free(tmp_fd->state);
156
157
158
        free(tmp_fd);
        return(NULL);
    }
159
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
160
161

    /* read the header from the log file to init fd data structures */
162
    ret = darshan_log_get_header(tmp_fd);
163
164
    if(ret < 0)
    {
165
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
166
167
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
168
        free(tmp_fd);
169
170
171
172
        return(NULL);
    }

    /* initialize compression data structures */
173
    ret = darshan_log_dzinit(tmp_fd);
174
175
176
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
177
178
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
179
180
        free(tmp_fd);
        return(NULL);
181
182
    }

183
184
185
186
187
188
189
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
190
 * returns file descriptor on success, NULL on failure
191
 */
192
193
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
194
195
{
    darshan_fd tmp_fd;
196
    int ret;
197

198
    /* allocate a darshan file descriptor */
199
200
201
202
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
203
204
205
206
207
208
209
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
210
    tmp_fd->comp_type = comp_type;
211

212
213
214
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
215
    {
216
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
217
        free(tmp_fd->state);
218
        free(tmp_fd);
219
        return(NULL);
220
    }
221
    tmp_fd->state->creat_flag = 1;
222
    tmp_fd->partial_flag = partial_flag;
223
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
224

225
226
227
228
229
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
230
    if(ret < 0)
231
232
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
233
234
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
235
236
237
        free(tmp_fd);
        unlink(name);
        return(NULL);
238
239
    }

240
    /* initialize compression data structures */
241
    ret = darshan_log_dzinit(tmp_fd);
242
243
    if(ret < 0)
    {
244
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
245
246
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
247
248
249
        free(tmp_fd);
        unlink(name);
        return(NULL);
250
251
    }

252
    return(tmp_fd);
253
254
}

255
/* darshan_log_get_job()
256
257
 *
 * read job level metadata from the darshan log file
258
 *
259
 * returns 0 on success, -1 on failure
260
 */
261
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
262
{
263
    struct darshan_fd_int_state *state = fd->state;
264
265
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
266
    int ret;
267

268
    assert(state);
269
270
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

271
    /* read the compressed job data from the log file */
272
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
273
    if(ret <= (int)sizeof(*job))
274
    {
275
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
276
277
        return(-1);
    }
278
279

    memcpy(job, job_buf, sizeof(*job));
280

281
    if(fd->swap_flag)
282
    {
283
284
285
286
287
288
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
289
290
    }

291
    /* save trailing exe & mount information, so it can be retrieved later */
292
293
294
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
295
        return(-1);
296
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
297

298
299
300
    return(0);
}

301
/* darshan_log_put_job()
302
 *
303
 * write job level metadata to darshan log file
304
305
306
 *
 * returns 0 on success, -1 on failure
 */
307
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
308
{
309
    struct darshan_fd_int_state *state = fd->state;
310
311
312
313
    struct darshan_job job_copy;
    int len;
    int ret;

314
315
    assert(state);

316
317
318
319
320
321
322
323
324
325
326
327
328
329
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

330
331
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
332
    if(ret != sizeof(*job))
333
    {
334
        state->err = -1;
335
336
337
338
339
340
341
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

342
/* darshan_log_get_exe()
343
344
345
346
347
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
348
int darshan_log_get_exe(darshan_fd fd, char *buf)
349
{
350
    struct darshan_fd_int_state *state = fd->state;
351
    char *newline;
352
    int ret;
353

354
355
    assert(state);

356
    /* if the exe/mount data has not been saved yet, read in the job info */
357
    if(!(state->exe_mnt_data))
358
    {
359
        struct darshan_job job;
360
        ret = darshan_log_get_job(fd, &job);
361

362
        if(ret < 0 || !(state->exe_mnt_data))
363
            return(-1);
364
    }
365

366
    /* exe string is located before the first line break */
367
    newline = strchr(state->exe_mnt_data, '\n');
368
369

    /* copy over the exe string */
370
    if(newline)
371
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
372
373
374
375

    return (0);
}

376
/* darshan_log_put_exe()
377
378
 *
 * wrties the application exe name to darshan log file
379
 * NOTE: this needs to be called immediately following put_job as it
380
 * expects the file pointer to be positioned immediately following
381
 * the darshan job information
382
383
384
 *
 * returns 0 on success, -1 on failure 
 */
385
int darshan_log_put_exe(darshan_fd fd, char *buf)
386
{
387
388
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
389
    int ret;
390

391
    assert(fd->state);
392

393
394
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
395
    {
396
        state->err = -1;
397
398
399
400
401
402
403
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

404
/* darshan_log_get_mounts()
405
 * 
406
407
408
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
409
410
 *
 * returns 0 on success, -1 on failure
411
 */
412
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
413
    int* count)
414
{
415
    struct darshan_fd_int_state *state = fd->state;
416
417
    char *pos;
    int array_index = 0;
418
    int ret;
419

420
421
    assert(state);

422
    /* if the exe/mount data has not been saved yet, read in the job info */
423
    if(!(state->exe_mnt_data))
424
    {
425
        struct darshan_job job;
426
        ret = darshan_log_get_job(fd, &job);
427

428
        if(ret < 0 || !(state->exe_mnt_data))
429
            return(-1);
430
    }
431

432
    /* count entries */
433
    *count = 0;
434
    pos = state->exe_mnt_data;
435
436
437
438
439
440
441
442
443
444
445
446
447
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
448
449
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
450

451
    /* work through the table and parse each line (except for
452
453
     * first, which holds command line information)
     */
454
455
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
456
    {
457
458
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
459
        if(ret != 2)
460
        {
461
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
462
463
464
465
466
            return(-1);
        }
        array_index++;
    }

467
468
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

469
470
471
    return(0);
}

472
/* darshan_log_put_mounts()
473
474
475
476
477
478
479
480
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
481
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
482
    int count)
483
{
484
    struct darshan_fd_int_state *state = fd->state;
485
    int i;
486
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
487
    int ret;
488
489
    int left = DARSHAN_EXE_LEN;
    int pos = 0;
490

491
492
    assert(state);

493
494
495
    /* write each mount entry to file */
    for(i=count-1; i>=0; i--)
    {
496
497
498
499
500
501
502
503
504
505
506
        if((strlen(mnt_data_array[i].mnt_type) + strlen(mnt_data_array[i].mnt_path) + 2) < left)
        {
            ret = snprintf(&mnt_dat[pos], left, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
            left -= ret;
            assert(left >= 0);
            pos += ret;
        }
        else
        {
            break;
        }
507
    }
508
    pos++;
509

510
511
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, pos);
    if (ret != pos)
512
    {
513
        state->err = -1;
514
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
515
516
517
518
519
520
        return(-1);
    }

    return(0);
}

521
/* darshan_log_get_namehash()
522
 *
523
524
 * read the set of name records from the darshan log file and add to the
 * given hash table
525
526
527
 *
 * returns 0 on success, -1 on failure
 */
528
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
529
{
530
    struct darshan_fd_int_state *state = fd->state;
531
532
    char *name_rec_buf;
    int name_rec_buf_sz;
533
534
    int read;
    int read_req_sz;
535
536
    int buf_len = 0;
    int buf_processed;
537

538
539
    assert(state);

540
541
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
542
543
544
545
546
    {
        *hash = NULL;
        return(0);
    }

547
548
549
550
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
551
        return(-1);
552
    memset(name_rec_buf, 0, name_rec_buf_sz);
553

554
    do
555
    {
556
        /* read chunks of the darshan record id -> name mapping from log file,
557
         * constructing a hash table in the process
558
         */
559
        read_req_sz = name_rec_buf_sz - buf_len;
560
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
561
            name_rec_buf + buf_len, read_req_sz);
562
        if(read < 0)
563
        {
564
565
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
566
            return(-1);
567
        }
568
        buf_len += read;
569

570
571
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
572

573
        /* copy any leftover data to beginning of buffer to parse next */
574
575
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
576
577
578
579
580

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
581
    assert(buf_len == 0);
582

583
    free(name_rec_buf);
584
585
586
    return(0);
}

587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665



/* darshan_log_get_filtered_namehash()
 *
 * read the set of name records from the darshan log file and add to the
 * given hash table
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_get_filtered_namehash(darshan_fd fd, 
        struct darshan_name_record_ref **hash,
        darshan_record_id *whitelist, int whitelist_count
        )
{
    struct darshan_fd_int_state *state = fd->state;
    char *name_rec_buf;
    int name_rec_buf_sz;
    int read;
    int read_req_sz;
    int buf_len = 0;
    int buf_processed;

    assert(state);

    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
        return(-1);
    memset(name_rec_buf, 0, name_rec_buf_sz);

    do
    {
        /* read chunks of the darshan record id -> name mapping from log file,
         * constructing a hash table in the process
         */
        read_req_sz = name_rec_buf_sz - buf_len;
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec_buf + buf_len, read_req_sz);
        if(read < 0)
        {
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
            return(-1);
        }
        buf_len += read;

        /* extract any name records in the buffer */
        //buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
        //buf_processed = state->get_filtered_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
        buf_processed = darshan_log_get_filtered_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash, whitelist, whitelist_count);

        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_len == 0);

    free(name_rec_buf);
    return(0);
}






666
/* darshan_log_put_namehash()
667
 *
668
 * writes the hash table of name records to the darshan log file
669
670
671
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
672
673
674
 *
 * returns 0 on success, -1 on failure
 */
675
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
676
{
677
    struct darshan_fd_int_state *state = fd->state;
678
    struct darshan_name_record_ref *ref, *tmp;
679
    struct darshan_name_record *name_rec;
680
    int name_rec_len;
681
    int wrote;
682

683
684
    assert(state);

685
    /* allocate memory for largest possible hash record */
686
    name_rec = malloc(sizeof(darshan_record_id) + PATH_MAX + 1);
687
    if(!name_rec)
688
        return(-1);
689
    memset(name_rec, 0, sizeof(darshan_record_id) + PATH_MAX + 1);
690

691
    /* individually serialize each hash record and write to log file */
692
693
    HASH_ITER(hlink, hash, ref, tmp)
    {
694
        name_rec_len = sizeof(darshan_record_id) + strlen(ref->name_record->name) + 1;
695
        memcpy(name_rec, ref->name_record, name_rec_len);
696

697
        /* write this hash entry to log file */
698
699
700
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
701
        {
702
            state->err = -1;
703
704
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
705
706
            return(-1);
        }
707
708
    }

709
    free(name_rec);
710
711
712
    return(0);
}

713
/* darshan_log_get_mod()
714
 *
715
716
 * get a chunk of module data from the darshan log file
 *
717
 * returns number of bytes read on success, -1 on failure
718
 */
719
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
720
    void *mod_buf, int mod_buf_sz)
721
{
722
    struct darshan_fd_int_state *state = fd->state;
723
    int ret;
724

725
726
    assert(state);

727
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
728
    {
729
730
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
731
732
    }

733
734
735
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

736
737
738
739
740
741
742
743
744
745
746
    /* assume module will support backwards compatibility, but we obviously
     * can't provide any sort of "forwards" compatibility
     */
    if(fd->mod_ver[mod_id] > darshan_module_versions[mod_id])
    {
        fprintf(stderr, "Error: invalid %s module log format version "
                "(expected %d, got %d)\n", darshan_module_names[mod_id],
                darshan_module_versions[mod_id], fd->mod_ver[mod_id]);
        return(-1);
    }

747
    /* read this module's data from the log file */
748
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
749
    if(ret < 0)
750
    {
751
        fprintf(stderr,
752
            "Error: failed to read module %s data from darshan log file.\n",
753
            darshan_module_names[mod_id]);
754
755
756
        return(-1);
    }

757
    return(ret);
758
759
}

760
/* darshan_log_put_mod()
761
762
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
763
764
765
766
767
768
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
769
 *
770
 * returns number of bytes written on success, -1 on failure
771
 */
772
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
773
    void *mod_buf, int mod_buf_sz, int ver)
774
{
775
    struct darshan_fd_int_state *state = fd->state;
776
777
    int ret;

778
779
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
780
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
781
    {
782
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
783
784
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
785
786
787
    }

    /* write the module chunk to the log file */
788
789
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
790
    {
791
        state->err = -1;
792
793
794
795
796
797
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

798
799
800
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

801
802
803
    return(0);
}

804
805
/* darshan_log_close()
 *
806
 * close an open darshan file descriptor, freeing any resources
807
808
809
 *
 */
void darshan_log_close(darshan_fd fd)
810
{
811
    struct darshan_fd_int_state *state = fd->state;
812
813
    int ret;

814
815
816
817
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
818
819
    {
        /* flush the last region of the log to file */
820
        switch(fd->comp_type)
821
822
        {
            case DARSHAN_ZLIB_COMP:
823
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
824
825
                if(ret == 0)
                    break;
826
827
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
828
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
829
830
831
                if(ret == 0)
                    break;
#endif 
832
833
            default:
                /* if flush fails, remove the output log file */
834
                state->err = -1;
835
836
837
838
839
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
840
        if(state->err != -1)
841
        {
842
            ret = darshan_log_put_header(fd);
843
            if(ret < 0)
844
                state->err = -1;
845
846
847
        }
    }

848
    close(state->fildes);
849
850

    /* remove output log file if error writing to it */
851
    if((state->creat_flag) && (state->err == -1))
852
853
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
854
855
            state->logfile_path);
        unlink(state->logfile_path);
856
    }
857

858
    darshan_log_dzdestroy(fd);
859
860
861
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
862
    free(fd);
863
864

    return;
865
866
}

867
868
void darshan_log_print_version_warnings(const char *version_string)
{
Shane Snyder's avatar
Shane Snyder committed
869
870
871
872
873
874
875
    if(strcmp(version_string, "3.20") == 0)
    {
        printf("# WARNING: Darshan logs generated by runtime version 3.2.0 likely exhibit some corruption in counter values.\n");
        printf("#          This bug clearly results in bogus values for some COMMON_ACCESS counters (POSIX, MPIIO, H5 modules),\n");
        printf("#          but preceding data may also be corrupted.\n");
    }

876
877
878
    return;
}

879
880
881
882
883
char *darshan_log_get_lib_version(void)
{
    return darshan_util_lib_ver;
}

884
885
886
/********************************************************
 *             internal helper functions                *
 ********************************************************/
887

888
889
890
891
892
893
894
895
896
897
898
899
900
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash)
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
                return(-1);

            ref->name_record = malloc(rec_len);
            if(!ref->name_record)
            {
                free(ref);
                return(-1);
            }

            /* copy the name record over from the hash buffer */
            memcpy(ref->name_record, name_rec, rec_len);

            /* add this record to the hash */
            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
        }

        tmp_p = (char *)name_rec + rec_len;
        name_rec = (struct darshan_name_record *)tmp_p;
        buf_len -= rec_len;
        buf_processed += rec_len;
    }

    return(buf_processed);
}

964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065






/* whitelist_filter
 *
 * A simple filter function, that tests if a provided value is in 
 *
 */
int whitelist_filter(darshan_record_id val, darshan_record_id *whitelist, int whitelist_count){
    int i;
    for(i = 0; i < whitelist_count; i++)
    {
        if (whitelist[i] == val)
        {
            return 1;
        }
    }
    return 0;
}

/* darshan_log_get_filtered_namerecs
 *
 * Buffered reader to to reconstruct name records from logfile
 *
 */
static int darshan_log_get_filtered_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash,
    darshan_record_id *whitelist, int whitelist_count
    )
// JL: would change interface to allow filter callback function instead of whitelist for more flexibility
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);