darshan-logutils.c 52.1 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32
33
struct darshan_dz_state
{
34
35
36
37
38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39
40
41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43
44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64
65
66
67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70
71
72
    struct darshan_dz_state dz;
};

73
74
75
76
77
78
79
80
81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83
84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85
86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87
88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90
91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92
93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94
95
96
97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100
101
102
103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104
105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106
107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108
109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111
112
113
114
115
116
117
118
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);


/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
119
120
121

/* darshan_log_open()
 *
122
 * open an existing darshan log file for reading only
123
 *
124
 * returns file descriptor on success, NULL on failure
125
 */
126
darshan_fd darshan_log_open(const char *name)
127
{
128
    darshan_fd tmp_fd;
129
    int ret;
130

131
    /* allocate a darshan file descriptor */
132
133
134
135
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
136
137
138
139
140
141
142
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
143

144
    /* open the log file in read mode */
145
146
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd->state);
150
151
152
        free(tmp_fd);
        return(NULL);
    }
153
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
154
155

    /* read the header from the log file to init fd data structures */
156
    ret = darshan_log_get_header(tmp_fd);
157
158
    if(ret < 0)
    {
159
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
160
161
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
162
        free(tmp_fd);
163
164
165
166
        return(NULL);
    }

    /* initialize compression data structures */
167
    ret = darshan_log_dzinit(tmp_fd);
168
169
170
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
171
172
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
173
174
        free(tmp_fd);
        return(NULL);
175
176
    }

177
178
179
180
181
182
183
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
184
 * returns file descriptor on success, NULL on failure
185
 */
186
187
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
188
189
{
    darshan_fd tmp_fd;
190
    int ret;
191

192
    /* allocate a darshan file descriptor */
193
194
195
196
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
197
198
199
200
201
202
203
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
204
    tmp_fd->comp_type = comp_type;
205

206
207
208
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
209
    {
210
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
211
        free(tmp_fd->state);
212
        free(tmp_fd);
213
        return(NULL);
214
    }
215
    tmp_fd->state->creat_flag = 1;
216
    tmp_fd->partial_flag = partial_flag;
217
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
218

219
220
221
222
223
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
224
    if(ret < 0)
225
226
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
227
228
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
229
230
231
        free(tmp_fd);
        unlink(name);
        return(NULL);
232
233
    }

234
    /* initialize compression data structures */
235
    ret = darshan_log_dzinit(tmp_fd);
236
237
    if(ret < 0)
    {
238
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
239
240
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
241
242
243
        free(tmp_fd);
        unlink(name);
        return(NULL);
244
245
    }

246
    return(tmp_fd);
247
248
}

249
/* darshan_log_get_job()
250
251
 *
 * read job level metadata from the darshan log file
252
 *
253
 * returns 0 on success, -1 on failure
254
 */
255
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
256
{
257
    struct darshan_fd_int_state *state = fd->state;
258
259
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
260
    int ret;
261

262
    assert(state);
263
264
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

265
    /* read the compressed job data from the log file */
266
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
267
    if(ret <= (int)sizeof(*job))
268
    {
269
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
270
271
        return(-1);
    }
272
273

    memcpy(job, job_buf, sizeof(*job));
274

275
    if(fd->swap_flag)
276
    {
277
278
279
280
281
282
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
283
284
    }

285
    /* save trailing exe & mount information, so it can be retrieved later */
286
287
288
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
289
        return(-1);
290
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
291

292
293
294
    return(0);
}

295
/* darshan_log_put_job()
296
 *
297
 * write job level metadata to darshan log file
298
299
300
 *
 * returns 0 on success, -1 on failure
 */
301
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
302
{
303
    struct darshan_fd_int_state *state = fd->state;
304
305
306
307
    struct darshan_job job_copy;
    int len;
    int ret;

308
309
    assert(state);

310
311
312
313
314
315
316
317
318
319
320
321
322
323
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

324
325
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
326
    if(ret != sizeof(*job))
327
    {
328
        state->err = -1;
329
330
331
332
333
334
335
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

336
/* darshan_log_get_exe()
337
338
339
340
341
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
342
int darshan_log_get_exe(darshan_fd fd, char *buf)
343
{
344
    struct darshan_fd_int_state *state = fd->state;
345
    char *newline;
346
    int ret;
347

348
349
    assert(state);

350
    /* if the exe/mount data has not been saved yet, read in the job info */
351
    if(!(state->exe_mnt_data))
352
    {
353
        struct darshan_job job;
354
        ret = darshan_log_get_job(fd, &job);
355

356
        if(ret < 0 || !(state->exe_mnt_data))
357
            return(-1);
358
    }
359

360
    /* exe string is located before the first line break */
361
    newline = strchr(state->exe_mnt_data, '\n');
362
363

    /* copy over the exe string */
364
    if(newline)
365
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
366
367
368
369

    return (0);
}

370
/* darshan_log_put_exe()
371
372
 *
 * wrties the application exe name to darshan log file
373
 * NOTE: this needs to be called immediately following put_job as it
374
 * expects the file pointer to be positioned immediately following
375
 * the darshan job information
376
377
378
 *
 * returns 0 on success, -1 on failure 
 */
379
int darshan_log_put_exe(darshan_fd fd, char *buf)
380
{
381
382
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
383
    int ret;
384

385
    assert(fd->state);
386

387
388
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
389
    {
390
        state->err = -1;
391
392
393
394
395
396
397
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
/* darshan_log_get_mounts()
399
 * 
400
401
402
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
403
404
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
407
    int* count)
408
{
409
    struct darshan_fd_int_state *state = fd->state;
410
411
    char *pos;
    int array_index = 0;
412
    int ret;
413

414
415
    assert(state);

416
    /* if the exe/mount data has not been saved yet, read in the job info */
417
    if(!(state->exe_mnt_data))
418
    {
419
        struct darshan_job job;
420
        ret = darshan_log_get_job(fd, &job);
421

422
        if(ret < 0 || !(state->exe_mnt_data))
423
            return(-1);
424
    }
425

426
    /* count entries */
427
    *count = 0;
428
    pos = state->exe_mnt_data;
429
430
431
432
433
434
435
436
437
438
439
440
441
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
442
443
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
444

445
    /* work through the table and parse each line (except for
446
447
     * first, which holds command line information)
     */
448
449
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
450
    {
451
452
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
453
        if(ret != 2)
454
        {
455
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
456
457
458
459
460
            return(-1);
        }
        array_index++;
    }

461
462
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

463
464
465
    return(0);
}

466
/* darshan_log_put_mounts()
467
468
469
470
471
472
473
474
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
475
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
476
    int count)
477
{
478
    struct darshan_fd_int_state *state = fd->state;
479
480
    int i;
    char line[1024];
481
482
483
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
484
485
    int ret;

486
487
    assert(state);

488
    /* write each mount entry to file */
489
    tmp = mnt_dat;
490
491
    for(i=count-1; i>=0; i--)
    {
492
        sprintf(line, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
493
494
495
496
497
498

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

499
500
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
501
    {
502
        state->err = -1;
503
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
504
505
506
507
508
509
        return(-1);
    }

    return(0);
}

510
/* darshan_log_get_namehash()
511
 *
512
513
 * read the set of name records from the darshan log file and add to the
 * given hash table
514
515
516
 *
 * returns 0 on success, -1 on failure
 */
517
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
518
{
519
    struct darshan_fd_int_state *state = fd->state;
520
521
    char *name_rec_buf;
    int name_rec_buf_sz;
522
523
    int read;
    int read_req_sz;
524
525
    int buf_len = 0;
    int buf_processed;
526

527
528
    assert(state);

529
530
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
531
532
533
534
535
    {
        *hash = NULL;
        return(0);
    }

536
537
538
539
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
540
        return(-1);
541
    memset(name_rec_buf, 0, name_rec_buf_sz);
542

543
    do
544
    {
545
        /* read chunks of the darshan record id -> name mapping from log file,
546
         * constructing a hash table in the process
547
         */
548
        read_req_sz = name_rec_buf_sz - buf_len;
549
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
550
            name_rec_buf + buf_len, read_req_sz);
551
        if(read < 0)
552
        {
553
554
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
555
            return(-1);
556
        }
557
        buf_len += read;
558

559
560
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
561

562
        /* copy any leftover data to beginning of buffer to parse next */
563
564
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
565
566
567
568
569

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
570
    assert(buf_len == 0);
571

572
    free(name_rec_buf);
573
574
575
    return(0);
}

576
/* darshan_log_put_namehash()
577
 *
578
 * writes the hash table of name records to the darshan log file
579
580
581
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
582
583
584
 *
 * returns 0 on success, -1 on failure
 */
585
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
586
{
587
    struct darshan_fd_int_state *state = fd->state;
588
589
590
    struct darshan_name_record_ref *ref, *tmp;
    struct darshan_name_record_ref *name_rec;
    int name_rec_len;
591
    int wrote;
592

593
594
    assert(state);

595
    /* allocate memory for largest possible hash record */
596
597
    name_rec = malloc(sizeof(struct darshan_name_record) + PATH_MAX);
    if(!name_rec)
598
        return(-1);
599
    memset(name_rec, 0, sizeof(struct darshan_name_record) + PATH_MAX);
600

601
    /* individually serialize each hash record and write to log file */
602
603
    HASH_ITER(hlink, hash, ref, tmp)
    {
604
605
        name_rec_len = sizeof(struct darshan_name_record) + strlen(ref->name_record->name);
        memcpy(name_rec, ref->name_record, name_rec_len);
606

607
        /* write this hash entry to log file */
608
609
610
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
611
        {
612
            state->err = -1;
613
614
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
615
616
            return(-1);
        }
617
618
    }

619
    free(name_rec);
620
621
622
    return(0);
}

623
/* darshan_log_get_mod()
624
 *
625
626
 * get a chunk of module data from the darshan log file
 *
627
 * returns number of bytes read on success, -1 on failure
628
 */
629
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
630
    void *mod_buf, int mod_buf_sz)
631
{
632
    struct darshan_fd_int_state *state = fd->state;
633
    int ret;
634

635
636
    assert(state);

637
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
638
    {
639
640
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
641
642
    }

643
644
645
646
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
647
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
648
    if(ret < 0)
649
    {
650
        fprintf(stderr,
651
            "Error: failed to read module %s data from darshan log file.\n",
652
            darshan_module_names[mod_id]);
653
654
655
        return(-1);
    }

656
    return(ret);
657
658
}

659
/* darshan_log_put_mod()
660
661
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
662
663
664
665
666
667
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
668
 *
669
 * returns number of bytes written on success, -1 on failure
670
 */
671
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
672
    void *mod_buf, int mod_buf_sz, int ver)
673
{
674
    struct darshan_fd_int_state *state = fd->state;
675
676
    int ret;

677
678
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
679
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
680
    {
681
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
682
683
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
684
685
686
    }

    /* write the module chunk to the log file */
687
688
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
689
    {
690
        state->err = -1;
691
692
693
694
695
696
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

697
698
699
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

700
701
702
    return(0);
}

703
704
/* darshan_log_close()
 *
705
 * close an open darshan file descriptor, freeing any resources
706
707
708
 *
 */
void darshan_log_close(darshan_fd fd)
709
{
710
    struct darshan_fd_int_state *state = fd->state;
711
712
    int ret;

713
714
715
716
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
717
718
    {
        /* flush the last region of the log to file */
719
        switch(fd->comp_type)
720
721
        {
            case DARSHAN_ZLIB_COMP:
722
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
723
724
                if(ret == 0)
                    break;
725
726
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
727
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
728
729
730
                if(ret == 0)
                    break;
#endif 
731
732
            default:
                /* if flush fails, remove the output log file */
733
                state->err = -1;
734
735
736
737
738
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
739
        if(state->err != -1)
740
        {
741
            ret = darshan_log_put_header(fd);
742
            if(ret < 0)
743
                state->err = -1;
744
745
746
        }
    }

747
    close(state->fildes);
748
749

    /* remove output log file if error writing to it */
750
    if((state->creat_flag) && (state->err == -1))
751
752
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
753
754
            state->logfile_path);
        unlink(state->logfile_path);
755
    }
756

757
    darshan_log_dzdestroy(fd);
758
759
760
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
761
    free(fd);
762
763

    return;
764
765
}

766
767
768
/********************************************************
 *             internal helper functions                *
 ********************************************************/
769

770
771
772
773
774
775
776
777
778
779
780
781
782
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash)
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
                return(-1);

            ref->name_record = malloc(rec_len);
            if(!ref->name_record)
            {
                free(ref);
                return(-1);
            }

            /* copy the name record over from the hash buffer */
            memcpy(ref->name_record, name_rec, rec_len);

            /* add this record to the hash */
            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
        }

        tmp_p = (char *)name_rec + rec_len;
        name_rec = (struct darshan_name_record *)tmp_p;
        buf_len -= rec_len;
        buf_processed += rec_len;
    }

    return(buf_processed);
}

846
847
848
849
850
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
851
static int darshan_log_get_header(darshan_fd fd)
852
853
854
855
856
857
858
859
860
861
862
863
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

864
865
866
867
868
869
870
871
872
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
873
874
875
876
877
878
879
880
881
    if(strcmp(fd->version, "3.00") == 0)
    {
        fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
    }
    else if(strcmp(fd->version, "3.01") == 0)
    {
        fd->state->get_namerecs = darshan_log_get_namerecs;
    }
    else
882
883
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
884
885
        fprintf(stderr, "Error: expected version %s, but got %s\n",
            DARSHAN_LOG_VERSION, fd->version);
886
887
888
889
890
891
892
893
894
895
896
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

897
898
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
899
    if(ret != (int)sizeof(header))
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
919
920
            DARSHAN_BSWAP64(&(header.name_map.off));
            DARSHAN_BSWAP64(&(header.name_map.len));
921
922
923
924
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
925
                DARSHAN_BSWAP32(&(header.mod_ver[i]));
926
927
928
929
930
931
932
933
934
935
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

936
    /* set some fd fields based on what's stored in the header */
937
    fd->comp_type = header.comp_type;
938
    fd->partial_flag = header.partial_flag;
939
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
940
941

    /* save the mapping of data within log file to this file descriptor */
942
    memcpy(&fd->name_map, &(header.name_map), sizeof(struct darshan_log_map));
943
944
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

945
946
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
947
    if(fd->name_map.off == 0)
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
971
        fd->job_map.len = fd->name_map.off - fd->job_map.off;
972
973
    }

974
975
976
977
978
979
980
    return(0);
}

/* write a darshan header to log file