darshan-logutils.c 52.6 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32 33
struct darshan_dz_state
{
34 35 36 37 38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39 40 41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43 44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64 65 66 67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70 71 72
    struct darshan_dz_state dz;
};

73 74 75 76 77 78 79 80 81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83 84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85 86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87 88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90 91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92 93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94 95 96 97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100 101 102 103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104 105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106 107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108 109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111 112 113 114 115 116 117 118
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);


/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
119 120 121

/* darshan_log_open()
 *
122
 * open an existing darshan log file for reading only
123
 *
124
 * returns file descriptor on success, NULL on failure
125
 */
126
darshan_fd darshan_log_open(const char *name)
127
{
128
    darshan_fd tmp_fd;
129
    int ret;
130

131
    /* allocate a darshan file descriptor */
132 133 134 135
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
136 137 138 139 140 141 142
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
143

144
    /* open the log file in read mode */
145 146
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd->state);
150 151 152
        free(tmp_fd);
        return(NULL);
    }
153
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
154 155

    /* read the header from the log file to init fd data structures */
156
    ret = darshan_log_get_header(tmp_fd);
157 158
    if(ret < 0)
    {
159
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
160 161
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
162
        free(tmp_fd);
163 164 165 166
        return(NULL);
    }

    /* initialize compression data structures */
167
    ret = darshan_log_dzinit(tmp_fd);
168 169 170
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
171 172
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
173 174
        free(tmp_fd);
        return(NULL);
175 176
    }

177 178 179 180 181 182 183
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
184
 * returns file descriptor on success, NULL on failure
185
 */
186 187
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
188 189
{
    darshan_fd tmp_fd;
190
    int ret;
191

192
    /* allocate a darshan file descriptor */
193 194 195 196
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
197 198 199 200 201 202 203
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
204
    tmp_fd->comp_type = comp_type;
205

206 207 208
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
209
    {
210
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
211
        free(tmp_fd->state);
212
        free(tmp_fd);
213
        return(NULL);
214
    }
215
    tmp_fd->state->creat_flag = 1;
216
    tmp_fd->partial_flag = partial_flag;
217
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
218

219 220 221 222 223
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
224
    if(ret < 0)
225 226
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
227 228
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
229 230 231
        free(tmp_fd);
        unlink(name);
        return(NULL);
232 233
    }

234
    /* initialize compression data structures */
235
    ret = darshan_log_dzinit(tmp_fd);
236 237
    if(ret < 0)
    {
238
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
239 240
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
241 242 243
        free(tmp_fd);
        unlink(name);
        return(NULL);
244 245
    }

246
    return(tmp_fd);
247 248
}

249
/* darshan_log_get_job()
250 251
 *
 * read job level metadata from the darshan log file
252
 *
253
 * returns 0 on success, -1 on failure
254
 */
255
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
256
{
257
    struct darshan_fd_int_state *state = fd->state;
258 259
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
260
    int ret;
261

262
    assert(state);
263 264
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

265
    /* read the compressed job data from the log file */
266
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
267
    if(ret <= (int)sizeof(*job))
268
    {
269
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
270 271
        return(-1);
    }
272 273

    memcpy(job, job_buf, sizeof(*job));
274

275
    if(fd->swap_flag)
276
    {
277 278 279 280 281 282
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
283 284
    }

285
    /* save trailing exe & mount information, so it can be retrieved later */
286 287 288
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
289
        return(-1);
290
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
291

292 293 294
    return(0);
}

295
/* darshan_log_put_job()
296
 *
297
 * write job level metadata to darshan log file
298 299 300
 *
 * returns 0 on success, -1 on failure
 */
301
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
302
{
303
    struct darshan_fd_int_state *state = fd->state;
304 305 306 307
    struct darshan_job job_copy;
    int len;
    int ret;

308 309
    assert(state);

310 311 312 313 314 315 316 317 318 319 320 321 322 323
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

324 325
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
326
    if(ret != sizeof(*job))
327
    {
328
        state->err = -1;
329 330 331 332 333 334 335
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

336
/* darshan_log_get_exe()
337 338 339 340 341
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
342
int darshan_log_get_exe(darshan_fd fd, char *buf)
343
{
344
    struct darshan_fd_int_state *state = fd->state;
345
    char *newline;
346
    int ret;
347

348 349
    assert(state);

350
    /* if the exe/mount data has not been saved yet, read in the job info */
351
    if(!(state->exe_mnt_data))
352
    {
353
        struct darshan_job job;
354
        ret = darshan_log_get_job(fd, &job);
355

356
        if(ret < 0 || !(state->exe_mnt_data))
357
            return(-1);
358
    }
359

360
    /* exe string is located before the first line break */
361
    newline = strchr(state->exe_mnt_data, '\n');
362 363

    /* copy over the exe string */
364
    if(newline)
365
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
366 367 368 369

    return (0);
}

370
/* darshan_log_put_exe()
371 372
 *
 * wrties the application exe name to darshan log file
373
 * NOTE: this needs to be called immediately following put_job as it
374
 * expects the file pointer to be positioned immediately following
375
 * the darshan job information
376 377 378
 *
 * returns 0 on success, -1 on failure 
 */
379
int darshan_log_put_exe(darshan_fd fd, char *buf)
380
{
381 382
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
383
    int ret;
384

385
    assert(fd->state);
386

387 388
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
389
    {
390
        state->err = -1;
391 392 393 394 395 396 397
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
/* darshan_log_get_mounts()
399
 * 
400 401 402
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
403 404
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
407
    int* count)
408
{
409
    struct darshan_fd_int_state *state = fd->state;
410 411
    char *pos;
    int array_index = 0;
412
    int ret;
413

414 415
    assert(state);

416
    /* if the exe/mount data has not been saved yet, read in the job info */
417
    if(!(state->exe_mnt_data))
418
    {
419
        struct darshan_job job;
420
        ret = darshan_log_get_job(fd, &job);
421

422
        if(ret < 0 || !(state->exe_mnt_data))
423
            return(-1);
424
    }
425

426
    /* count entries */
427
    *count = 0;
428
    pos = state->exe_mnt_data;
429 430 431 432 433 434 435 436 437 438 439 440 441
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
442 443
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
444

445
    /* work through the table and parse each line (except for
446 447
     * first, which holds command line information)
     */
448 449
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
450
    {
451 452
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
453
        if(ret != 2)
454
        {
455
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
456 457 458 459 460
            return(-1);
        }
        array_index++;
    }

461 462
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

463 464 465
    return(0);
}

466
/* darshan_log_put_mounts()
467 468 469 470 471 472 473 474
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
475
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
476
    int count)
477
{
478
    struct darshan_fd_int_state *state = fd->state;
479 480
    int i;
    char line[1024];
481 482 483
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
484 485
    int ret;

486 487
    assert(state);

488
    /* write each mount entry to file */
489
    tmp = mnt_dat;
490 491
    for(i=count-1; i>=0; i--)
    {
492
        sprintf(line, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
493 494 495 496 497 498

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

499 500
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
501
    {
502
        state->err = -1;
503
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
504 505 506 507 508 509
        return(-1);
    }

    return(0);
}

510
/* darshan_log_get_namehash()
511
 *
512 513
 * read the set of name records from the darshan log file and add to the
 * given hash table
514 515 516
 *
 * returns 0 on success, -1 on failure
 */
517
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
518
{
519
    struct darshan_fd_int_state *state = fd->state;
520 521
    char *name_rec_buf;
    int name_rec_buf_sz;
522 523
    int read;
    int read_req_sz;
524 525
    int buf_len = 0;
    int buf_processed;
526

527 528
    assert(state);

529 530
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
531 532 533 534 535
    {
        *hash = NULL;
        return(0);
    }

536 537 538 539
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
540
        return(-1);
541
    memset(name_rec_buf, 0, name_rec_buf_sz);
542

543
    do
544
    {
545
        /* read chunks of the darshan record id -> name mapping from log file,
546
         * constructing a hash table in the process
547
         */
548
        read_req_sz = name_rec_buf_sz - buf_len;
549
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
550
            name_rec_buf + buf_len, read_req_sz);
551
        if(read < 0)
552
        {
553 554
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
555
            return(-1);
556
        }
557
        buf_len += read;
558

559 560
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
561

562
        /* copy any leftover data to beginning of buffer to parse next */
563 564
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
565 566 567 568 569

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
570
    assert(buf_len == 0);
571

572
    free(name_rec_buf);
573 574 575
    return(0);
}

576
/* darshan_log_put_namehash()
577
 *
578
 * writes the hash table of name records to the darshan log file
579 580 581
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
582 583 584
 *
 * returns 0 on success, -1 on failure
 */
585
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
586
{
587
    struct darshan_fd_int_state *state = fd->state;
588 589 590
    struct darshan_name_record_ref *ref, *tmp;
    struct darshan_name_record_ref *name_rec;
    int name_rec_len;
591
    int wrote;
592

593 594
    assert(state);

595
    /* allocate memory for largest possible hash record */
596
    name_rec = malloc(sizeof(darshan_record_id) + PATH_MAX + 1);
597
    if(!name_rec)
598
        return(-1);
599
    memset(name_rec, 0, sizeof(darshan_record_id) + PATH_MAX + 1);
600

601
    /* individually serialize each hash record and write to log file */
602 603
    HASH_ITER(hlink, hash, ref, tmp)
    {
604
        name_rec_len = sizeof(darshan_record_id) + strlen(ref->name_record->name) + 1;
605
        memcpy(name_rec, ref->name_record, name_rec_len);
606

607
        /* write this hash entry to log file */
608 609 610
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
611
        {
612
            state->err = -1;
613 614
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
615 616
            return(-1);
        }
617 618
    }

619
    free(name_rec);
620 621 622
    return(0);
}

623
/* darshan_log_get_mod()
624
 *
625 626
 * get a chunk of module data from the darshan log file
 *
627
 * returns number of bytes read on success, -1 on failure
628
 */
629
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
630
    void *mod_buf, int mod_buf_sz)
631
{
632
    struct darshan_fd_int_state *state = fd->state;
633
    int ret;
634

635 636
    assert(state);

637
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
638
    {
639 640
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
641 642
    }

643 644 645 646
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
647
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
648
    if(ret < 0)
649
    {
650
        fprintf(stderr,
651
            "Error: failed to read module %s data from darshan log file.\n",
652
            darshan_module_names[mod_id]);
653 654 655
        return(-1);
    }

656
    return(ret);
657 658
}

659
/* darshan_log_put_mod()
660 661
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
662 663 664 665 666 667
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
668
 *
669
 * returns number of bytes written on success, -1 on failure
670
 */
671
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
672
    void *mod_buf, int mod_buf_sz, int ver)
673
{
674
    struct darshan_fd_int_state *state = fd->state;
675 676
    int ret;

677 678
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
679
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
680
    {
681
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
682 683
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
684 685 686
    }

    /* write the module chunk to the log file */
687 688
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
689
    {
690
        state->err = -1;
691 692 693 694 695 696
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

697 698 699
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

700 701 702
    return(0);
}

703 704
/* darshan_log_close()
 *
705
 * close an open darshan file descriptor, freeing any resources
706 707 708
 *
 */
void darshan_log_close(darshan_fd fd)
709
{
710
    struct darshan_fd_int_state *state = fd->state;
711 712
    int ret;

713 714 715 716
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
717 718
    {
        /* flush the last region of the log to file */
719
        switch(fd->comp_type)
720 721
        {
            case DARSHAN_ZLIB_COMP:
722
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
723 724
                if(ret == 0)
                    break;
725 726
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
727
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
728 729 730
                if(ret == 0)
                    break;
#endif 
731 732
            default:
                /* if flush fails, remove the output log file */
733
                state->err = -1;
734 735 736 737 738
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
739
        if(state->err != -1)
740
        {
741
            ret = darshan_log_put_header(fd);
742
            if(ret < 0)
743
                state->err = -1;
744 745 746
        }
    }

747
    close(state->fildes);
748 749

    /* remove output log file if error writing to it */
750
    if((state->creat_flag) && (state->err == -1))
751 752
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
753 754
            state->logfile_path);
        unlink(state->logfile_path);
755
    }
756

757
    darshan_log_dzdestroy(fd);
758 759 760
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
761
    free(fd);
762 763

    return;
764 765
}

766 767 768 769 770 771 772 773 774 775 776 777 778
void darshan_log_print_version_warnings(const char *version_string)
{
    if(strcmp(version_string, "3.00") == 0)
    {
        printf("# WARNING: version 3.00 log format has the following limitations:\n");
        printf("# - Partial instrumentation of stdio stream I/O functions not parsable by Darshan versions >= 3.1.0\n");
        printf("#     * Using darshan-logutils versions < 3.1.0, this data can be found in the following POSIX counters:\n");
        printf("#         * POSIX_FOPENS, POSIX_FREADS, POSIX_FWRITES, POSIX_FSEEKS\n");
    }

    return;
}

779 780 781
/********************************************************
 *             internal helper functions                *
 ********************************************************/
782

783 784 785 786 787 788 789 790 791 792 793 794 795
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return