darshan-logutils.c 52.6 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32 33
struct darshan_dz_state
{
34 35 36 37 38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39 40 41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43 44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64 65 66 67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70 71 72
    struct darshan_dz_state dz;
};

73 74 75 76 77 78 79 80 81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83 84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85 86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87 88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90 91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92 93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94 95 96 97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100 101 102 103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104 105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106 107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108 109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111 112 113 114 115 116 117 118
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);


/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
119 120 121

/* darshan_log_open()
 *
122
 * open an existing darshan log file for reading only
123
 *
124
 * returns file descriptor on success, NULL on failure
125
 */
126
darshan_fd darshan_log_open(const char *name)
127
{
128
    darshan_fd tmp_fd;
129
    int ret;
130

131
    /* allocate a darshan file descriptor */
132 133 134 135
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
136 137 138 139 140 141 142
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
143

144
    /* open the log file in read mode */
145 146
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd->state);
150 151 152
        free(tmp_fd);
        return(NULL);
    }
153
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
154 155

    /* read the header from the log file to init fd data structures */
156
    ret = darshan_log_get_header(tmp_fd);
157 158
    if(ret < 0)
    {
159
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
160 161
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
162
        free(tmp_fd);
163 164 165 166
        return(NULL);
    }

    /* initialize compression data structures */
167
    ret = darshan_log_dzinit(tmp_fd);
168 169 170
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
171 172
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
173 174
        free(tmp_fd);
        return(NULL);
175 176
    }

177 178 179 180 181 182 183
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
184
 * returns file descriptor on success, NULL on failure
185
 */
186 187
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
188 189
{
    darshan_fd tmp_fd;
190
    int ret;
191

192
    /* allocate a darshan file descriptor */
193 194 195 196
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
197 198 199 200 201 202 203
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
204
    tmp_fd->comp_type = comp_type;
205

206 207 208
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
209
    {
210
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
211
        free(tmp_fd->state);
212
        free(tmp_fd);
213
        return(NULL);
214
    }
215
    tmp_fd->state->creat_flag = 1;
216
    tmp_fd->partial_flag = partial_flag;
217
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
218

219 220 221 222 223
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
224
    if(ret < 0)
225 226
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
227 228
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
229 230 231
        free(tmp_fd);
        unlink(name);
        return(NULL);
232 233
    }

234
    /* initialize compression data structures */
235
    ret = darshan_log_dzinit(tmp_fd);
236 237
    if(ret < 0)
    {
238
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
239 240
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
241 242 243
        free(tmp_fd);
        unlink(name);
        return(NULL);
244 245
    }

246
    return(tmp_fd);
247 248
}

249
/* darshan_log_get_job()
250 251
 *
 * read job level metadata from the darshan log file
252
 *
253
 * returns 0 on success, -1 on failure
254
 */
255
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
256
{
257
    struct darshan_fd_int_state *state = fd->state;
258 259
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
260
    int ret;
261

262
    assert(state);
263 264
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

265
    /* read the compressed job data from the log file */
266
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
267
    if(ret <= (int)sizeof(*job))
268
    {
269
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
270 271
        return(-1);
    }
272 273

    memcpy(job, job_buf, sizeof(*job));
274

275
    if(fd->swap_flag)
276
    {
277 278 279 280 281 282
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
283 284
    }

285
    /* save trailing exe & mount information, so it can be retrieved later */
286 287 288
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
289
        return(-1);
290
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
291

292 293 294
    return(0);
}

295
/* darshan_log_put_job()
296
 *
297
 * write job level metadata to darshan log file
298 299 300
 *
 * returns 0 on success, -1 on failure
 */
301
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
302
{
303
    struct darshan_fd_int_state *state = fd->state;
304 305 306 307
    struct darshan_job job_copy;
    int len;
    int ret;

308 309
    assert(state);

310 311 312 313 314 315 316 317 318 319 320 321 322 323
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

324 325
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
326
    if(ret != sizeof(*job))
327
    {
328
        state->err = -1;
329 330 331 332 333 334 335
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

336
/* darshan_log_get_exe()
337 338 339 340 341
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
342
int darshan_log_get_exe(darshan_fd fd, char *buf)
343
{
344
    struct darshan_fd_int_state *state = fd->state;
345
    char *newline;
346
    int ret;
347

348 349
    assert(state);

350
    /* if the exe/mount data has not been saved yet, read in the job info */
351
    if(!(state->exe_mnt_data))
352
    {
353
        struct darshan_job job;
354
        ret = darshan_log_get_job(fd, &job);
355

356
        if(ret < 0 || !(state->exe_mnt_data))
357
            return(-1);
358
    }
359

360
    /* exe string is located before the first line break */
361
    newline = strchr(state->exe_mnt_data, '\n');
362 363

    /* copy over the exe string */
364
    if(newline)
365
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
366 367 368 369

    return (0);
}

370
/* darshan_log_put_exe()
371 372
 *
 * wrties the application exe name to darshan log file
373
 * NOTE: this needs to be called immediately following put_job as it
374
 * expects the file pointer to be positioned immediately following
375
 * the darshan job information
376 377 378
 *
 * returns 0 on success, -1 on failure 
 */
379
int darshan_log_put_exe(darshan_fd fd, char *buf)
380
{
381 382
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
383
    int ret;
384

385
    assert(fd->state);
386

387 388
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
389
    {
390
        state->err = -1;
391 392 393 394 395 396 397
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
/* darshan_log_get_mounts()
399
 * 
400 401 402
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
403 404
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
407
    int* count)
408
{
409
    struct darshan_fd_int_state *state = fd->state;
410 411
    char *pos;
    int array_index = 0;
412
    int ret;
413

414 415
    assert(state);

416
    /* if the exe/mount data has not been saved yet, read in the job info */
417
    if(!(state->exe_mnt_data))
418
    {
419
        struct darshan_job job;
420
        ret = darshan_log_get_job(fd, &job);
421

422
        if(ret < 0 || !(state->exe_mnt_data))
423
            return(-1);
424
    }
425

426
    /* count entries */
427
    *count = 0;
428
    pos = state->exe_mnt_data;
429 430 431 432 433 434 435 436 437 438 439 440 441
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
442 443
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
444

445
    /* work through the table and parse each line (except for
446 447
     * first, which holds command line information)
     */
448 449
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
450
    {
451 452
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
453
        if(ret != 2)
454
        {
455
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
456 457 458 459 460
            return(-1);
        }
        array_index++;
    }

461 462
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

463 464 465
    return(0);
}

466
/* darshan_log_put_mounts()
467 468 469 470 471 472 473 474
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
475
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
476
    int count)
477
{
478
    struct darshan_fd_int_state *state = fd->state;
479 480
    int i;
    char line[1024];
481 482 483
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
484 485
    int ret;

486 487
    assert(state);

488
    /* write each mount entry to file */
489
    tmp = mnt_dat;
490 491
    for(i=count-1; i>=0; i--)
    {
492
        sprintf(line, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
493 494 495 496 497 498

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

499 500
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
501
    {
502
        state->err = -1;
503
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
504 505 506 507 508 509
        return(-1);
    }

    return(0);
}

510
/* darshan_log_get_namehash()
511
 *
512 513
 * read the set of name records from the darshan log file and add to the
 * given hash table
514 515 516
 *
 * returns 0 on success, -1 on failure
 */
517
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
518
{
519
    struct darshan_fd_int_state *state = fd->state;
520 521
    char *name_rec_buf;
    int name_rec_buf_sz;
522 523
    int read;
    int read_req_sz;
524 525
    int buf_len = 0;
    int buf_processed;
526

527 528
    assert(state);

529 530
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
531 532 533 534 535
    {
        *hash = NULL;
        return(0);
    }

536 537 538 539
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
540
        return(-1);
541
    memset(name_rec_buf, 0, name_rec_buf_sz);
542

543
    do
544
    {
545
        /* read chunks of the darshan record id -> name mapping from log file,
546
         * constructing a hash table in the process
547
         */
548
        read_req_sz = name_rec_buf_sz - buf_len;
549
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
550
            name_rec_buf + buf_len, read_req_sz);
551
        if(read < 0)
552
        {
553 554
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
555
            return(-1);
556
        }
557
        buf_len += read;
558

559 560
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
561

562
        /* copy any leftover data to beginning of buffer to parse next */
563 564
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
565 566 567 568 569

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
570
    assert(buf_len == 0);
571

572
    free(name_rec_buf);
573 574 575
    return(0);
}

576
/* darshan_log_put_namehash()
577
 *
578
 * writes the hash table of name records to the darshan log file
579 580 581
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
582 583 584
 *
 * returns 0 on success, -1 on failure
 */
585
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
586
{
587
    struct darshan_fd_int_state *state = fd->state;
588 589 590
    struct darshan_name_record_ref *ref, *tmp;
    struct darshan_name_record_ref *name_rec;
    int name_rec_len;
591
    int wrote;
592

593 594
    assert(state);

595
    /* allocate memory for largest possible hash record */
596
    name_rec = malloc(sizeof(darshan_record_id) + PATH_MAX + 1);
597
    if(!name_rec)
598
        return(-1);
599
    memset(name_rec, 0, sizeof(darshan_record_id) + PATH_MAX + 1);
600

601
    /* individually serialize each hash record and write to log file */
602 603
    HASH_ITER(hlink, hash, ref, tmp)
    {
604
        name_rec_len = sizeof(darshan_record_id) + strlen(ref->name_record->name) + 1;
605
        memcpy(name_rec, ref->name_record, name_rec_len);
606

607
        /* write this hash entry to log file */
608 609 610
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
611
        {
612
            state->err = -1;
613 614
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
615 616
            return(-1);
        }
617 618
    }

619
    free(name_rec);
620 621 622
    return(0);
}

623
/* darshan_log_get_mod()
624
 *
625 626
 * get a chunk of module data from the darshan log file
 *
627
 * returns number of bytes read on success, -1 on failure
628
 */
629
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
630
    void *mod_buf, int mod_buf_sz)
631
{
632
    struct darshan_fd_int_state *state = fd->state;
633
    int ret;
634

635 636
    assert(state);

637
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
638
    {
639 640
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
641 642
    }

643 644 645 646
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
647
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
648
    if(ret < 0)
649
    {
650
        fprintf(stderr,
651
            "Error: failed to read module %s data from darshan log file.\n",
652
            darshan_module_names[mod_id]);
653 654 655
        return(-1);
    }

656
    return(ret);
657 658
}

659
/* darshan_log_put_mod()
660 661
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
662 663 664 665 666 667
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
668
 *
669
 * returns number of bytes written on success, -1 on failure
670
 */
671
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
672
    void *mod_buf, int mod_buf_sz, int ver)
673
{
674
    struct darshan_fd_int_state *state = fd->state;
675 676
    int ret;

677 678
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
679
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
680
    {
681
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
682 683
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
684 685 686
    }

    /* write the module chunk to the log file */
687 688
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
689
    {
690
        state->err = -1;
691 692 693 694 695 696
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

697 698 699
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

700 701 702
    return(0);
}

703 704
/* darshan_log_close()
 *
705
 * close an open darshan file descriptor, freeing any resources
706 707 708
 *
 */
void darshan_log_close(darshan_fd fd)
709
{
710
    struct darshan_fd_int_state *state = fd->state;
711 712
    int ret;

713 714 715 716
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
717 718
    {
        /* flush the last region of the log to file */
719
        switch(fd->comp_type)
720 721
        {
            case DARSHAN_ZLIB_COMP:
722
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
723 724
                if(ret == 0)
                    break;
725 726
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
727
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
728 729 730
                if(ret == 0)
                    break;
#endif 
731 732
            default:
                /* if flush fails, remove the output log file */
733
                state->err = -1;
734 735 736 737 738
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
739
        if(state->err != -1)
740
        {
741
            ret = darshan_log_put_header(fd);
742
            if(ret < 0)
743
                state->err = -1;
744 745 746
        }
    }

747
    close(state->fildes);
748 749

    /* remove output log file if error writing to it */
750
    if((state->creat_flag) && (state->err == -1))
751 752
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
753 754
            state->logfile_path);
        unlink(state->logfile_path);
755
    }
756

757
    darshan_log_dzdestroy(fd);
758 759 760
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
761
    free(fd);
762 763

    return;
764 765
}

766 767 768 769 770 771 772 773 774 775 776 777 778
void darshan_log_print_version_warnings(const char *version_string)
{
    if(strcmp(version_string, "3.00") == 0)
    {
        printf("# WARNING: version 3.00 log format has the following limitations:\n");
        printf("# - Partial instrumentation of stdio stream I/O functions not parsable by Darshan versions >= 3.1.0\n");
        printf("#     * Using darshan-logutils versions < 3.1.0, this data can be found in the following POSIX counters:\n");
        printf("#         * POSIX_FOPENS, POSIX_FREADS, POSIX_FWRITES, POSIX_FSEEKS\n");
    }

    return;
}

779 780 781
/********************************************************
 *             internal helper functions                *
 ********************************************************/
782

783 784 785 786 787 788 789 790 791 792 793 794 795
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash)
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
                return(-1);

            ref->name_record = malloc(rec_len);
            if(!ref->name_record)
            {
                free(ref);
                return(-1);
            }

            /* copy the name record over from the hash buffer */
            memcpy(ref->name_record, name_rec, rec_len);

            /* add this record to the hash */
            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
        }

        tmp_p = (char *)name_rec + rec_len;
        name_rec = (struct darshan_name_record *)tmp_p;
        buf_len -= rec_len;
        buf_processed += rec_len;
    }

    return(buf_processed);
}

859 860 861 862 863
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
864
static int darshan_log_get_header(darshan_fd fd)
865 866 867 868 869 870 871 872 873 874 875 876
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

877 878 879 880 881 882 883 884 885
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
886 887 888 889
    if(strcmp(fd->version, "3.00") == 0)
    {
        fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
    }
890
    else if(strcmp(fd->version, "3.10") == 0)
891 892 893 894
    {
        fd->state->get_namerecs = darshan_log_get_namerecs;
    }
    else
895 896
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
897 898
        fprintf(stderr, "Error: expected version %s, but got %s\n",
            DARSHAN_LOG_VERSION, fd->version);
899 900 901 902 903 904 905 906 907 908 909
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

910 911
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
912
    if(ret != (int)sizeof(header))
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
932 933
            DARSHAN_BSWAP64(&(header.name_map.off));
            DARSHAN_BSWAP64(&(header.name_map.len));
934 935 936 937
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
938
                DARSHAN_BSWAP32(&(header.mod_ver[i]));
939 940 941 942 943 944 945 946 947 948
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

949
    /* set some fd fields based on what's stored in the header */
950
    fd->comp_type = header.comp_type;
951
    fd->partial_flag = header.partial_flag;
952
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
953 954

    /* save the mapping of data within log file to this file descriptor */
955
    memcpy(&fd->name_map, &(header.name_map), sizeof(struct darshan_log_map));
956 957
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

958 959
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
960
    if(fd->name_map.off == 0)
961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
984
        fd->job_map.len = fd->name_map.off - fd->job_map.off;
985 986
    }

987 988 989 990 991 992 993
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
994
static int darshan_log_put_header(darshan_fd fd)
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
1009
    header.comp_type = fd->comp_type;
1010
    header.partial_flag = fd->partial_flag;
1011
    memcpy(&header.name_map, &fd->name_map, sizeof(struct darshan_log_map));
1012 1013
    memcpy(header.mod_map, fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
    memcpy(header.mod_ver, fd->mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
1014 1015 1016

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
1017
    if(ret != (int)sizeof(header))
1018 1019 1020 1021 1022 1023 1024 1025
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

1026 1027 1028 1029
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
1030
    struct darshan_fd_int_state *state = fd->state;
1031
    off_t ret_off;
1032

1033
    if(state->pos == offset)
1034 1035
        return(0);

1036
    ret_off = lseek(state->fildes, offset, SEEK_SET);
1037
    if(ret_off == offset)
1038
    {
1039
        state->pos = offset;
1040
        return(0);
1041 1042 1043 1044 1045
    }

    return(-1);
}

1046
/* return amount read on success, 0 on EOF, -1 on failure.
1047
 */
1048
static int darshan_log_read(darshan_fd fd, void* buf, int len)
1049
{
1050
    struct darshan_fd_int_state *state = fd->state;
1051
    int ret;
1052
    unsigned int read_so_far = 0;
1053

1054 1055 1056 1057 1058 1059 1060 1061 1062
    do
    {
        ret = read(state->fildes, buf + read_so_far, len - read_so_far);
        if(ret <= 0)
            break;
        read_so_far += ret;
    } while(read_so_far < len);
    if(ret < 0)
        return(-1);
1063

1064 1065
    state->pos += read_so_far;
    return(read_so_far);
1066 1067
}

1068
/* return amount written on success, -1 on failure.
1069
 */
1070
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1071
{
1072
    struct darshan_fd_int_state *state = fd->state;
1073
    int ret;
1074
    unsigned int wrote_so_far = 0;
1075

1076 1077 1078 1079 1080 1081 1082 1083 1084
    do
    {
        ret = write(state->fildes, buf + wrote_so_far, len - wrote_so_far);
        if(ret <= 0)
            break;
        wrote_so_far += ret;
    } while(wrote_so_far < len);
    if(ret < 0)
        return(-1);
1085

1086 1087
    state->pos += wrote_so_far;
    return(wrote_so_far);
1088 1089
}

1090
static int darshan_log_dzinit(darshan_fd fd)
1091
{
1092
    struct darshan_fd_int_state *state = fd->state;
1093 1094
    int ret;

1095 1096 1097
    /* initialize buffers for staging compressed data
     * to/from log file
     */
1098 1099
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1100
        return(-1);
1101
    state->dz.size = 0;
1102
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1103

1104
    switch(fd->comp_type)
1105 1106
    {
        case DARSHAN_ZLIB_COMP:
1107 1108 1109 1110
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1111
                free(state->dz.buf);
1112 1113 1114 1115 1116 1117 1118 1119 1120
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1121
            if(!(state->creat_flag))
1122 1123 1124 1125 1126 1127 1128 1129 1130
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1131
                tmp_zstrm->next_out = state->dz.buf;
1132 1133 1134 1135
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1136
                free(state->dz.buf);
1137 1138
                return(-1);
            }
1139
            state->dz.comp_dat = tmp_zstrm;
1140
            break;
1141
        }
1142 1143 1144 1145 1146 1147
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1148
                free(state->dz.buf);
1149 1150 1151 1152 1153 1154
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
1155
            tmp_bzstrm->next_in = NULL;
1156

1157
            if(!(state->creat_flag))
1158 1159 1160 1161 1162 1163 1164 1165 1166
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1167
                tmp_bzstrm->next_out = (char *)state->dz.buf;
1168 1169 1170 1171
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
1172
                free(state->dz.buf);
1173 1174
                return(-1);
            }
1175
            state->dz.comp_dat = tmp_bzstrm;
1176 1177 1178
            break;
        }
#endif
1179 1180 1181 1182 1183 1184 1185 1186
        case DARSHAN_NO_COMP:
        {
            /* we just track an offset into the staging buffers for no_comp */
            int *buf_off = malloc(sizeof(int));
            *buf_off = 0;
            state->dz.comp_dat = buf_off;
            break;
        }
1187 1188 1189 1190 1191 1192 1193 1194
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

1195
static void darshan_log_dzdestroy(darshan_fd fd)
1196
{
1197 1198 1199
    struct darshan_fd_int_state *state = fd->state;

    switch(fd->comp_type)
1200 1201
    {
        case DARSHAN_ZLIB_COMP:
1202
            if(!(state->creat_flag))
1203
                inflateEnd((z_stream *)state->dz.comp_dat);
1204
            else
1205
                deflateEnd((z_stream *)state->dz.comp_dat);
1206
            break;
1207 1208
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1209
            if(!(state->creat_flag))
1210
                BZ2_bzDecompressEnd((bz_stream *)state->dz.comp_dat);
1211
            else
1212
                BZ2_bzCompressEnd((bz_stream *)state->dz.comp_dat);
1213 1214
            break;
#endif
1215 1216 1217
        case DARSHAN_NO_COMP:
            /* do nothing */
            break;
1218 1219 1220 1221
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

1222
    free(state->dz.comp_dat);
1223
    free(state->dz.buf);
1224 1225 1226 1227 1228
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
1229
    struct darshan_fd_int_state *state = fd->state;
1230 1231
    struct darshan_log_map map;
    int reset_strm_flag = 0;
1232 1233
    int ret;