darshan-logutils.c 46.5 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29 30 31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
struct darshan_dz_state
{
    /* (libz/bzip2) stream data structure for managing
     * compression and decompression state */
    void *strm;
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
    int size;
    /* for reading logs, flag indicating end of log file region */
    int eor;
    /* the region we last tried reading/writing */
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* compression type used on log file (libz or bzip2) */
    enum darshan_comp_type comp_type;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

    /* compression/decompression state */
    struct darshan_dz_state dz;
};

69 70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71 72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74 75
static int darshan_log_dzinit(struct darshan_fd_int_state *state);
static void darshan_log_dzdestroy(struct darshan_fd_int_state *state);
76 77 78 79 80
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
81 82 83 84 85
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
86 87 88
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

Shane Snyder's avatar
Shane Snyder committed
89
/* each module's implementation of the darshan logutil functions */
90
#define X(a, b, c, d) d,
91 92
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
93
    DARSHAN_MODULE_IDS
94
};
Shane Snyder's avatar
Shane Snyder committed
95
#undef X
96 97 98

/* darshan_log_open()
 *
99
 * open an existing darshan log file for reading only
100
 *
101
 * returns file descriptor on success, NULL on failure
102
 */
103
darshan_fd darshan_log_open(const char *name)
104
{
105
    darshan_fd tmp_fd;
106
    int ret;
107

108
    /* allocate a darshan file descriptor */
109 110 111 112
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
113 114 115 116 117 118 119
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
120

121
    /* open the log file in read mode */
122 123
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
124
    {
125
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
126
        free(tmp_fd->state);
127 128 129
        free(tmp_fd);
        return(NULL);
    }
130
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
131 132 133 134 135

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
136
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
137 138
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
139
        free(tmp_fd);
140 141 142 143
        return(NULL);
    }

    /* initialize compression data structures */
144
    ret = darshan_log_dzinit(tmp_fd->state);
145 146 147
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
148 149
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
150 151
        free(tmp_fd);
        return(NULL);
152 153
    }

154 155 156 157 158 159 160
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
161
 * returns file descriptor on success, NULL on failure
162
 */
163 164
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
165 166
{
    darshan_fd tmp_fd;
167
    int ret;
168

169
    /* allocate a darshan file descriptor */
170 171 172 173
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
174 175 176 177 178 179 180
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
181

182 183 184
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
185
    {
186
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
187
        free(tmp_fd->state);
188
        free(tmp_fd);
189
        return(NULL);
190
    }
191 192
    tmp_fd->state->creat_flag = 1;
    tmp_fd->state->comp_type = comp_type;
193
    tmp_fd->partial_flag = partial_flag;
194
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
195

196 197 198 199 200
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
201
    if(ret < 0)
202 203
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
204 205
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
206 207 208
        free(tmp_fd);
        unlink(name);
        return(NULL);
209 210
    }

211
    /* initialize compression data structures */
212
    ret = darshan_log_dzinit(tmp_fd->state);
213 214
    if(ret < 0)
    {
215
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
216 217
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
218 219 220
        free(tmp_fd);
        unlink(name);
        return(NULL);
221 222
    }

223
    return(tmp_fd);
224 225
}

226
/* darshan_log_getjob()
227 228
 *
 * read job level metadata from the darshan log file
229
 *
230
 * returns 0 on success, -1 on failure
231
 */
232
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
233
{
234
    struct darshan_fd_int_state *state = fd->state;
235 236
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
237
    int ret;
238

239
    assert(state);
240 241
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

242
    /* read the compressed job data from the log file */
243 244
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
245
    {
246
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
247 248
        return(-1);
    }
249 250

    memcpy(job, job_buf, sizeof(*job));
251

252
    if(fd->swap_flag)
253
    {
254 255 256 257 258 259
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
260 261
    }

262
    /* save trailing exe & mount information, so it can be retrieved later */
263 264 265
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
266
        return(-1);
267
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
268

269 270 271
    return(0);
}

272 273
/* darshan_log_putjob()
 *
274
 * write job level metadata to darshan log file
275 276 277 278 279
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
280
    struct darshan_fd_int_state *state = fd->state;
281 282 283 284
    struct darshan_job job_copy;
    int len;
    int ret;

285 286
    assert(state);

287 288 289 290 291 292 293 294 295 296 297 298 299 300
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

301 302 303
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
304
    {
305
        state->err = -1;
306 307 308 309 310 311 312 313 314 315 316 317 318
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
319 320
int darshan_log_getexe(darshan_fd fd, char *buf)
{
321
    struct darshan_fd_int_state *state = fd->state;
322
    char *newline;
323
    int ret;
324

325 326
    assert(state);

327
    /* if the exe/mount data has not been saved yet, read in the job info */
328
    if(!(state->exe_mnt_data))
329
    {
330 331
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
332

333
        if(ret < 0 || !(state->exe_mnt_data))
334
            return(-1);
335
    }
336

337
    /* exe string is located before the first line break */
338
    newline = strchr(state->exe_mnt_data, '\n');
339 340

    /* copy over the exe string */
341
    if(newline)
342
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
343 344 345 346

    return (0);
}

347 348 349
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
350
 * NOTE: this needs to be called immediately following put_job as it
351
 * expects the file pointer to be positioned immediately following
352
 * the darshan job information
353 354 355 356 357
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
358 359
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
360
    int ret;
361

362
    assert(fd->state);
363

364 365
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
366
    {
367
        state->err = -1;
368 369 370 371 372 373 374
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

375 376
/* darshan_log_getmounts()
 * 
377
 * retrieves mount table information from the log. Note that mnt_pts and
378
 * fs_types are arrays that will be allocated by the function and must be
379 380 381
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
382
 */
383
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
384 385
    char*** fs_types, int* count)
{
386
    struct darshan_fd_int_state *state = fd->state;
387 388
    char *pos;
    int array_index = 0;
389
    int ret;
390

391 392
    assert(state);

393
    /* if the exe/mount data has not been saved yet, read in the job info */
394
    if(!(state->exe_mnt_data))
395
    {
396 397
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
398

399
        if(ret < 0 || !(state->exe_mnt_data))
400
            return(-1);
401
    }
402

403
    /* count entries */
404
    *count = 0;
405
    pos = state->exe_mnt_data;
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
427
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
428 429
    {
        /* overestimate string lengths */
430
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
431
        assert((*mnt_pts)[array_index]);
432
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
433 434
        assert((*fs_types)[array_index]);

435 436 437
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
438
        {
439
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
440 441 442 443 444 445 446
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

447 448 449
    return(0);
}

450 451 452 453 454 455 456 457 458 459 460
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
461
    struct darshan_fd_int_state *state = fd->state;
462 463
    int i;
    char line[1024];
464 465 466
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
467 468
    int ret;

469 470
    assert(state);

471
    /* write each mount entry to file */
472
    tmp = mnt_dat;
473 474 475
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
476 477 478 479 480 481

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

482 483
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
484
    {
485
        state->err = -1;
486
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
487 488 489 490 491 492
        return(-1);
    }

    return(0);
}

493
/* darshan_log_gethash()
494
 *
495
 * read the hash of records from the darshan log file
496 497 498
 *
 * returns 0 on success, -1 on failure
 */
499
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
500
{
501
    struct darshan_fd_int_state *state = fd->state;
502
    char *hash_buf;
503
    int hash_buf_sz;
504
    char *buf_ptr;
505
    darshan_record_id *rec_id_ptr;
506
    uint32_t *path_len_ptr, tmp_path_len;
507 508
    char *path_ptr;
    struct darshan_record_ref *ref;
509 510 511
    int read;
    int read_req_sz;
    int buf_remaining = 0;
512

513 514
    assert(state);

515 516 517 518 519 520 521
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

522 523 524
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
525
        return(-1);
526 527
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
528

529
    do
530
    {
531 532
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
533
         */
534 535 536 537
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
538
        {
539 540 541
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
542 543
        }

544 545 546 547 548 549 550 551
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
552
        {
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
574
            {
575 576 577
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
578
            }
579 580 581

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
582
            {
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
604 605
            }

606 607 608
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
609
        }
610

611 612 613 614 615 616 617 618
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
619

620
    free(hash_buf);
621 622 623
    return(0);
}

624 625 626
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
627 628 629
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
630 631 632 633 634
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
635
    struct darshan_fd_int_state *state = fd->state;
636
    char *hash_buf;
637
    int hash_buf_sz;
638
    struct darshan_record_ref *ref, *tmp;
639 640 641
    char *buf_ptr;
    int path_len;
    int wrote;
642

643 644
    assert(state);

645 646
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
647 648 649
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
650
    memset(hash_buf, 0, hash_buf_sz);
651

652
    /* individually serialize each hash record and write to log file */
653 654
    HASH_ITER(hlink, hash, ref, tmp)
    {
655 656
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
657

658
        /* the hash buffer has space to serialize this record
659 660 661
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
662 663 664 665 666 667
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
668

669 670 671 672 673
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
674
            state->err = -1;
675 676 677 678
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
679 680 681 682 683 684 685 686
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
687 688
 * get a chunk of module data from the darshan log file
 *
689
 * returns number of bytes read on success, -1 on failure
690 691
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
692
    void *mod_buf, int mod_buf_sz)
693
{
694
    struct darshan_fd_int_state *state = fd->state;
695
    int ret;
696

697 698
    assert(state);

699
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
700
    {
701 702
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
703 704
    }

705 706 707 708
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
709
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
710
    if(ret < 0)
711
    {
712
        fprintf(stderr,
713
            "Error: failed to read module %s data from darshan log file.\n",
714
            darshan_module_names[mod_id]);
715 716 717
        return(-1);
    }

718
    return(ret);
719 720
}

721 722 723
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
724 725 726 727 728 729
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
730
 *
731
 * returns number of bytes written on success, -1 on failure
732 733
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
734
    void *mod_buf, int mod_buf_sz, int ver)
735
{
736
    struct darshan_fd_int_state *state = fd->state;
737 738
    int ret;

739 740
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
741
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
742
    {
743
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
744 745
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
746 747 748
    }

    /* write the module chunk to the log file */
749 750
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
751
    {
752
        state->err = -1;
753 754 755 756 757 758
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

759 760 761
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

762 763 764
    return(0);
}

765 766
/* darshan_log_close()
 *
767
 * close an open darshan file descriptor, freeing any resources
768 769 770
 *
 */
void darshan_log_close(darshan_fd fd)
771
{
772
    struct darshan_fd_int_state *state = fd->state;
773 774
    int ret;

775 776 777 778
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
779 780
    {
        /* flush the last region of the log to file */
781
        switch(state->comp_type)
782 783
        {
            case DARSHAN_ZLIB_COMP:
784
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
785 786
                if(ret == 0)
                    break;
787 788
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
789
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
790 791 792
                if(ret == 0)
                    break;
#endif 
793 794
            default:
                /* if flush fails, remove the output log file */
795
                state->err = -1;
796 797 798 799 800
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
801
        if(state->err != -1)
802 803 804
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
805
                state->err = -1;
806 807 808
        }
    }

809
    close(state->fildes);
810 811

    /* remove output log file if error writing to it */
812
    if((state->creat_flag) && (state->err == -1))
813 814
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
815 816
            state->logfile_path);
        unlink(state->logfile_path);
817
    }
818

819 820 821 822
    darshan_log_dzdestroy(state);
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
823
    free(fd);
824 825

    return;
826 827
}

828
/* **************************************************** */
829

830 831 832 833 834 835 836
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
837
    struct darshan_fd_int_state *state = fd->state;
838 839 840 841 842 843 844 845 846 847 848
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
    if(strcmp(fd->version, "3.00"))
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
        fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

911
    /* set some fd fields based on what's stored in the header */
912
    state->comp_type = header.comp_type;
913
    fd->partial_flag = header.partial_flag;
914
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
931
    struct darshan_fd_int_state *state = fd->state;
932 933 934 935 936 937 938 939 940 941 942 943 944
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
945
    header.comp_type = state->comp_type;
946
    header.partial_flag = fd->partial_flag;
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

963 964 965 966
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
967
    struct darshan_fd_int_state *state = fd->state;
968
    off_t ret_off;
969

970
    if(state->pos == offset)
971 972
        return(0);

973
    ret_off = lseek(state->fildes, offset, SEEK_SET);
974
    if(ret_off == offset)
975
    {
976
        state->pos = offset;
977
        return(0);
978 979 980 981 982
    }

    return(-1);
}

983
/* return amount read on success, 0 on EOF, -1 on failure.
984
 */
985
static int darshan_log_read(darshan_fd fd, void* buf, int len)
986
{
987
    struct darshan_fd_int_state *state = fd->state;
988 989
    int ret;

990
    /* read data from the log file using the given map */
991
    ret = read(state->fildes, buf, len);
992
    if(ret > 0)
993
        state->pos += ret;
994

995
    return(ret);
996 997
}

998
/* return amount written on success, -1 on failure.
999
 */
1000
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1001
{
1002
    struct darshan_fd_int_state *state = fd->state;
1003 1004
    int ret;

1005
    ret = write(state->fildes, buf, len);
1006
    if(ret > 0)
1007
        state->pos += ret;
1008 1009 1010 1011

    return(ret);
}

1012
static int darshan_log_dzinit(struct darshan_fd_int_state *state)
1013 1014 1015
{
    int ret;

1016
    /* initialize buffers for staging compressed data to/from log file */
1017 1018
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1019 1020
        return(-1);

1021
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1022

1023
    switch(state->comp_type)
1024 1025
    {
        case DARSHAN_ZLIB_COMP:
1026 1027 1028 1029
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1030
                free(state->dz.buf);
1031 1032 1033 1034 1035 1036 1037 1038 1039
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1040
            if(!(state->creat_flag))
1041 1042 1043 1044 1045 1046 1047 1048 1049
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1050
                tmp_zstrm->next_out = state->dz.buf;
1051 1052 1053 1054
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1055
                free(state->dz.buf);
1056 1057
                return(-1);
            }
1058
            state->dz.strm = tmp_zstrm;
1059
            break;
1060
        }
1061 1062 1063 1064 1065 1066
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1067
                free(state->dz.buf);
1068 1069 1070 1071 1072 1073 1074 1075
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
            tmp_bzstrm->next_in = Z_NULL;

1076
            if(!(state->creat_flag))
1077 1078 1079 1080 1081 1082 1083 1084 1085
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1086
                tmp_bzstrm->next_out = (char *)state->dz.buf;
1087 1088 1089 1090
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
1091
                free(state->dz.buf);
1092 1093
                return(-1);
            }
1094
            state->dz.strm = tmp_bzstrm;
1095 1096 1097
            break;
        }
#endif
1098 1099 1100 1101 1102 1103 1104 1105
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

1106
static void darshan_log_dzdestroy(struct darshan_fd_int_state *state)
1107
{
1108
    switch(state->comp_type)
1109 1110
    {
        case DARSHAN_ZLIB_COMP:
1111 1112
            if(!(state->creat_flag))
                inflateEnd(state->dz.strm);
1113
            else
1114 1115
                deflateEnd(state->dz.strm);
            free(state->dz.strm);
1116
            break;
1117 1118
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1119 1120
            if(!(state->creat_flag))
                BZ2_bzDecompressEnd(state->dz.strm);
<