darshan-logutils.c 42 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24 25 26 27 28 29 30 31 32 33
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

/* special identifers for referring to header, job,  and
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
34 35
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
36
static int darshan_log_write(darshan_fd fd, void *buf, int len);
37 38 39 40 41 42 43
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
44 45 46 47 48
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
49 50 51 52
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

/* TODO: check comments on functions to make sure they are right /cleanup */
53 54

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
55
/* see gzip interface for ideas */
56 57
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
58
    NULL,               /* NULL */
59
    &posix_logutils,    /* POSIX */
60 61 62
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
63
    &bgq_logutils,      /* BG/Q */
64 65 66 67 68 69 70 71 72 73 74
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
75 76 77

/* darshan_log_open()
 *
78
 * open an existing darshan log file for reading only
79
 *
80
 * returns file descriptor on success, NULL on failure
81
 */
82
darshan_fd darshan_log_open(const char *name)
83
{
84
    darshan_fd tmp_fd;
85 86
    int o_flags = O_RDONLY;
    int ret;
87

88 89 90 91 92
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

93 94
    /* open the log file in read mode */
    tmp_fd->fildes = open(name, o_flags);
95
    if(tmp_fd->fildes < 0)
96
    {
97 98 99 100 101 102 103 104 105 106 107 108
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
        free(tmp_fd);
        return(NULL);
    }
    tmp_fd->o_flags = o_flags;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
        close(tmp_fd->fildes);
109
        free(tmp_fd);
110 111 112 113 114 115 116 117 118 119 120
        return(NULL);
    }

    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        return(NULL);
121 122
    }

123 124 125 126 127 128 129
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
130
 * returns file descriptor on success, NULL on failure
131 132 133 134
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;
135 136
    int o_flags = O_WRONLY | O_CREAT | O_EXCL;
    int ret;
137 138 139 140 141 142

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

143
    /* open the log for writing, making sure to not overwrite existing log */
144
    /* TODO: permissions when creating?  umask */
145
    tmp_fd->fildes = open(name, o_flags, 0400);
146
    if(tmp_fd->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd);
150
        return(NULL);
151
    }
152 153 154
    tmp_fd->o_flags = o_flags;
    tmp_fd->comp_type = comp_type;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);
155

156 157 158 159 160
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
161
    if(ret < 0)
162 163
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
164 165 166 167
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
168 169
    }

170 171
    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
172 173
    if(ret < 0)
    {
174 175 176 177 178
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
179 180
    }

181
    return(tmp_fd);
182 183
}

184
/* darshan_log_getjob()
185 186
 *
 * read job level metadata from the darshan log file
187
 *
188
 * returns 0 on success, -1 on failure
189
 */
190
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
191
{
192 193
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
194
    int ret;
195

196 197
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

198
    /* read the compressed job data from the log file */
199 200
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
201
    {
202
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
203 204
        return(-1);
    }
205 206

    memcpy(job, job_buf, sizeof(*job));
207

208
    if(fd->swap_flag)
209
    {
210 211 212 213 214 215
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
216 217
    }

218 219 220 221 222 223 224
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

225 226 227
    return(0);
}

228 229
/* darshan_log_putjob()
 *
230
 * write job level metadata to darshan log file
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
    int len;
    int ret;

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

254 255 256
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
257
    {
258
        fd->err = -1;
259 260 261 262 263 264 265 266 267 268 269 270 271
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
272 273 274
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
275
    int ret;
276

277 278
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
279
    {
280 281
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
282

283 284
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
285
    }
286

287 288 289 290
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
291
    if(newline)
292
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
293 294 295 296

    return (0);
}

297 298 299
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
300
 * NOTE: this needs to be called immediately following put_job as it
301
 * expects the file pointer to be positioned immediately following
302
 * the darshan job information
303 304 305 306 307 308 309
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
310 311

    len = strlen(buf);
312

313 314
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
315
    {
316
        fd->err = -1;
317 318 319 320 321 322 323
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

324 325
/* darshan_log_getmounts()
 * 
326
 * retrieves mount table information from the log. Note that mnt_pts and
327
 * fs_types are arrays that will be allocated by the function and must be
328 329 330
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
331
 */
332
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
333 334 335 336
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
337
    int ret;
338

339 340
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
341
    {
342 343
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
344

345 346
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
347
    }
348

349
    /* count entries */
350
    *count = 0;
351
    pos = fd->exe_mnt_data;
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
373
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
374 375
    {
        /* overestimate string lengths */
376
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
377
        assert((*mnt_pts)[array_index]);
378
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
379 380
        assert((*fs_types)[array_index]);

381 382 383
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
384
        {
385
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
386 387 388 389 390 391 392
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

393 394 395
    return(0);
}

396 397 398 399 400 401 402 403 404 405 406 407 408
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
409 410 411
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
412 413 414
    int ret;

    /* write each mount entry to file */
415
    tmp = mnt_dat;
416 417 418
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
419 420 421 422 423 424

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

425 426
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
427
    {
428
        fd->err = -1;
429
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
430 431 432 433 434 435
        return(-1);
    }

    return(0);
}

436
/* darshan_log_gethash()
437
 *
438
 * read the hash of records from the darshan log file
439 440 441
 *
 * returns 0 on success, -1 on failure
 */
442
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
443
{
444
    char *hash_buf;
445
    int hash_buf_sz;
446
    char *buf_ptr;
447
    darshan_record_id *rec_id_ptr;
448
    uint32_t *path_len_ptr, tmp_path_len;
449 450
    char *path_ptr;
    struct darshan_record_ref *ref;
451 452 453
    int read;
    int read_req_sz;
    int buf_remaining = 0;
454

455 456 457 458 459 460 461
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

462 463 464
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
465
        return(-1);
466 467
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
468

469
    do
470
    {
471 472
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
473
         */
474 475 476 477
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
478
        {
479 480 481
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
482 483
        }

484 485 486 487 488 489 490 491
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
492
        {
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
514
            {
515 516 517
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
518
            }
519 520 521

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
522
            {
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
544 545
            }

546 547 548
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
549
        }
550

551 552 553 554 555 556 557 558
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
559

560
    free(hash_buf);
561 562 563
    return(0);
}

564 565 566
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
567 568 569
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
570 571 572 573 574 575
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    char *hash_buf;
576
    int hash_buf_sz;
577
    struct darshan_record_ref *ref, *tmp;
578 579 580
    char *buf_ptr;
    int path_len;
    int wrote;
581

582 583
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
584 585 586
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
587
    memset(hash_buf, 0, hash_buf_sz);
588

589
    /* individually serialize each hash record and write to log file */
590 591
    HASH_ITER(hlink, hash, ref, tmp)
    {
592 593
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
594

595
        /* the hash buffer has space to serialize this record
596 597 598
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
599 600 601 602 603 604
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
605

606 607 608 609 610 611 612 613 614 615
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
            fd->err = -1;
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
616 617 618 619 620 621 622 623
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
624 625 626
 * get a chunk of module data from the darshan log file
 *
 * returns 0 on success, -1 on failure
627 628
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
629
    void *buf, int len)
630 631
{
    int ret;
632

633
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
634
    {
635 636
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
637 638
    }

639 640 641 642
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
643 644
    ret = darshan_log_dzread(fd, mod_id, buf, len);
    if(ret < 0)
645
    {
646
        fprintf(stderr,
647
            "Error: failed to read module %s data from darshan log file.\n",
648
            darshan_module_names[mod_id]);
649 650 651
        return(-1);
    }

652
    return(ret);
653 654
}

655 656 657
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
658 659 660 661 662 663
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
664 665 666 667 668 669 670 671
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
    int ret;

Shane Snyder's avatar
Shane Snyder committed
672
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
673
    {
674
        fd->err = -1;
Shane Snyder's avatar
Shane Snyder committed
675 676
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
677 678 679
    }

    /* write the module chunk to the log file */
680 681
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
682
    {
683
        fd->err = -1;
684 685 686 687 688 689 690 691 692
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

693 694
/* darshan_log_close()
 *
695
 * close an open darshan file descriptor, freeing any resources
696 697 698 699
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
700
{
701 702 703 704 705 706 707 708 709 710 711
    int ret;

    if(fd->o_flags & O_WRONLY)
    {
        /* flush the last region of the log to file */
        switch(fd->comp_type)
        {
            case DARSHAN_ZLIB_COMP:
                ret = darshan_log_libz_flush(fd, fd->dz_prev_reg_id);
                if(ret == 0)
                    break;
712 713 714 715 716 717
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
                ret = darshan_log_bzip2_flush(fd, fd->dz_prev_reg_id);
                if(ret == 0)
                    break;
#endif 
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
            default:
                /* if flush fails, remove the output log file */
                fd->err = -1;
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
        if(fd->err != -1)
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
                fd->err = -1;
        }
    }

    close(fd->fildes);

    /* remove output log file if error writing to it */
    if((fd->o_flags & O_WRONLY ) && (fd->err == -1))
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
            fd->logfile_path);
        unlink(fd->logfile_path);
    }
743

744
    darshan_log_dzdestroy(fd);
745 746
    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
747
    free(fd);
748 749

    return;
750 751
}

752
/* **************************************************** */
753

754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

    fd->comp_type = header.comp_type;

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
    header.comp_type = fd->comp_type;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

860 861 862 863
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
864
    off_t ret_off;
865 866 867 868

    if(fd->pos == offset)
        return(0);

869 870
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
871
    {
872 873
        fd->pos = offset;
        return(0);
874 875 876 877 878
    }

    return(-1);
}

879
/* return amount read on success, 0 on EOF, -1 on failure.
880
 */
881
static int darshan_log_read(darshan_fd fd, void* buf, int len)
882 883 884
{
    int ret;

885 886 887 888
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
889

890
    return(ret);
891 892
}

893
/* return amount written on success, -1 on failure.
894
 */
895
static int darshan_log_write(darshan_fd fd, void* buf, int len)
896 897 898
{
    int ret;

899 900 901 902 903 904 905
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

906
static int darshan_log_dzinit(darshan_fd fd)
907 908 909
{
    int ret;

910 911 912 913 914 915 916 917
    /* initialize buffers for staging compressed data to/from log file */
    fd->dz_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(fd->dz_buf == NULL)
        return(-1);

    fd->dz_prev_reg_id = DARSHAN_HEADER_REGION_ID;

    switch(fd->comp_type)
918 919
    {
        case DARSHAN_ZLIB_COMP:
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
                free(fd->dz_buf);
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
            if(fd->o_flags == O_RDONLY)
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
                tmp_zstrm->next_out = fd->dz_buf;
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
                free(fd->dz_buf);
                return(-1);
            }
            fd->dz_strm = tmp_zstrm;
953
            break;
954
        }
955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
                free(fd->dz_buf);
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
            tmp_bzstrm->next_in = Z_NULL;

            if(fd->o_flags == O_RDONLY)
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
                tmp_bzstrm->next_out = (char *)fd->dz_buf;
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
                free(fd->dz_buf);
                return(-1);
            }
            fd->dz_strm = tmp_bzstrm;
            break;
        }
#endif
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

static void darshan_log_dzdestroy(darshan_fd fd)
{
    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            if(fd->o_flags == O_RDONLY)
                inflateEnd(fd->dz_strm);
            else
                deflateEnd(fd->dz_strm);
            free(fd->dz_strm);
            break;
1011 1012 1013 1014 1015 1016 1017 1018 1019
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            if(fd->o_flags == O_RDONLY)
                BZ2_bzDecompressEnd(fd->dz_strm);
            else
                BZ2_bzCompressEnd(fd->dz_strm);
            free(fd->dz_strm);
            break;
#endif
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

    free(fd->dz_buf);
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
    int ret;

    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_log_libz_read(fd, region_id, buf, len);
1036
            break;
1037 1038 1039 1040 1041
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_log_bzip2_read(fd, region_id, buf, len);
            break;
#endif
1042
        default:
1043
            fprintf(stderr, "Error: invalid compression type.\n");
1044 1045 1046 1047 1048 1049
            return(-1);
    }

    return(ret);
}

1050
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len)
Shane Snyder's avatar
Shane Snyder committed
1051 1052 1053
{
    int ret;

1054
    switch(fd->comp_type)
Shane Snyder's avatar
Shane Snyder committed
1055 1056
    {
        case DARSHAN_ZLIB_COMP:
1057
            ret = darshan_log_libz_write(fd, region_id, buf, len);
Shane Snyder's avatar
Shane Snyder committed
1058
            break;
1059 1060 1061 1062 1063
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_log_bzip2_write(fd, region_id, buf, len);
            break;
#endif
Shane Snyder's avatar
Shane Snyder committed
1064
        default:
1065
            fprintf(stderr, "Error: invalid compression type.\n");
Shane Snyder's avatar
Shane Snyder committed
1066 1067 1068 1069 1070 1071
            return(-1);
    }

    return(ret);
}

1072
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len)
1073 1074
{
    int ret;
1075 1076 1077 1078
    int total_bytes = 0;
    int tmp_out_bytes;
    struct darshan_log_map map;
    z_stream *z_strmp = (z_stream *)fd->dz_strm;
1079

1080
    assert(z_strmp);
1081

1082 1083
    /* if new log region, we reload buffers and clear eor flag */
    if(region_id != fd->dz_prev_reg_id)
1084
    {
1085 1086 1087
        z_strmp->avail_in = 0;
        fd->dz_eor = 0;
        fd->dz_prev_reg_id = region_id;
1088 1089
    }