darshan-logutils.c 41.9 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24 25 26 27 28 29 30 31 32 33
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

/* special identifers for referring to header, job,  and
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
34 35
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
36
static int darshan_log_write(darshan_fd fd, void *buf, int len);
37 38 39 40 41 42 43
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
44 45 46 47 48
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
49 50 51 52
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

/* TODO: check comments on functions to make sure they are right /cleanup */
53 54

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
55
/* see gzip interface for ideas */
56 57
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
58
    NULL,               /* NULL */
59
    &posix_logutils,    /* POSIX */
60 61 62
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
63
    &bgq_logutils,      /* BG/Q */
64 65 66 67 68 69 70 71 72 73 74
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
75 76 77

/* darshan_log_open()
 *
78
 * open an existing darshan log file for reading only
79
 *
80
 * returns file descriptor on success, NULL on failure
81
 */
82
darshan_fd darshan_log_open(const char *name)
83
{
84
    darshan_fd tmp_fd;
85 86
    int o_flags = O_RDONLY;
    int ret;
87

88 89 90 91 92
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

93 94
    /* open the log file in read mode */
    tmp_fd->fildes = open(name, o_flags);
95
    if(tmp_fd->fildes < 0)
96
    {
97 98 99 100 101 102 103 104 105 106 107 108
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
        free(tmp_fd);
        return(NULL);
    }
    tmp_fd->o_flags = o_flags;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
        close(tmp_fd->fildes);
109
        free(tmp_fd);
110 111 112 113 114 115 116 117 118 119 120
        return(NULL);
    }

    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        return(NULL);
121 122
    }

123 124 125 126 127 128 129
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
130
 * returns file descriptor on success, NULL on failure
131 132 133 134
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;
135 136
    int o_flags = O_WRONLY | O_CREAT | O_EXCL;
    int ret;
137 138 139 140 141 142

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

143
    /* open the log for writing, making sure to not overwrite existing log */
144
    /* TODO: permissions when creating?  umask */
145
    tmp_fd->fildes = open(name, o_flags, 0400);
146
    if(tmp_fd->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd);
150
        return(NULL);
151
    }
152 153 154
    tmp_fd->o_flags = o_flags;
    tmp_fd->comp_type = comp_type;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);
155

156 157 158 159 160
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
161
    if(ret < 0)
162 163
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
164 165 166 167
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
168 169
    }

170 171
    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
172 173
    if(ret < 0)
    {
174 175 176 177 178
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
179 180
    }

181
    return(tmp_fd);
182 183
}

184
/* darshan_log_getjob()
185 186
 *
 * read job level metadata from the darshan log file
187
 *
188
 * returns 0 on success, -1 on failure
189
 */
190
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
191
{
192 193
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
194
    int ret;
195

196 197
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

198
    /* read the compressed job data from the log file */
199 200
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
201
    {
202
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
203 204
        return(-1);
    }
205 206

    memcpy(job, job_buf, sizeof(*job));
207

208
    if(fd->swap_flag)
209
    {
210 211 212 213 214 215
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
216 217
    }

218 219 220 221 222 223 224
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

225 226 227
    return(0);
}

228 229
/* darshan_log_putjob()
 *
230
 * write job level metadata to darshan log file
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
    int len;
    int ret;

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

254 255 256
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
257
    {
258
        fd->err = -1;
259 260 261 262 263 264 265 266 267 268 269 270 271
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
272 273 274
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
275
    int ret;
276

277 278
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
279
    {
280 281
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
282

283 284
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
285
    }
286

287 288 289 290
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
291
    if(newline)
292
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
293 294 295 296

    return (0);
}

297 298 299
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
300
 * NOTE: this needs to be called immediately following put_job as it
301
 * expects the file pointer to be positioned immediately following
302
 * the darshan job information
303 304 305 306 307 308 309
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
310 311

    len = strlen(buf);
312

313 314
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
315
    {
316
        fd->err = -1;
317 318 319 320 321 322 323
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

324 325
/* darshan_log_getmounts()
 * 
326
 * retrieves mount table information from the log. Note that mnt_pts and
327
 * fs_types are arrays that will be allocated by the function and must be
328 329 330
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
331
 */
332
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
333 334 335 336
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
337
    int ret;
338

339 340
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
341
    {
342 343
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
344

345 346
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
347
    }
348

349
    /* count entries */
350
    *count = 0;
351
    pos = fd->exe_mnt_data;
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
373
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
374 375
    {
        /* overestimate string lengths */
376
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
377
        assert((*mnt_pts)[array_index]);
378
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
379 380
        assert((*fs_types)[array_index]);

381 382 383
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
384
        {
385
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
386 387 388 389 390 391 392
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

393 394 395
    return(0);
}

396 397 398 399 400 401 402 403 404 405 406 407 408
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
409 410 411
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
412 413 414
    int ret;

    /* write each mount entry to file */
415
    tmp = mnt_dat;
416 417 418
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
419 420 421 422 423 424

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

425 426
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
427
    {
428
        fd->err = -1;
429
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
430 431 432 433 434 435
        return(-1);
    }

    return(0);
}

436
/* darshan_log_gethash()
437
 *
438
 * read the hash of records from the darshan log file
439 440 441
 *
 * returns 0 on success, -1 on failure
 */
442
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
443
{
444
    char *hash_buf;
445
    int hash_buf_sz;
446
    char *buf_ptr;
447
    darshan_record_id *rec_id_ptr;
448
    uint32_t *path_len_ptr, tmp_path_len;
449 450
    char *path_ptr;
    struct darshan_record_ref *ref;
451 452 453
    int read;
    int read_req_sz;
    int buf_remaining = 0;
454

455 456 457
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
458
        return(-1);
459 460
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
461

462
    do
463
    {
464 465
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
466
         */
467 468 469 470
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
471
        {
472 473 474
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
475 476
        }

477 478 479 480 481 482 483 484
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
485
        {
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
507
            {
508 509 510
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
511
            }
512 513 514

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
515
            {
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
537 538
            }

539 540 541
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
542
        }
543

544 545 546 547 548 549 550 551
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
552

553
    free(hash_buf);
554 555 556
    return(0);
}

557 558 559
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
560 561 562
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
563 564 565 566 567 568
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    char *hash_buf;
569
    int hash_buf_sz;
570
    struct darshan_record_ref *ref, *tmp;
571 572 573
    char *buf_ptr;
    int path_len;
    int wrote;
574

575 576
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
577 578 579
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
580
    memset(hash_buf, 0, hash_buf_sz);
581

582
    /* individually serialize each hash record and write to log file */
583 584
    HASH_ITER(hlink, hash, ref, tmp)
    {
585 586
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
587

588
        /* the hash buffer has space to serialize this record
589 590 591
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
592 593 594 595 596 597
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
598

599 600 601 602 603 604 605 606 607 608
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
            fd->err = -1;
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
609 610 611 612 613 614 615 616
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
617 618 619
 * get a chunk of module data from the darshan log file
 *
 * returns 0 on success, -1 on failure
620 621
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
622
    void *buf, int len)
623 624
{
    int ret;
625

626
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
627
    {
628 629
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
630 631
    }

632 633 634 635
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
636 637
    ret = darshan_log_dzread(fd, mod_id, buf, len);
    if(ret < 0)
638
    {
639
        fprintf(stderr,
640
            "Error: failed to read module %s data from darshan log file.\n",
641
            darshan_module_names[mod_id]);
642 643 644
        return(-1);
    }

645
    return(ret);
646 647
}

648 649 650
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
651 652 653 654 655 656
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
657 658 659 660 661 662 663 664
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
    int ret;

Shane Snyder's avatar
Shane Snyder committed
665
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
666
    {
667
        fd->err = -1;
Shane Snyder's avatar
Shane Snyder committed
668 669
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
670 671 672
    }

    /* write the module chunk to the log file */
673 674
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
675
    {
676
        fd->err = -1;
677 678 679 680 681 682 683 684 685
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

686 687
/* darshan_log_close()
 *
688
 * close an open darshan file descriptor, freeing any resources
689 690 691 692
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
693
{
694 695 696 697 698 699 700 701 702 703 704
    int ret;

    if(fd->o_flags & O_WRONLY)
    {
        /* flush the last region of the log to file */
        switch(fd->comp_type)
        {
            case DARSHAN_ZLIB_COMP:
                ret = darshan_log_libz_flush(fd, fd->dz_prev_reg_id);
                if(ret == 0)
                    break;
705 706 707 708 709 710
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
                ret = darshan_log_bzip2_flush(fd, fd->dz_prev_reg_id);
                if(ret == 0)
                    break;
#endif 
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
            default:
                /* if flush fails, remove the output log file */
                fd->err = -1;
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
        if(fd->err != -1)
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
                fd->err = -1;
        }
    }

    close(fd->fildes);

    /* remove output log file if error writing to it */
    if((fd->o_flags & O_WRONLY ) && (fd->err == -1))
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
            fd->logfile_path);
        unlink(fd->logfile_path);
    }
736

737
    darshan_log_dzdestroy(fd);
738 739
    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
740
    free(fd);
741 742

    return;
743 744
}

745
/* **************************************************** */
746

747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

    fd->comp_type = header.comp_type;

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
    header.comp_type = fd->comp_type;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

853 854 855 856
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
857
    off_t ret_off;
858 859 860 861

    if(fd->pos == offset)
        return(0);

862 863
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
864
    {
865 866
        fd->pos = offset;
        return(0);
867 868 869 870 871
    }

    return(-1);
}

872
/* return amount read on success, 0 on EOF, -1 on failure.
873
 */
874
static int darshan_log_read(darshan_fd fd, void* buf, int len)
875 876 877
{
    int ret;

878 879 880 881
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
882

883
    return(ret);
884 885
}

886
/* return amount written on success, -1 on failure.
887
 */
888
static int darshan_log_write(darshan_fd fd, void* buf, int len)
889 890 891
{
    int ret;

892 893 894 895 896 897 898
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

899
static int darshan_log_dzinit(darshan_fd fd)
900 901 902
{
    int ret;

903 904 905 906 907 908 909 910
    /* initialize buffers for staging compressed data to/from log file */
    fd->dz_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(fd->dz_buf == NULL)
        return(-1);

    fd->dz_prev_reg_id = DARSHAN_HEADER_REGION_ID;

    switch(fd->comp_type)
911 912
    {
        case DARSHAN_ZLIB_COMP:
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
                free(fd->dz_buf);
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
            if(fd->o_flags == O_RDONLY)
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
                tmp_zstrm->next_out = fd->dz_buf;
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
                free(fd->dz_buf);
                return(-1);
            }
            fd->dz_strm = tmp_zstrm;
946
            break;
947
        }
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
                free(fd->dz_buf);
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
            tmp_bzstrm->next_in = Z_NULL;

            if(fd->o_flags == O_RDONLY)
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
                tmp_bzstrm->next_out = (char *)fd->dz_buf;
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
                free(fd->dz_buf);
                return(-1);
            }
            fd->dz_strm = tmp_bzstrm;
            break;
        }
#endif
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

static void darshan_log_dzdestroy(darshan_fd fd)
{
    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            if(fd->o_flags == O_RDONLY)
                inflateEnd(fd->dz_strm);
            else
                deflateEnd(fd->dz_strm);
            free(fd->dz_strm);
            break;
1004 1005 1006 1007 1008 1009 1010 1011 1012
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            if(fd->o_flags == O_RDONLY)
                BZ2_bzDecompressEnd(fd->dz_strm);
            else
                BZ2_bzCompressEnd(fd->dz_strm);
            free(fd->dz_strm);
            break;
#endif
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

    free(fd->dz_buf);
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
    int ret;

    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_log_libz_read(fd, region_id, buf, len);
1029
            break;
1030 1031 1032 1033 1034
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_log_bzip2_read(fd, region_id, buf, len);
            break;
#endif
1035
        default:
1036
            fprintf(stderr, "Error: invalid compression type.\n");
1037 1038 1039 1040 1041 1042
            return(-1);
    }

    return(ret);
}

1043
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len)
Shane Snyder's avatar
Shane Snyder committed
1044 1045 1046
{
    int ret;

1047
    switch(fd->comp_type)
Shane Snyder's avatar
Shane Snyder committed
1048 1049
    {
        case DARSHAN_ZLIB_COMP:
1050
            ret = darshan_log_libz_write(fd, region_id, buf, len);
Shane Snyder's avatar
Shane Snyder committed
1051
            break;
1052 1053 1054 1055 1056
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_log_bzip2_write(fd, region_id, buf, len);
            break;
#endif
Shane Snyder's avatar
Shane Snyder committed
1057
        default:
1058
            fprintf(stderr, "Error: invalid compression type.\n");
Shane Snyder's avatar
Shane Snyder committed
1059 1060 1061 1062 1063 1064
            return(-1);
    }

    return(ret);
}

1065
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len)
1066 1067
{
    int ret;
1068 1069 1070 1071
    int total_bytes = 0;
    int tmp_out_bytes;
    struct darshan_log_map map;
    z_stream *z_strmp = (z_stream *)fd->dz_strm;
1072

1073
    assert(z_strmp);
1074

1075 1076
    /* if new log region, we reload buffers and clear eor flag */
    if(region_id != fd->dz_prev_reg_id)
1077
    {
1078 1079 1080
        z_strmp->avail_in = 0;
        fd->dz_eor = 0;
        fd->dz_prev_reg_id = region_id;
1081 1082
    }

1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
    if(region_id == DARSHAN_JOB_REGION_ID)
        map = fd->job_map;