darshan-logutils.c 29.8 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
24
static int darshan_log_write(darshan_fd fd, void *buf, int len);
25
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
26 27
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type);
28
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
29 30 31 32 33 34 35 36 37 38
    char* comp_buf, int* inout_comp_buf_sz,
    enum darshan_comp_type comp_type);
static int darshan_zlib_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_zlib_comp(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz);
#ifdef HAVE_LIBBZ2
static int darshan_bzip2_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_bzip2_comp(char* decomp_buf, int decomp_buf_sz,
39
    char* comp_buf, int* inout_comp_buf_sz);
40 41
#endif

42 43 44 45

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
46
    NULL,               /* NULL */
47
    &posix_logutils,    /* POSIX */
48 49 50
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
51 52 53 54 55 56 57 58 59 60 61 62
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
63 64 65

/* darshan_log_open()
 *
66
 * open an existing darshan log file for reading only
67 68 69
 *
 * returns 0 on success, -1 on failure
 */
70
darshan_fd darshan_log_open(const char *name)
71
{
72
    darshan_fd tmp_fd;
73

74 75 76 77 78
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

79 80
    tmp_fd->fildes = open(name, O_RDONLY);
    if(tmp_fd->fildes < 0)
81
    {
82 83 84
        perror("darshan_log_open: ");
        free(tmp_fd);
        tmp_fd = NULL;
85 86
    }

87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
 * returns 0 on success, -1 on failure
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

    /* TODO: permissions when creating?  umask */
    /* when writing, we create the log file making sure not to overwrite
     * an existing log
     */
    tmp_fd->comp_type = comp_type;
    tmp_fd->fildes = open(name, O_WRONLY | O_CREAT | O_EXCL, 0400);
111
    if(tmp_fd->fildes < 0)
112
    {
113
        perror("darshan_log_create: ");
114
        free(tmp_fd);
115
        tmp_fd = NULL;
116
    }
117 118

    return(tmp_fd);
119 120
}

121 122 123 124
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
125
 * NOTE: this is the only portion of the darshan log which is uncompressed
126 127 128 129
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
130
{
131
    int i;
132
    int ret;
133

134
    ret = darshan_log_seek(fd, 0);
135
    if(ret < 0)
136 137
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
138
        return(-1);
139
    }
140

141
    /* read uncompressed header from log file */
142
    ret = darshan_log_read(fd, header, sizeof(*header));
143
    if(ret != sizeof(*header))
144
    {
145
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
146 147 148
        return(-1);
    }

149 150
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
151

152
    if(header->magic_nr == DARSHAN_MAGIC_NR)
153
    {
154
        /* no byte swapping needed, this file is in host format already */
155 156 157 158 159 160
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
161
        if(header->magic_nr == DARSHAN_MAGIC_NR)
162 163
        {
            fd->swap_flag = 1;
164 165 166 167

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
168
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
169 170 171 172
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
173 174 175 176 177 178 179
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
180 181
    }

182 183
    fd->comp_type = header->comp_type;

184
    /* save the mapping of data within log file to this file descriptor */
185 186
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
187 188
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
189 190

    return(0);
191
}
192

193 194 195
/* darshan_log_putheader()
 *
 * write a darshan header to log file
196 197
 * NOTE: the header is not passed in, and is instead built using
 * contents of the given file descriptor
198 199 200
 *
 * returns 0 on success, -1 on failure
 */
201
int darshan_log_putheader(darshan_fd fd)
202
{
203
    struct darshan_header header;
204 205 206 207 208 209 210 211 212
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

213 214 215 216 217 218 219 220
    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

221
    /* write header to file */
222 223
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
224 225 226 227 228 229 230 231
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

232
/* darshan_log_getjob()
233 234
 *
 * read job level metadata from the darshan log file
235
 *
236
 * returns 0 on success, -1 on failure
237
 */
238
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
239
{
240 241 242
    char *comp_buf;
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
243
    int ret;
244

245 246
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

247 248 249 250 251 252
    /* allocate buffer to store compressed job info */
    comp_buf = malloc(fd->job_map.len);
    if(!comp_buf)
        return(-1);

    ret = darshan_log_seek(fd, fd->job_map.off);
253
    if(ret < 0)
254
    {
255
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
256
        free(comp_buf);
257
        return(-1);
258 259
    }

260 261 262
    /* read the compressed job data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
    if(ret != fd->job_map.len)
263
    {
264
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
265 266 267 268 269
        free(comp_buf);
        return(-1);
    }

    /* decompress the job data */
270 271
    ret = darshan_decompress_buf(comp_buf, fd->job_map.len,
        job_buf, &job_buf_sz, fd->comp_type);
272 273 274 275
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
276 277
        return(-1);
    }
278 279 280
    free(comp_buf);

    memcpy(job, job_buf, sizeof(*job));
281

282
    if(fd->swap_flag)
283
    {
284 285 286 287 288 289
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
290 291
    }

292 293 294 295 296 297 298
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

299 300 301
    return(0);
}

302 303 304 305 306 307 308 309 310
/* darshan_log_putjob()
 *
 * write job level metadat to darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
311 312
    char *comp_buf;
    int comp_buf_sz;
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
    int len;
    int ret;

    ret = darshan_log_seek(fd, sizeof(struct darshan_header));
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

337 338 339 340 341 342
    comp_buf = malloc(sizeof(*job));
    if(!comp_buf)
        return(-1);
    comp_buf_sz = sizeof(*job);

    /* compress the job data */
343 344
    ret = darshan_compress_buf((char*)&job_copy, sizeof(*job),
        comp_buf, &comp_buf_sz, fd->comp_type);
345 346 347 348 349 350 351
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
        return(-1);
    }    

352
    /* write job data to file */
353 354
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
355 356 357 358 359 360 361 362 363 364 365 366 367 368
    {
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
369 370 371
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
372
    int ret;
373

374 375
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
376
    {
377 378
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
379

380 381
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
382
    }
383

384 385 386 387
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
388
    if(newline)
389
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
390 391 392 393

    return (0);
}

394 395 396
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
397 398 399
 * NOTE: this needs to be called immediately following put_job as it
 * expects the final pointer to be positioned immediately following
 * the darshan job information
400 401 402 403 404 405 406
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
407 408 409 410
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;

    len = strlen(buf);
411

412
    /* compress the input exe string */
413
    ret = darshan_compress_buf(buf, len, comp_buf, &comp_buf_sz, fd->comp_type);
414 415
    if(ret < 0)
    {
416
        fprintf(stderr, "Error: unable to compress exe string.\n");
417 418 419
        return(-1);
    }

420 421
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
422 423 424 425 426 427 428 429
    {
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

430 431
/* darshan_log_getmounts()
 * 
432
 * retrieves mount table information from the log. Note that mnt_pts and
433
 * fs_types are arrays that will be allocated by the function and must be
434 435 436
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
437
 */
438
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
439 440 441 442
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
443
    int ret;
444

445 446
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
447
    {
448 449
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
450

451 452
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
453
    }
454

455
    /* count entries */
456
    *count = 0;
457
    pos = fd->exe_mnt_data;
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
479
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
480 481
    {
        /* overestimate string lengths */
482
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
483
        assert((*mnt_pts)[array_index]);
484
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
485 486
        assert((*fs_types)[array_index]);

487 488 489
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
490
        {
491
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
492 493 494 495 496 497 498
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

499 500 501
    return(0);
}

502 503 504 505 506 507 508 509 510 511 512 513 514
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
515 516 517 518 519
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;
    char *tmp;
520 521 522
    int ret;

    /* write each mount entry to file */
523
    tmp = mnt_dat;
524 525 526
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
527 528 529 530 531 532

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

533
    ret = darshan_compress_buf(mnt_dat, mnt_dat_sz, comp_buf, &comp_buf_sz, fd->comp_type);
534 535 536 537
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress mount data.\n");
        return(-1);
538 539
    }

540 541
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if (ret != comp_buf_sz)
542
    {
543
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
544 545 546 547 548 549
        return(-1);
    }

    return(0);
}

550
/* darshan_log_gethash()
551
 *
552
 * read the hash of records from the darshan log file
553 554 555
 *
 * returns 0 on success, -1 on failure
 */
556
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
557
{
558
    char *comp_buf;
559
    char *hash_buf;
560
    int hash_buf_sz;
561
    char *buf_ptr;
562 563 564 565
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
566 567
    int ret;

568 569 570 571 572 573 574 575 576 577
    /* allocate buffers to store compressed and decompressed record
     * hash data
     */
    comp_buf = malloc(fd->rec_map.len);
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(!comp_buf || !hash_buf)
        return(-1);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);

578
    ret = darshan_log_seek(fd, fd->rec_map.off);
579
    if(ret < 0)
580 581
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
582
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
583
        free(hash_buf);
584
        return(-1);
585
    }
586

587
    /* read the record hash from the log file */
588
    ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
589
    if(ret != fd->rec_map.len)
590
    {
591
        fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
592
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
593
        free(hash_buf);
594 595 596 597 598
        return(-1);
    }

    /* decompress the record hash buffer */
    ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
599
        hash_buf, &hash_buf_sz, fd->comp_type);
600 601 602 603
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress darshan job data.\n");
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
604
        free(hash_buf);
605 606
        return(-1);
    }
607
    free(comp_buf);
608

609
    buf_ptr = hash_buf;
610
    while(buf_ptr < (hash_buf + hash_buf_sz))
611
    {
612
        /* get pointers for each field of this darshan record */
613
        /* NOTE: darshan record hash serialization method: 
614 615
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
616 617 618 619 620 621 622 623
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        if(fd->swap_flag)
624
        {
625 626 627
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
628 629
        }

630 631 632 633 634 635
        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
            {
Shane Snyder's avatar
Shane Snyder committed
636
                free(hash_buf);
637 638 639 640 641 642
                return(-1);
            }
            ref->rec.name = malloc(*path_len_ptr + 1);
            if(!ref->rec.name)
            {
                free(ref);
Shane Snyder's avatar
Shane Snyder committed
643
                free(hash_buf);
644 645 646 647 648 649 650
                return(-1);
            }

            /* set the fields for this record */
            ref->rec.id = *rec_id_ptr;
            memcpy(ref->rec.name, path_ptr, *path_len_ptr);
            ref->rec.name[*path_len_ptr] = '\0';
651

652 653 654
            /* add this record to the hash */
            HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
        }
655 656
    }

657
    free(hash_buf);
658 659 660
    return(0);
}

661 662 663
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
664 665 666
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
667 668 669 670 671 672 673 674 675 676 677
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    size_t hash_buf_sz;
    char *hash_buf;
    char *hash_buf_off;
    struct darshan_record_ref *ref, *tmp;
    uint32_t name_len;
    size_t record_sz;
678
    char *comp_buf;
Shane Snyder's avatar
Shane Snyder committed
679
    int comp_buf_sz;
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
    int ret;

    /* allocate a buffer to store 2 MiB worth of record data */
    /* NOTE: this buffer may be reallocated if estimate is too small */
    hash_buf_sz = 2 * 1024 * 1024;
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
    {
        return(-1);
    }

    /* serialize the record hash into a buffer for writing */
    hash_buf_off = hash_buf;
    HASH_ITER(hlink, hash, ref, tmp)
    {
        name_len = strlen(ref->rec.name);
        record_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + name_len;
        /* make sure there is room in the buffer for this record */
        if((hash_buf_off + record_sz) > (hash_buf + hash_buf_sz))
        {
            char *tmp_buf;
            size_t old_buf_sz;

            /* if no room, reallocate the hash buffer at twice the current size */
            old_buf_sz = hash_buf_off - hash_buf;
            hash_buf_sz *= 2;
            tmp_buf = malloc(hash_buf_sz);
            if(!tmp_buf)
            {
                free(hash_buf);
                return(-1);
            }

            memcpy(tmp_buf, hash_buf, old_buf_sz);
            free(hash_buf);
            hash_buf = tmp_buf;
            hash_buf_off = hash_buf + old_buf_sz;
        }

        /* now serialize the record into the hash buffer.
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        *((darshan_record_id *)hash_buf_off) = ref->rec.id;
        hash_buf_off += sizeof(darshan_record_id);
        *((uint32_t *)hash_buf_off) = name_len;
        hash_buf_off += sizeof(uint32_t);
        memcpy(hash_buf_off, ref->rec.name, name_len);
        hash_buf_off += name_len;
    }
    hash_buf_sz = hash_buf_off - hash_buf;

Shane Snyder's avatar
Shane Snyder committed
732
    comp_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
733 734
    if(!comp_buf)
        return(-1);
Shane Snyder's avatar
Shane Snyder committed
735
    comp_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
736 737

    /* compress the record hash */
738 739
    ret = darshan_compress_buf(hash_buf, hash_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
740 741 742
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress darshan record hash.\n");
Shane Snyder's avatar
Shane Snyder committed
743 744
        free(comp_buf);
        free(hash_buf);
745 746 747 748 749 750 751
        return(-1);
    }

    /* set the appropriate mapping info for the record hash in the file descriptor */
    fd->rec_map.off = fd->pos;
    fd->rec_map.len = comp_buf_sz;

752
    /* write the record hash to file */
753 754
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
755 756
    {
        fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
Shane Snyder's avatar
Shane Snyder committed
757 758
        free(comp_buf);
        free(hash_buf);
759 760 761
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
762
    free(comp_buf);
763 764 765 766 767 768 769
    free(hash_buf);

    return(0);
}

/* darshan_log_getmod()
 *
770
 * returns 1 on successful read of module data, 0 on no data, -1 on failure
771 772
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
773
    void *mod_buf, int *mod_buf_sz)
774
{
775
    char *comp_buf;
776
    int ret;
777

778
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
779
    {
780 781
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
782 783
    }

784 785 786 787 788
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    comp_buf = malloc(fd->mod_map[mod_id].len);
    if(!comp_buf)
789
        return(-1);
790 791 792 793 794 795 796

    ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        free(comp_buf);
        return(ret);
797 798
    }

799 800 801
    /* read this module's data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
    if(ret != fd->mod_map[mod_id].len)
802
    {
803
        fprintf(stderr,
804
            "Error: failed to read module %s data from darshan log file.\n",
805
            darshan_module_names[mod_id]);
806
        free(comp_buf);
807 808 809
        return(-1);
    }

810 811
    /* decompress this module's data */
    ret = darshan_decompress_buf(comp_buf, fd->mod_map[mod_id].len,
812
        mod_buf, mod_buf_sz, fd->comp_type);
813 814 815 816 817 818 819 820 821
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
        return(-1);
    }
    free(comp_buf);

822
    return(1);
823 824
}

825 826 827
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
828 829 830 831 832 833
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
834 835 836 837 838 839
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
Shane Snyder's avatar
Shane Snyder committed
840 841
    char *comp_buf;
    int comp_buf_sz;
842 843
    int ret;

Shane Snyder's avatar
Shane Snyder committed
844
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
845
    {
Shane Snyder's avatar
Shane Snyder committed
846 847
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
848 849
    }

Shane Snyder's avatar
Shane Snyder committed
850 851 852 853 854 855
    comp_buf = malloc(mod_buf_sz);
    if(!comp_buf)
        return(-1);
    comp_buf_sz = mod_buf_sz;

    /* compress the module's data */
856 857
    ret = darshan_compress_buf(mod_buf, mod_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
Shane Snyder's avatar
Shane Snyder committed
858
    if(ret < 0)
859
    {
Shane Snyder's avatar
Shane Snyder committed
860 861 862
        fprintf(stderr, "Error: unable to compress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
863 864 865
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
866 867 868 869
    /* set the appropriate mapping information for this module */
    fd->mod_map[mod_id].off = fd->pos;
    fd->mod_map[mod_id].len = comp_buf_sz;

870
    /* write the module chunk to the log file */
Shane Snyder's avatar
Shane Snyder committed
871 872
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
873 874 875 876 877 878 879 880 881 882
    {
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

883 884 885 886 887 888 889
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
890
{
891 892 893 894 895
    if(fd->fildes)
        close(fd->fildes);

    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
896

897
    free(fd);
898 899

    return;
900 901
}

902
/* **************************************************** */
903

904 905 906 907
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
908
    off_t ret_off;
909 910 911 912

    if(fd->pos == offset)
        return(0);

913 914
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
915
    {
916 917
        fd->pos = offset;
        return(0);
918 919 920 921 922
    }

    return(-1);
}

923
/* return amount read on success, 0 on EOF, -1 on failure.
924
 */
925
static int darshan_log_read(darshan_fd fd, void* buf, int len)
926 927 928
{
    int ret;

929 930 931 932
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
933

934
    return(ret);
935 936
}

937
/* return amount written on success, -1 on failure.
938
 */
939
static int darshan_log_write(darshan_fd fd, void* buf, int len)
940 941 942
{
    int ret;

943 944 945 946 947 948 949 950 951 952
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

/* return 0 on successful decompression, -1 on failure
 */
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type)
{
    int ret;

    switch(comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_zlib_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_bzip2_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#endif
        default:
            fprintf(stderr, "Error: invalid decompression method.\n");
            return(-1);
    }

    return(ret);
}

static int darshan_zlib_decomp(char* comp_buf, int comp_buf_sz,
979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
    char* decomp_buf, int* inout_decomp_buf_sz)
{
    int ret;
    int total_out = 0;
    z_stream tmp_stream;

    memset(&tmp_stream, 0, sizeof(tmp_stream));
    tmp_stream.zalloc = Z_NULL;
    tmp_stream.zfree = Z_NULL;
    tmp_stream.opaque = Z_NULL;
    tmp_stream.next_in = (unsigned char*)comp_buf;
    tmp_stream.avail_in = comp_buf_sz;
    tmp_stream.next_out = (unsigned char*)decomp_buf;
    tmp_stream.avail_out = *inout_decomp_buf_sz;

    /* initialize the zlib decompression parameters */
    /* TODO: check these parameters? */
    //ret = inflateInit2(&tmp_stream, 31);
    ret = inflateInit(&tmp_stream);
    if(ret != Z_OK)
999
    {
1000
        return(-1);