darshan-logutils.c 27.2 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23
/* TODO: for log reads, we need to make sure the header has been read prior */

24 25
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
26
static int darshan_log_write(darshan_fd fd, void *buf, int len);
27 28
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
29 30
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz);
31 32 33 34

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
35
    NULL,               /* NULL */
36
    &posix_logutils,    /* POSIX */
37 38 39
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
40 41 42 43 44 45 46 47 48 49 50 51
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
52 53 54 55 56 57 58

/* darshan_log_open()
 *
 * open a darshan log file for reading/writing
 *
 * returns 0 on success, -1 on failure
 */
59
darshan_fd darshan_log_open(const char *name, const char *mode)
60
{
61
    darshan_fd tmp_fd;
62

63
    /* we only allow "w" or "r" modes, nothing fancy */
64 65 66
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');

67 68 69 70 71
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

72 73
    if(mode[0] == 'r')
    {
74
        tmp_fd->fildes = open(name, O_RDONLY);
75 76 77
    }
    else if (mode[0] == 'w')
    {
78
        /* TODO: permissions when creating?  umask */
79 80 81
        /* when writing, we create the log file making sure not to overwrite
         * an existing log
         */
82
        tmp_fd->fildes = open(name, O_WRONLY | O_CREAT | O_EXCL, 0400);
83 84 85
    }

    if(tmp_fd->fildes < 0)
86
    {
87
        perror("darshan_log_open: ");
88
        free(tmp_fd);
89
        tmp_fd = NULL;
90
    }
91 92

    return(tmp_fd);
93 94
}

95 96 97 98
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
99
 * NOTE: this is the only portion of the darshan log which is uncompressed
100 101 102 103
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
104
{
105
    int i;
106
    int ret;
107

108
    ret = darshan_log_seek(fd, 0);
109
    if(ret < 0)
110 111
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
112
        return(-1);
113
    }
114

115
    /* read uncompressed header from log file */
116
    ret = darshan_log_read(fd, header, sizeof(*header));
117
    if(ret != sizeof(*header))
118
    {
119
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
120 121 122
        return(-1);
    }

123 124
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
125

126
    if(header->magic_nr == DARSHAN_MAGIC_NR)
127
    {
128
        /* no byte swapping needed, this file is in host format already */
129 130 131 132 133 134
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
135
        if(header->magic_nr == DARSHAN_MAGIC_NR)
136 137
        {
            fd->swap_flag = 1;
138 139 140 141

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
142
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
143 144 145 146
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
147 148 149 150 151 152 153
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
154 155
    }

156
    /* save the mapping of data within log file to this file descriptor */
157 158
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
159 160
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
161 162

    return(0);
163
}
164

165 166 167
/* darshan_log_putheader()
 *
 * write a darshan header to log file
168 169
 * NOTE: the header is not passed in, and is instead built using
 * contents of the given file descriptor
170 171 172
 *
 * returns 0 on success, -1 on failure
 */
173
int darshan_log_putheader(darshan_fd fd)
174
{
175
    struct darshan_header header;
176 177 178 179 180 181 182 183 184
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

185 186 187 188 189 190 191 192
    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

193
    /* write header to file */
194 195
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
196 197 198 199 200 201 202 203
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

204
/* darshan_log_getjob()
205 206
 *
 * read job level metadata from the darshan log file
207
 *
208
 * returns 0 on success, -1 on failure
209
 */
210
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
211
{
212 213 214
    char *comp_buf;
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
215
    int ret;
216

217 218 219 220 221 222
    /* allocate buffer to store compressed job info */
    comp_buf = malloc(fd->job_map.len);
    if(!comp_buf)
        return(-1);

    ret = darshan_log_seek(fd, fd->job_map.off);
223
    if(ret < 0)
224
    {
225
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
226
        free(comp_buf);
227
        return(-1);
228 229
    }

230 231 232
    /* read the compressed job data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
    if(ret != fd->job_map.len)
233
    {
234
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
235 236 237 238 239 240 241 242 243 244
        free(comp_buf);
        return(-1);
    }

    /* decompress the job data */
    ret = darshan_decompress_buf(comp_buf, fd->job_map.len, job_buf, &job_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
245 246
        return(-1);
    }
247 248 249
    free(comp_buf);

    memcpy(job, job_buf, sizeof(*job));
250

251
    if(fd->swap_flag)
252
    {
253 254 255 256 257 258
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
259 260
    }

261 262 263 264 265 266 267
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

268 269 270
    return(0);
}

271 272 273 274 275 276 277 278 279
/* darshan_log_putjob()
 *
 * write job level metadat to darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
280 281
    char *comp_buf;
    int comp_buf_sz;
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
    int len;
    int ret;

    ret = darshan_log_seek(fd, sizeof(struct darshan_header));
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

306 307 308 309 310 311 312 313 314 315 316 317 318 319
    comp_buf = malloc(sizeof(*job));
    if(!comp_buf)
        return(-1);
    comp_buf_sz = sizeof(*job);

    /* compress the job data */
    ret = darshan_compress_buf((char*)&job_copy, sizeof(*job), comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
        return(-1);
    }    

320
    /* write job data to file */
321 322
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
323 324 325 326 327 328 329 330 331 332 333 334 335 336
    {
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
337 338 339
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
340
    int ret;
341

342 343
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
344
    {
345 346
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
347

348 349
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
350
    }
351

352 353 354 355
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
356
    if(newline)
357
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
358 359 360 361

    return (0);
}

362 363 364
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
365 366 367
 * NOTE: this needs to be called immediately following put_job as it
 * expects the final pointer to be positioned immediately following
 * the darshan job information
368 369 370 371 372 373 374
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
375 376 377 378
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;

    len = strlen(buf);
379

380 381
    /* compress the input exe string */
    ret = darshan_compress_buf(buf, len, comp_buf, &comp_buf_sz);
382 383
    if(ret < 0)
    {
384
        fprintf(stderr, "Error: unable to compress exe string.\n");
385 386 387
        return(-1);
    }

388 389
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
390 391 392 393 394 395 396 397
    {
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398 399
/* darshan_log_getmounts()
 * 
400
 * retrieves mount table information from the log. Note that mnt_pts and
401
 * fs_types are arrays that will be allocated by the function and must be
402 403 404
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
407 408 409 410
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
411
    int ret;
412

413 414
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
415
    {
416 417
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
418

419 420
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
421
    }
422

423
    /* count entries */
424
    *count = 0;
425
    pos = fd->exe_mnt_data;
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
447
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
448 449
    {
        /* overestimate string lengths */
450
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
451
        assert((*mnt_pts)[array_index]);
452
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
453 454
        assert((*fs_types)[array_index]);

455 456 457
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
458
        {
459
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
460 461 462 463 464 465 466
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

467 468 469
    return(0);
}

470 471 472 473 474 475 476 477 478 479 480 481 482
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
483 484 485 486 487
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;
    char *tmp;
488 489 490
    int ret;

    /* write each mount entry to file */
491
    tmp = mnt_dat;
492 493 494
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
495 496 497 498 499 500 501 502 503 504 505

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

    ret = darshan_compress_buf(mnt_dat, mnt_dat_sz, comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress mount data.\n");
        return(-1);
506 507
    }

508 509
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if (ret != comp_buf_sz)
510
    {
511
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
512 513 514 515 516 517
        return(-1);
    }

    return(0);
}

518
/* darshan_log_gethash()
519
 *
520
 * read the hash of records from the darshan log file
521 522 523
 *
 * returns 0 on success, -1 on failure
 */
524
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
525
{
526
    char *comp_buf;
527
    char *hash_buf;
528
    int hash_buf_sz;
529
    char *buf_ptr;
530 531 532 533
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
534 535
    int ret;

536 537 538 539 540 541 542 543 544 545
    /* allocate buffers to store compressed and decompressed record
     * hash data
     */
    comp_buf = malloc(fd->rec_map.len);
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(!comp_buf || !hash_buf)
        return(-1);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);

546
    ret = darshan_log_seek(fd, fd->rec_map.off);
547
    if(ret < 0)
548 549
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
550
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
551
        free(hash_buf);
552
        return(-1);
553
    }
554

555
    /* read the record hash from the log file */
556
    ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
557
    if(ret != fd->rec_map.len)
558
    {
559
        fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
560
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
561
        free(hash_buf);
562 563 564 565 566 567 568 569 570 571
        return(-1);
    }

    /* decompress the record hash buffer */
    ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
        hash_buf, &hash_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress darshan job data.\n");
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
572
        free(hash_buf);
573 574
        return(-1);
    }
575
    free(comp_buf);
576

577
    buf_ptr = hash_buf;
578
    while(buf_ptr < (hash_buf + hash_buf_sz))
579
    {
580
        /* get pointers for each field of this darshan record */
581
        /* NOTE: darshan record hash serialization method: 
582 583
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
584 585 586 587 588 589 590 591
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        if(fd->swap_flag)
592
        {
593 594 595
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
596 597
        }

598 599 600 601 602 603
        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
            {
Shane Snyder's avatar
Shane Snyder committed
604
                free(hash_buf);
605 606 607 608 609 610
                return(-1);
            }
            ref->rec.name = malloc(*path_len_ptr + 1);
            if(!ref->rec.name)
            {
                free(ref);
Shane Snyder's avatar
Shane Snyder committed
611
                free(hash_buf);
612 613 614 615 616 617 618
                return(-1);
            }

            /* set the fields for this record */
            ref->rec.id = *rec_id_ptr;
            memcpy(ref->rec.name, path_ptr, *path_len_ptr);
            ref->rec.name[*path_len_ptr] = '\0';
619

620 621 622
            /* add this record to the hash */
            HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
        }
623 624
    }

625
    free(hash_buf);
626 627 628
    return(0);
}

629 630 631
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
632 633 634
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
635 636 637 638 639 640 641 642 643 644 645
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    size_t hash_buf_sz;
    char *hash_buf;
    char *hash_buf_off;
    struct darshan_record_ref *ref, *tmp;
    uint32_t name_len;
    size_t record_sz;
646
    char *comp_buf;
Shane Snyder's avatar
Shane Snyder committed
647
    int comp_buf_sz;
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
    int ret;

    /* allocate a buffer to store 2 MiB worth of record data */
    /* NOTE: this buffer may be reallocated if estimate is too small */
    hash_buf_sz = 2 * 1024 * 1024;
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
    {
        return(-1);
    }

    /* serialize the record hash into a buffer for writing */
    hash_buf_off = hash_buf;
    HASH_ITER(hlink, hash, ref, tmp)
    {
        name_len = strlen(ref->rec.name);
        record_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + name_len;
        /* make sure there is room in the buffer for this record */
        if((hash_buf_off + record_sz) > (hash_buf + hash_buf_sz))
        {
            char *tmp_buf;
            size_t old_buf_sz;

            /* if no room, reallocate the hash buffer at twice the current size */
            old_buf_sz = hash_buf_off - hash_buf;
            hash_buf_sz *= 2;
            tmp_buf = malloc(hash_buf_sz);
            if(!tmp_buf)
            {
                free(hash_buf);
                return(-1);
            }

            memcpy(tmp_buf, hash_buf, old_buf_sz);
            free(hash_buf);
            hash_buf = tmp_buf;
            hash_buf_off = hash_buf + old_buf_sz;
        }

        /* now serialize the record into the hash buffer.
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        *((darshan_record_id *)hash_buf_off) = ref->rec.id;
        hash_buf_off += sizeof(darshan_record_id);
        *((uint32_t *)hash_buf_off) = name_len;
        hash_buf_off += sizeof(uint32_t);
        memcpy(hash_buf_off, ref->rec.name, name_len);
        hash_buf_off += name_len;
    }
    hash_buf_sz = hash_buf_off - hash_buf;

Shane Snyder's avatar
Shane Snyder committed
700
    comp_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
701 702
    if(!comp_buf)
        return(-1);
Shane Snyder's avatar
Shane Snyder committed
703
    comp_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
704 705 706 707 708 709

    /* compress the record hash */
    ret = darshan_compress_buf(hash_buf, hash_buf_sz, comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress darshan record hash.\n");
Shane Snyder's avatar
Shane Snyder committed
710 711
        free(comp_buf);
        free(hash_buf);
712 713 714 715 716 717 718
        return(-1);
    }

    /* set the appropriate mapping info for the record hash in the file descriptor */
    fd->rec_map.off = fd->pos;
    fd->rec_map.len = comp_buf_sz;

719
    /* write the record hash to file */
720 721
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
722 723
    {
        fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
Shane Snyder's avatar
Shane Snyder committed
724 725
        free(comp_buf);
        free(hash_buf);
726 727 728
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
729
    free(comp_buf);
730 731 732 733 734 735 736
    free(hash_buf);

    return(0);
}

/* darshan_log_getmod()
 *
737
 * returns 1 on successful read of module data, 0 on no data, -1 on failure
738 739
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
740
    void *mod_buf, int *mod_buf_sz)
741
{
742
    char *comp_buf;
743
    int ret;
744

745
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
746
    {
747 748
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
749 750
    }

751 752 753 754 755
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    comp_buf = malloc(fd->mod_map[mod_id].len);
    if(!comp_buf)
756
        return(-1);
757 758 759 760 761 762 763

    ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        free(comp_buf);
        return(ret);
764 765
    }

766 767 768
    /* read this module's data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
    if(ret != fd->mod_map[mod_id].len)
769
    {
770
        fprintf(stderr,
771
            "Error: failed to read module %s data from darshan log file.\n",
772
            darshan_module_names[mod_id]);
773
        free(comp_buf);
774 775 776
        return(-1);
    }

777 778 779 780 781 782 783 784 785 786 787 788
    /* decompress this module's data */
    ret = darshan_decompress_buf(comp_buf, fd->mod_map[mod_id].len,
        mod_buf, mod_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
        return(-1);
    }
    free(comp_buf);

789
    return(1);
790 791
}

792 793 794
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
795 796 797 798 799 800
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
801 802 803 804 805 806
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
Shane Snyder's avatar
Shane Snyder committed
807 808
    char *comp_buf;
    int comp_buf_sz;
809 810
    int ret;

Shane Snyder's avatar
Shane Snyder committed
811
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
812
    {
Shane Snyder's avatar
Shane Snyder committed
813 814
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
815 816
    }

Shane Snyder's avatar
Shane Snyder committed
817 818 819 820 821 822 823 824
    comp_buf = malloc(mod_buf_sz);
    if(!comp_buf)
        return(-1);
    comp_buf_sz = mod_buf_sz;

    /* compress the module's data */
    ret = darshan_compress_buf(mod_buf, mod_buf_sz, comp_buf, &comp_buf_sz);
    if(ret < 0)
825
    {
Shane Snyder's avatar
Shane Snyder committed
826 827 828
        fprintf(stderr, "Error: unable to compress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
829 830 831
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
832 833 834 835
    /* set the appropriate mapping information for this module */
    fd->mod_map[mod_id].off = fd->pos;
    fd->mod_map[mod_id].len = comp_buf_sz;

836
    /* write the module chunk to the log file */
Shane Snyder's avatar
Shane Snyder committed
837 838
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
839 840 841 842 843 844 845 846 847 848
    {
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

849 850 851 852 853 854 855
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
856
{
857 858 859 860 861
    if(fd->fildes)
        close(fd->fildes);

    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
862

863
    free(fd);
864 865

    return;
866 867
}

868
/* **************************************************** */
869

870 871 872 873
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
874
    off_t ret_off;
875 876 877 878

    if(fd->pos == offset)
        return(0);

879 880
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
881
    {
882 883
        fd->pos = offset;
        return(0);
884 885 886 887 888
    }

    return(-1);
}

889
/* return amount read on success, 0 on EOF, -1 on failure.
890
 */
891
static int darshan_log_read(darshan_fd fd, void* buf, int len)
892 893 894
{
    int ret;

895 896 897 898
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
899

900
    return(ret);
901 902
}

903
/* return amount written on success, -1 on failure.
904
 */
905
static int darshan_log_write(darshan_fd fd, void* buf, int len)
906 907 908
{
    int ret;

909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

/* return 0 on successful decompression, -1 on failure
 */
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz)
{
    int ret;
    int total_out = 0;
    z_stream tmp_stream;

    memset(&tmp_stream, 0, sizeof(tmp_stream));
    tmp_stream.zalloc = Z_NULL;
    tmp_stream.zfree = Z_NULL;
    tmp_stream.opaque = Z_NULL;
    tmp_stream.next_in = (unsigned char*)comp_buf;
    tmp_stream.avail_in = comp_buf_sz;
    tmp_stream.next_out = (unsigned char*)decomp_buf;
    tmp_stream.avail_out = *inout_decomp_buf_sz;

    /* initialize the zlib decompression parameters */
    /* TODO: check these parameters? */
    //ret = inflateInit2(&tmp_stream, 31);
    ret = inflateInit(&tmp_stream);
    if(ret != Z_OK)
939
    {
940
        return(-1);
941 942
    }

943 944 945 946 947
    /* while we have not finished consuming all of the compressed input data */
    while(tmp_stream.avail_in)
    {
        if(tmp_stream.avail_out == 0)
        {
948
            /* We ran out of buffer space for decompression.  In theory,
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
             * we could just alloc more space, but probably just easier
             * to bump up the default size of the output buffer.
             */
            inflateEnd(&tmp_stream);
            return(-1);
        }

        /* decompress data */
        ret = inflate(&tmp_stream, Z_FINISH);
        if(ret != Z_STREAM_END)
        {
            inflateEnd(&tmp_stream);
            return(-1);
        }

        total_out += tmp_stream.total_out;
        if(tmp_stream.avail_in)
            inflateReset(&tmp_stream);
    }
    inflateEnd(&tmp_stream);

    *inout_decomp_buf_sz = total_out;
    return(0);
}

static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz)
{
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
    int ret;
    int total_out = 0;
    z_stream tmp_stream;

    memset(&tmp_stream, 0, sizeof(tmp_stream));
    tmp_stream.zalloc = Z_NULL;
    tmp_stream.zfree = Z_NULL;
    tmp_stream.opaque = Z_NULL;
    tmp_stream.next_in = (unsigned char*)decomp_buf;
    tmp_stream.avail_in = decomp_buf_sz;
    tmp_stream.next_out = (unsigned char*)comp_buf;
    tmp_stream.avail_out = *inout_comp_buf_sz;

    ret = deflateInit(&tmp_stream, Z_DEFAULT_COMPRESSION);
    if(ret != Z_OK)
    {
        return(-1);
    }

    /* while we have not finished consuming all of the uncompressed input data */
    while(tmp_stream.avail_in)
    {
        if(tmp_stream.avail_out == 0)
        {
            /* We ran out of buffer space for compression.  In theory,
             * we could just alloc more space, but probably just easier
             * to bump up the default size of the output buffer.
             */
            deflateEnd(&tmp_stream);
            return(-1);
        }

        /* compress data */
        ret = deflate(&tmp_stream, Z_FINISH);
        if(ret != Z_STREAM_END)
        {
            deflateEnd(&tmp_stream);
            return(-1);
        }

        total_out += tmp_stream.total_out;
        if(tmp_stream.avail_in)
            deflateReset(&tmp_stream);
    }
    deflateEnd(&tmp_stream);
1022

1023
    *inout_comp_buf_sz = total_out;
1024
    return(0);
1025
}
1026 1027 1028 1029 1030 1031 1032 1033 1034

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */