darshan-logutils.c 37.1 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24 25 26 27 28 29 30 31 32 33
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

/* special identifers for referring to header, job,  and
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
34 35
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
36
static int darshan_log_write(darshan_fd fd, void *buf, int len);
37 38 39 40 41 42 43 44 45 46 47
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

/* TODO: check comments on functions to make sure they are right /cleanup */
48 49

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
50
/* see gzip interface for ideas */
51 52
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
53
    NULL,               /* NULL */
54
    &posix_logutils,    /* POSIX */
55 56 57
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
58
    &bgq_logutils,      /* BG/Q */
59 60 61 62 63 64 65 66 67 68 69
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
70 71 72

/* darshan_log_open()
 *
73
 * open an existing darshan log file for reading only
74
 *
75
 * returns file descriptor on success, NULL on failure
76
 */
77
darshan_fd darshan_log_open(const char *name)
78
{
79
    darshan_fd tmp_fd;
80 81
    int o_flags = O_RDONLY;
    int ret;
82

83 84 85 86 87
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

88 89
    /* open the log file in read mode */
    tmp_fd->fildes = open(name, o_flags);
90
    if(tmp_fd->fildes < 0)
91
    {
92 93 94 95 96 97 98 99 100 101 102 103
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
        free(tmp_fd);
        return(NULL);
    }
    tmp_fd->o_flags = o_flags;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
        close(tmp_fd->fildes);
104
        free(tmp_fd);
105 106 107 108 109 110 111 112 113 114 115
        return(NULL);
    }

    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        return(NULL);
116 117
    }

118 119 120 121 122 123 124
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
125
 * returns file descriptor on success, NULL on failure
126 127 128 129
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;
130 131
    int o_flags = O_WRONLY | O_CREAT | O_EXCL;
    int ret;
132 133 134 135 136 137

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

138
    /* open the log for writing, making sure to not overwrite existing log */
139
    /* TODO: permissions when creating?  umask */
140
    tmp_fd->fildes = open(name, o_flags, 0400);
141
    if(tmp_fd->fildes < 0)
142
    {
143
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
144
        free(tmp_fd);
145
        return(NULL);
146
    }
147 148 149
    tmp_fd->o_flags = o_flags;
    tmp_fd->comp_type = comp_type;
    strncpy(tmp_fd->logfile_path, name, PATH_MAX);
150

151 152 153 154 155
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
156
    if(ret < 0)
157 158
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
159 160 161 162
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
163 164
    }

165 166
    /* initialize compression data structures */
    ret = darshan_log_dzinit(tmp_fd);
167 168
    if(ret < 0)
    {
169 170 171 172 173
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
        close(tmp_fd->fildes);
        free(tmp_fd);
        unlink(name);
        return(NULL);
174 175
    }

176
    return(tmp_fd);
177 178
}

179
/* darshan_log_getjob()
180 181
 *
 * read job level metadata from the darshan log file
182
 *
183
 * returns 0 on success, -1 on failure
184
 */
185
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
186
{
187 188
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
189
    int ret;
190

191 192
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

193
    /* read the compressed job data from the log file */
194 195
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
196
    {
197
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
198 199
        return(-1);
    }
200 201

    memcpy(job, job_buf, sizeof(*job));
202

203
    if(fd->swap_flag)
204
    {
205 206 207 208 209 210
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
211 212
    }

213 214 215 216 217 218 219
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

220 221 222
    return(0);
}

223 224
/* darshan_log_putjob()
 *
225
 * write job level metadata to darshan log file
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
    int len;
    int ret;

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

249 250 251
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
252
    {
253
        fd->err = -1;
254 255 256 257 258 259 260 261 262 263 264 265 266
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
267 268 269
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
270
    int ret;
271

272 273
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
274
    {
275 276
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
277

278 279
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
280
    }
281

282 283 284 285
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
286
    if(newline)
287
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
288 289 290 291

    return (0);
}

292 293 294
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
295
 * NOTE: this needs to be called immediately following put_job as it
296
 * expects the file pointer to be positioned immediately following
297
 * the darshan job information
298 299 300 301 302 303 304
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
305 306

    len = strlen(buf);
307

308 309
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
310
    {
311
        fd->err = -1;
312 313 314 315 316 317 318
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

319 320
/* darshan_log_getmounts()
 * 
321
 * retrieves mount table information from the log. Note that mnt_pts and
322
 * fs_types are arrays that will be allocated by the function and must be
323 324 325
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
326
 */
327
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
328 329 330 331
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
332
    int ret;
333

334 335
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
336
    {
337 338
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
339

340 341
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
342
    }
343

344
    /* count entries */
345
    *count = 0;
346
    pos = fd->exe_mnt_data;
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
368
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
369 370
    {
        /* overestimate string lengths */
371
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
372
        assert((*mnt_pts)[array_index]);
373
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
374 375
        assert((*fs_types)[array_index]);

376 377 378
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
379
        {
380
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
381 382 383 384 385 386 387
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

388 389 390
    return(0);
}

391 392 393 394 395 396 397 398 399 400 401 402 403
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
404 405 406
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
407 408 409
    int ret;

    /* write each mount entry to file */
410
    tmp = mnt_dat;
411 412 413
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
414 415 416 417 418 419

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

420 421
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
422
    {
423
        fd->err = -1;
424
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
425 426 427 428 429 430
        return(-1);
    }

    return(0);
}

431
/* darshan_log_gethash()
432
 *
433
 * read the hash of records from the darshan log file
434 435 436
 *
 * returns 0 on success, -1 on failure
 */
437
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
438
{
439
    char *hash_buf;
440
    int hash_buf_sz;
441
    char *buf_ptr;
442
    darshan_record_id *rec_id_ptr;
443
    uint32_t *path_len_ptr, tmp_path_len;
444 445
    char *path_ptr;
    struct darshan_record_ref *ref;
446 447 448
    int read;
    int read_req_sz;
    int buf_remaining = 0;
449

450 451 452
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
453
        return(-1);
454 455
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
456

457
    do
458
    {
459 460
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
461
         */
462 463 464 465
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
466
        {
467 468 469
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
470 471
        }

472 473 474 475 476 477 478 479
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
480
        {
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
502
            {
503 504 505
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
506
            }
507 508 509

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
510
            {
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
532 533
            }

534 535 536
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
537
        }
538

539 540 541 542 543 544 545 546
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
547

548
    free(hash_buf);
549 550 551
    return(0);
}

552 553 554
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
555 556 557
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
558 559 560 561 562 563
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    char *hash_buf;
564
    int hash_buf_sz;
565
    struct darshan_record_ref *ref, *tmp;
566 567 568
    char *buf_ptr;
    int path_len;
    int wrote;
569

570 571
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
572 573 574
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
575
    memset(hash_buf, 0, hash_buf_sz);
576

577
    /* individually serialize each hash record and write to log file */
578 579
    HASH_ITER(hlink, hash, ref, tmp)
    {
580 581
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
582

583
        /* the hash buffer has space to serialize this record
584 585 586
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
587 588 589 590 591 592
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
593

594 595 596 597 598 599 600 601 602 603
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
            fd->err = -1;
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
604 605 606 607 608 609 610 611
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
612 613 614
 * get a chunk of module data from the darshan log file
 *
 * returns 0 on success, -1 on failure
615 616
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
617
    void *buf, int len)
618 619
{
    int ret;
620

621
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
622
    {
623 624
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
625 626
    }

627 628 629 630
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
631 632
    ret = darshan_log_dzread(fd, mod_id, buf, len);
    if(ret < 0)
633
    {
634
        fprintf(stderr,
635
            "Error: failed to read module %s data from darshan log file.\n",
636
            darshan_module_names[mod_id]);
637 638 639
        return(-1);
    }

640
    return(ret);
641 642
}

643 644 645
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
646 647 648 649 650 651
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
652 653 654 655 656 657 658 659
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
    int ret;

Shane Snyder's avatar
Shane Snyder committed
660
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
661
    {
662
        fd->err = -1;
Shane Snyder's avatar
Shane Snyder committed
663 664
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
665 666 667
    }

    /* write the module chunk to the log file */
668 669
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
670
    {
671
        fd->err = -1;
672 673 674 675 676 677 678 679 680
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

681 682
/* darshan_log_close()
 *
683
 * close an open darshan file descriptor, freeing any resources
684 685 686 687
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
688
{
689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
    int ret;

    if(fd->o_flags & O_WRONLY)
    {
        /* flush the last region of the log to file */
        switch(fd->comp_type)
        {
            case DARSHAN_ZLIB_COMP:
                ret = darshan_log_libz_flush(fd, fd->dz_prev_reg_id);
                if(ret == 0)
                    break;
            default:
                /* if flush fails, remove the output log file */
                fd->err = -1;
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
        if(fd->err != -1)
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
                fd->err = -1;
        }
    }

    close(fd->fildes);

    /* remove output log file if error writing to it */
    if((fd->o_flags & O_WRONLY ) && (fd->err == -1))
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
            fd->logfile_path);
        unlink(fd->logfile_path);
    }
725

726
    darshan_log_dzdestroy(fd);
727 728
    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
729
    free(fd);
730 731

    return;
732 733
}

734
/* **************************************************** */
735

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

    fd->comp_type = header.comp_type;

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
    header.comp_type = fd->comp_type;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

842 843 844 845
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
846
    off_t ret_off;
847 848 849 850

    if(fd->pos == offset)
        return(0);

851 852
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
853
    {
854 855
        fd->pos = offset;
        return(0);
856 857 858 859 860
    }

    return(-1);
}

861
/* return amount read on success, 0 on EOF, -1 on failure.
862
 */
863
static int darshan_log_read(darshan_fd fd, void* buf, int len)
864 865 866
{
    int ret;

867 868 869 870
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
871

872
    return(ret);
873 874
}

875
/* return amount written on success, -1 on failure.
876
 */
877
static int darshan_log_write(darshan_fd fd, void* buf, int len)
878 879 880
{
    int ret;

881 882 883 884 885 886 887
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

888
static int darshan_log_dzinit(darshan_fd fd)
889 890 891
{
    int ret;

892 893 894 895 896 897 898 899
    /* initialize buffers for staging compressed data to/from log file */
    fd->dz_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(fd->dz_buf == NULL)
        return(-1);

    fd->dz_prev_reg_id = DARSHAN_HEADER_REGION_ID;

    switch(fd->comp_type)
900 901
    {
        case DARSHAN_ZLIB_COMP:
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
                free(fd->dz_buf);
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
            if(fd->o_flags == O_RDONLY)
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
                tmp_zstrm->next_out = fd->dz_buf;
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
                free(fd->dz_buf);
                return(-1);
            }
            fd->dz_strm = tmp_zstrm;
935
            break;
936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
        }
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

static void darshan_log_dzdestroy(darshan_fd fd)
{
    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            if(fd->o_flags == O_RDONLY)
            {
                inflateEnd(fd->dz_strm);
            }
            else
            {
                deflateEnd(fd->dz_strm);
            }
            free(fd->dz_strm);
            break;
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

    free(fd->dz_buf);
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
    int ret;

    switch(fd->comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_log_libz_read(fd, region_id, buf, len);
976 977
            break;
        default:
978
            fprintf(stderr, "Error: invalid compression type.\n");
979 980 981 982 983 984
            return(-1);
    }

    return(ret);
}

985
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len)
Shane Snyder's avatar
Shane Snyder committed
986 987 988
{
    int ret;

989
    switch(fd->comp_type)
Shane Snyder's avatar
Shane Snyder committed
990 991
    {
        case DARSHAN_ZLIB_COMP:
992
            ret = darshan_log_libz_write(fd, region_id, buf, len);
Shane Snyder's avatar
Shane Snyder committed
993 994
            break;
        default:
995
            fprintf(stderr, "Error: invalid compression type.\n");
Shane Snyder's avatar
Shane Snyder committed
996 997 998 999 1000 1001
            return(-1);
    }

    return(ret);
}

1002
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len)
1003 1004
{
    int ret;
1005 1006 1007 1008
    int total_bytes = 0;
    int tmp_out_bytes;
    struct darshan_log_map map;
    z_stream *z_strmp = (z_stream *)fd->dz_strm;
1009

1010
    assert(fd->dz_strm);
1011

1012 1013
    /* if new log region, we reload buffers and clear eor flag */
    if(region_id != fd->dz_prev_reg_id)
1014
    {
1015 1016 1017
        z_strmp->avail_in = 0;
        fd->dz_eor = 0;
        fd->dz_prev_reg_id = region_id;
1018 1019
    }

1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
    if(region_id == DARSHAN_JOB_REGION_ID)
        map = fd->job_map;
    else if(region_id == DARSHAN_REC_MAP_REGION_ID)
        map = fd->rec_map;
    else
        map = fd->mod_map[region_id];

    z_strmp->avail_out = len;
    z_strmp->next_out = buf;

    /* we just decompress until the output buffer is full, assuming there
     * is enough compressed data in file to satisfy the request size.
     */
    while(z_strmp->avail_out)
1034
    {
1035 1036
        /* check if we need more compressed data */
        if(z_strmp->avail_in == 0)
1037
        {
1038 1039
            /* if the eor flag is set, clear it and return -- future
             * reads of this log region will restart at the beginning
1040
             */
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
            if(fd->dz_eor)
            {
                fd->dz_eor = 0;
                break;
            }

            /* read more data from input file */
            ret = darshan_log_dzload(fd, map);
            if(ret < 0)
                return(-1);
            assert(fd->dz_size > 0);

            z_strmp->avail_in = fd->dz_size;
            z_strmp->next_in = fd->dz_buf;
1055 1056
        }

1057 1058 1059
        tmp_out_bytes = z_strmp->total_out;
        ret = inflate(z_strmp, Z_NO_FLUSH);
        if(ret != Z_OK && ret != Z_STREAM_END)
1060
        {
1061
            fprintf(stderr, "Error: unable to inflate darshan log data.\n");
1062 1063
            return(-1);
        }
1064
        total_bytes += (z_strmp->total_out - tmp_out_bytes);
1065

1066 1067 1068
        /* reset the decompression if we encountered end of stream */
        if(ret == Z_STREAM_END)
            inflateReset(z_strmp);
1069 1070
    }

1071
    return(total_bytes);
1072 1073
}

1074
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len)