darshan-logutils.c 47.5 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29 30 31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32 33
struct darshan_dz_state
{
34 35 36 37 38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39 40 41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43 44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

65
    /* compression/decompression stream read/write state */
66 67 68
    struct darshan_dz_state dz;
};

69 70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71 72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74 75
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
76 77
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
78 79 80 81
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
82
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
83
#ifdef HAVE_LIBBZ2
84 85 86 87
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
88 89
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
90 91
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
92 93
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
94

Shane Snyder's avatar
Shane Snyder committed
95 96
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c) c,
97 98
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
99
    DARSHAN_MODULE_IDS
100
};
Shane Snyder's avatar
Shane Snyder committed
101
#undef X
102 103 104

/* darshan_log_open()
 *
105
 * open an existing darshan log file for reading only
106
 *
107
 * returns file descriptor on success, NULL on failure
108
 */
109
darshan_fd darshan_log_open(const char *name)
110
{
111
    darshan_fd tmp_fd;
112
    int ret;
113

114
    /* allocate a darshan file descriptor */
115 116 117 118
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
119 120 121 122 123 124 125
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
126

127
    /* open the log file in read mode */
128 129
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
130
    {
131
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
132
        free(tmp_fd->state);
133 134 135
        free(tmp_fd);
        return(NULL);
    }
136
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
137 138 139 140 141

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
142 143
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
144
        free(tmp_fd);
145 146 147 148
        return(NULL);
    }

    /* initialize compression data structures */
149
    ret = darshan_log_dzinit(tmp_fd);
150 151 152
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
153 154
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
155 156
        free(tmp_fd);
        return(NULL);
157 158
    }

159 160 161 162 163 164 165
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
166
 * returns file descriptor on success, NULL on failure
167
 */
168 169
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
170 171
{
    darshan_fd tmp_fd;
172
    int ret;
173

174
    /* allocate a darshan file descriptor */
175 176 177 178
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
179 180 181 182 183 184 185
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
186
    tmp_fd->comp_type = comp_type;
187

188 189 190
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
191
    {
192
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
193
        free(tmp_fd->state);
194
        free(tmp_fd);
195
        return(NULL);
196
    }
197
    tmp_fd->state->creat_flag = 1;
198
    tmp_fd->partial_flag = partial_flag;
199
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
200

201 202 203 204 205
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
206
    if(ret < 0)
207 208
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
209 210
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
211 212 213
        free(tmp_fd);
        unlink(name);
        return(NULL);
214 215
    }

216
    /* initialize compression data structures */
217
    ret = darshan_log_dzinit(tmp_fd);
218 219
    if(ret < 0)
    {
220
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
221 222
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
223 224 225
        free(tmp_fd);
        unlink(name);
        return(NULL);
226 227
    }

228
    return(tmp_fd);
229 230
}

231
/* darshan_log_getjob()
232 233
 *
 * read job level metadata from the darshan log file
234
 *
235
 * returns 0 on success, -1 on failure
236
 */
237
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
238
{
239
    struct darshan_fd_int_state *state = fd->state;
240 241
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
242
    int ret;
243

244
    assert(state);
245 246
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

247
    /* read the compressed job data from the log file */
248
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
249
    if(ret <= (int)sizeof(*job))
250
    {
251
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
252 253
        return(-1);
    }
254 255

    memcpy(job, job_buf, sizeof(*job));
256

257
    if(fd->swap_flag)
258
    {
259 260 261 262 263 264
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
265 266
    }

267
    /* save trailing exe & mount information, so it can be retrieved later */
268 269 270
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
271
        return(-1);
272
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
273

274 275 276
    return(0);
}

277 278
/* darshan_log_putjob()
 *
279
 * write job level metadata to darshan log file
280 281 282 283 284
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
285
    struct darshan_fd_int_state *state = fd->state;
286 287 288 289
    struct darshan_job job_copy;
    int len;
    int ret;

290 291
    assert(state);

292 293 294 295 296 297 298 299 300 301 302 303 304 305
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

306 307
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
308
    if(ret != sizeof(*job))
309
    {
310
        state->err = -1;
311 312 313 314 315 316 317 318 319 320 321 322 323
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
324 325
int darshan_log_getexe(darshan_fd fd, char *buf)
{
326
    struct darshan_fd_int_state *state = fd->state;
327
    char *newline;
328
    int ret;
329

330 331
    assert(state);

332
    /* if the exe/mount data has not been saved yet, read in the job info */
333
    if(!(state->exe_mnt_data))
334
    {
335 336
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
337

338
        if(ret < 0 || !(state->exe_mnt_data))
339
            return(-1);
340
    }
341

342
    /* exe string is located before the first line break */
343
    newline = strchr(state->exe_mnt_data, '\n');
344 345

    /* copy over the exe string */
346
    if(newline)
347
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
348 349 350 351

    return (0);
}

352 353 354
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
355
 * NOTE: this needs to be called immediately following put_job as it
356
 * expects the file pointer to be positioned immediately following
357
 * the darshan job information
358 359 360 361 362
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
363 364
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
365
    int ret;
366

367
    assert(fd->state);
368

369 370
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
371
    {
372
        state->err = -1;
373 374 375 376 377 378 379
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

380 381
/* darshan_log_getmounts()
 * 
382
 * retrieves mount table information from the log. Note that mnt_pts and
383
 * fs_types are arrays that will be allocated by the function and must be
384 385 386
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
387
 */
388
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
389 390
    char*** fs_types, int* count)
{
391
    struct darshan_fd_int_state *state = fd->state;
392 393
    char *pos;
    int array_index = 0;
394
    int ret;
395

396 397
    assert(state);

398
    /* if the exe/mount data has not been saved yet, read in the job info */
399
    if(!(state->exe_mnt_data))
400
    {
401 402
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
403

404
        if(ret < 0 || !(state->exe_mnt_data))
405
            return(-1);
406
    }
407

408
    /* count entries */
409
    *count = 0;
410
    pos = state->exe_mnt_data;
411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
432
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
433 434
    {
        /* overestimate string lengths */
435
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
436
        assert((*mnt_pts)[array_index]);
437
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
438 439
        assert((*fs_types)[array_index]);

440 441 442
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
443
        {
444
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
445 446 447 448 449 450 451
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

452 453 454
    return(0);
}

455 456 457 458 459 460 461 462 463 464 465
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
466
    struct darshan_fd_int_state *state = fd->state;
467 468
    int i;
    char line[1024];
469 470 471
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
472 473
    int ret;

474 475
    assert(state);

476
    /* write each mount entry to file */
477
    tmp = mnt_dat;
478 479 480
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
481 482 483 484 485 486

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

487 488
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
489
    {
490
        state->err = -1;
491
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
492 493 494 495 496 497
        return(-1);
    }

    return(0);
}

498
/* darshan_log_gethash()
499
 *
500
 * read the hash of records from the darshan log file
501 502 503
 *
 * returns 0 on success, -1 on failure
 */
504
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
505
{
506
    struct darshan_fd_int_state *state = fd->state;
507
    char *hash_buf;
508
    int hash_buf_sz;
509
    char *buf_ptr;
510 511
    darshan_record_id *rec_id_ptr;
    char *path_ptr;
512
    char *tmp_p;
513
    struct darshan_record_ref *ref;
514 515
    int read;
    int read_req_sz;
516
    int buf_rem = 0;
517

518 519
    assert(state);

520 521 522 523 524 525 526
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

527 528 529
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
530
        return(-1);
531 532
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
533

534
    do
535
    {
536 537
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
538
         */
539
        read_req_sz = hash_buf_sz - buf_rem;
540
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
541
            hash_buf + buf_rem, read_req_sz);
542
        if(read < 0)
543
        {
544 545 546
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
547 548
        }

549 550 551 552 553 554
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
555 556
        buf_rem += read;
        while(buf_rem > (sizeof(darshan_record_id) + 1))
557
        {
558 559 560 561 562 563 564 565 566
            tmp_p = buf_ptr + sizeof(darshan_record_id);
            while(tmp_p < (buf_ptr + buf_rem))
            {
                /* look for terminating null character for record name */
                if(*tmp_p == '\0')
                    break;
                tmp_p++;
            }
            if(*tmp_p != '\0')
567 568 569 570
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
571
             *          ... darshan_record_id | path '\0' ...
572 573 574 575 576 577
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
578
            {
579 580
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
581
            }
582 583 584

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
585
            {
586 587 588 589 590 591
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
592
                ref->rec.name = malloc(strlen(path_ptr) + 1);
593 594 595 596 597 598 599 600 601
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
602
                strcpy(ref->rec.name, path_ptr);
603 604 605

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
606 607
            }

608 609
            buf_ptr += strlen(path_ptr) + 1;
            buf_rem -= (sizeof(darshan_record_id) + strlen(path_ptr) + 1);
610
        }
611

612
        /* copy any leftover data to beginning of buffer to parse next */
613
        memcpy(hash_buf, buf_ptr, buf_rem);
614 615 616 617 618

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
619
    assert(buf_rem == 0);
620

621
    free(hash_buf);
622 623 624
    return(0);
}

625 626 627
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
628 629 630
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
631 632 633 634 635
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
636
    struct darshan_fd_int_state *state = fd->state;
637
    char *hash_buf;
638
    int hash_buf_sz;
639
    struct darshan_record_ref *ref, *tmp;
640 641 642
    char *buf_ptr;
    int path_len;
    int wrote;
643

644 645
    assert(state);

646 647
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
648 649 650
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
651
    memset(hash_buf, 0, hash_buf_sz);
652

653
    /* individually serialize each hash record and write to log file */
654 655
    HASH_ITER(hlink, hash, ref, tmp)
    {
656 657
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
658

659
        /* the hash buffer has space to serialize this record
660 661 662
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
663 664 665 666 667 668
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
669

670 671 672 673 674
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
675
            state->err = -1;
676 677 678 679
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
680 681 682 683 684 685 686 687
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
688 689
 * get a chunk of module data from the darshan log file
 *
690
 * returns number of bytes read on success, -1 on failure
691 692
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
693
    void *mod_buf, int mod_buf_sz)
694
{
695
    struct darshan_fd_int_state *state = fd->state;
696
    int ret;
697

698 699
    assert(state);

700
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
701
    {
702 703
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
704 705
    }

706 707 708 709
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
710
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
711
    if(ret < 0)
712
    {
713
        fprintf(stderr,
714
            "Error: failed to read module %s data from darshan log file.\n",
715
            darshan_module_names[mod_id]);
716 717 718
        return(-1);
    }

719
    return(ret);
720 721
}

722 723 724
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
725 726 727 728 729 730
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
731
 *
732
 * returns number of bytes written on success, -1 on failure
733 734 735 736
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
737
    struct darshan_fd_int_state *state = fd->state;
738 739
    int ret;

740 741
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
742
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
743
    {
744
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
745 746
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
747 748 749
    }

    /* write the module chunk to the log file */
750 751
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
752
    {
753
        state->err = -1;
754 755 756 757 758 759 760 761 762
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

763 764
/* darshan_log_close()
 *
765
 * close an open darshan file descriptor, freeing any resources
766 767 768
 *
 */
void darshan_log_close(darshan_fd fd)
769
{
770
    struct darshan_fd_int_state *state = fd->state;
771 772
    int ret;

773 774 775 776
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
777 778
    {
        /* flush the last region of the log to file */
779
        switch(fd->comp_type)
780 781
        {
            case DARSHAN_ZLIB_COMP:
782
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
783 784
                if(ret == 0)
                    break;
785 786
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
787
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
788 789 790
                if(ret == 0)
                    break;
#endif 
791 792
            default:
                /* if flush fails, remove the output log file */
793
                state->err = -1;
794 795 796 797 798
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
799
        if(state->err != -1)
800 801 802
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
803
                state->err = -1;
804 805 806
        }
    }

807
    close(state->fildes);
808 809

    /* remove output log file if error writing to it */
810
    if((state->creat_flag) && (state->err == -1))
811 812
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
813 814
            state->logfile_path);
        unlink(state->logfile_path);
815
    }
816

817
    darshan_log_dzdestroy(fd);
818 819 820
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
821
    free(fd);
822 823

    return;
824 825
}

826
/* **************************************************** */
827

828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
848
    if(ret != (int)sizeof(header))
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

887
    fd->comp_type = header.comp_type;
888
    fd->partial_flag = header.partial_flag;
889 890 891 892 893

    /* save the mapping of data within log file to this file descriptor */
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
    if(fd->rec_map.off == 0)
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
        fd->job_map.len = fd->rec_map.off - fd->job_map.off;
    }

923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
945
    header.comp_type = fd->comp_type;
946
    header.partial_flag = fd->partial_flag;
947 948 949 950 951 952 953

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
954
    if(ret != (int)sizeof(header))
955 956 957 958 959 960 961 962
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

963 964 965 966
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
967
    struct darshan_fd_int_state *state = fd->state;
968
    off_t ret_off;
969

970
    if(state->pos == offset)
971 972
        return(0);

973
    ret_off = lseek(state->fildes, offset, SEEK_SET);
974
    if(ret_off == offset)
975
    {
976
        state->pos = offset;
977
        return(0);
978 979 980 981 982
    }

    return(-1);
}

983
/* return amount read on success, 0 on EOF, -1 on failure.
984
 */
985
static int darshan_log_read(darshan_fd fd, void* buf, int len)
986
{
987
    struct darshan_fd_int_state *state = fd->state;
988
    int ret;
989
    unsigned int read_so_far = 0;
990

991 992 993 994 995 996 997 998 999
    do
    {
        ret = read(state->fildes, buf + read_so_far, len - read_so_far);
        if(ret <= 0)
            break;
        read_so_far += ret;
    } while(read_so_far < len);
    if(ret < 0)
        return(-1);
1000

1001 1002
    state->pos += read_so_far;
    return(read_so_far);
1003 1004
}

1005
/* return amount written on success, -1 on failure.
1006
 */
1007
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1008
{
1009
    struct darshan_fd_int_state *state = fd->state;
1010
    int ret;
1011
    unsigned int wrote_so_far = 0;
1012

1013 1014 1015 1016 1017 1018 1019 1020 1021
    do
    {
        ret = write(state->fildes, buf + wrote_so_far, len - wrote_so_far);
        if(ret <= 0)
            break;
        wrote_so_far += ret;
    } while(wrote_so_far < len);
    if(ret < 0)
        return(-1);
1022

1023 1024
    state->pos += wrote_so_far;
    return(wrote_so_far);
1025 1026
}

1027
static int darshan_log_dzinit(darshan_fd fd)
1028
{
1029
    struct darshan_fd_int_state *state = fd->state;
1030 1031
    int ret;

1032 1033 1034
    /* initialize buffers for staging compressed data
     * to/from log file
     */
1035 1036
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1037
        return(-1);
1038
    state->dz.size = 0;
1039
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1040

1041
    switch(fd->comp_type)
1042 1043
    {
        case DARSHAN_ZLIB_COMP:
1044 1045 1046 1047
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1048
                free(state->dz.buf);
1049 1050 1051 1052 1053 1054 1055 1056 1057
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1058
            if(!(state->creat_flag))
1059 1060 1061 1062 1063 1064 1065 1066 1067
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1068
                tmp_zstrm->next_out = state->dz.buf;
1069 1070 1071 1072
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1073
                free(state->dz.buf);
1074 1075
                return(-1);
            }
1076
            state->dz.comp_dat = tmp_zstrm;
1077
            break;
1078
        }
1079 1080 1081 1082 1083 1084
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1085
                free(state->dz.buf);
1086 1087 1088 1089 1090 1091
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
1092
            tmp_bzstrm->next_in = NULL;
1093

1094
            if(!(state->creat_flag))
1095 1096 1097 1098 1099 1100 1101 1102 1103
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1104
                tmp_bzstrm->next_out = (char *)state->dz.buf;
1105 1106 1107 1108
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
1109
                free(state->dz.buf);
1110 1111
                return(-1);
            }
1112
            state->dz.comp_dat = tmp_bzstrm;
1113 1114 1115
            break;
        }
#endif
1116 1117 1118 1119 1120 1121 1122 1123
        case DARSHAN_NO_COMP:
        {
            /* we just track an offset into the staging buffers for no_comp */
            int *buf_off = malloc(sizeof(int));
            *buf_off = 0;
            state->dz.comp_dat = buf_off;
            break;
        }
1124 1125 1126 1127 1128 1129 1130 1131
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

1132
static void darshan_log_dzdestroy(darshan_fd fd)
1133
{
1134 1135 1136
    struct darshan_fd_int_state *state = fd->state;

    switch(fd->comp_type)
1137 1138
    {
        case DARSHAN_ZLIB_COMP:
1139
            if(!(state->creat_flag))
1140
                inflateEnd((z_stream *)state->dz.comp_dat);
1141
            else
1142
                deflateEnd((z_stream *)state->dz.comp_dat);
1143
            break;
1144 1145
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1146
            if(!(state->creat_flag))
1147
                BZ2_bzDecompressEnd((bz_stream *)state->dz.comp_dat);
1148
            else
1149
                BZ2_bzCompressEnd((bz_stream *)state->dz.comp_dat);
1150 1151
            break;
#endif
1152 1153 1154 1155 1156
        case DARSHAN_NO_COMP:
        {
            /* do nothing */
            break;
        }
1157 1158 1159 1160
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

1161
    free(state->dz.comp_dat);
1162
    free(state->dz.buf);
1163 1164 1165 1166 1167
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
1168
    struct darshan_fd_int_state *state = fd->state;
1169 1170
    struct darshan_log_map map;
    int reset_strm_flag = 0;
1171 1172
    int ret;

1173 1174 1175 1176
    /* if new log region, we reload buffers and clear eor flag */
    if(region_id != state->dz.prev_reg_id)
    {
        state->dz.eor = 0;
1177
        state->dz.size = 0;
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
        reset_strm_flag = 1; /* reset libz/bzip2 streams */
    }

    if(region_id == DARSHAN_JOB_REGION_ID)
        map = fd->job_map;
    else if(region_id == DARSHAN_REC_MAP_REGION_ID)
        map = fd->rec_map;
    else
        map = fd->mod_map[region_id];

    switch(fd->comp_type)
1189 1190
    {
        case DARSHAN_ZLIB_COMP:
1191
            ret = darshan_log_libz_read(fd, map, buf, len, reset_strm_flag);
1192
            break;
1193 1194
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1195
            ret = darshan_log_bzip2_read(fd, map, buf, len, reset_strm_flag);
1196 1197
            break;
#endif
1198 1199 1200 1201 1202
        case DARSHAN_NO_COMP:
        {
            ret = darshan_log_noz_read(fd, map, buf, len, reset_strm_flag);
            break;
        }
1203
        default:
1204
            fprintf(stderr, "Error: invalid compression type.\n");
1205 1206 1207
            return(-1);
    }

1208
    state->dz.prev_reg_id = region_id;
1209 1210 1211
    return(ret);
}

1212
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len)
1213
{
1214
    struct darshan_fd_int_state *state = fd->state;
1215 1216
    struct darshan_log_map *map_p;
    int flush_strm_flag = 0;
1217 1218
    int ret;

1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
    /* if new log region, finish prev region's zstream and flush to log file */
    if(region_id != state->dz.prev_reg_id)
    {
        /* error out if the region we are writing to precedes the previous
         * region we wrote -- we shouldn't be moving backwards in the log
         */
        if(region_id < state->dz.prev_reg_id)
            return(-1);

        if(state->dz.prev_reg_id != DARSHAN_HEADER_REGION_ID)
            flush_strm_flag = 1;
    }

    if(region_id == DARSHAN_JOB_REGION_ID)
        map_p = &(fd->job_map);
    else if(region_id == DARSHAN_REC_MAP_REGION_ID)
        map_p = &(fd->rec_map);
    else
        map_p = &(fd->mod_map[region_id]);

    switch(fd->comp_type)
1240 1241
    {
        case DARSHAN_ZLIB_COMP:
1242
            ret = darshan_log_libz_write(fd, map_p, buf, len, flush_strm_flag);
1243
            break;
1244 1245
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1246
            ret = darshan_log_bzip2_write(fd, map_p, buf, len, flush_strm_flag);
1247 1248
            break;
#endif
1249 1250 1251 1252
        case DARSHAN_NO_COMP:
            fprintf(stderr,
                "Error: uncompressed writing of log files is not supported.\n");
            return(-1);
1253
        default:
1254
            fprintf(stderr, "Error: invalid compression type.\n");
1255 1256 1257
            return(-1);
    }

1258
    state->dz.prev_reg_id = region_id;
1259 1260 1261
    return(ret);
}

1262 1263
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_stream_flag)
1264
{
1265
    struct darshan_fd_int_state *state = fd->state;
1266
    int ret;
1267 1268
    int total_bytes = 0;
    int tmp_out_bytes;
1269
    z_stream *z_strmp = (z_stream *)state->dz.comp_dat;
1270

1271
    assert(z_strmp);
1272

1273
    if(reset_stream_flag)
1274 1275 1276 1277 1278 1279