darshan-logutils.c 52.6 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32 33
struct darshan_dz_state
{
34 35 36 37 38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39 40 41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43 44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64 65 66 67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70 71 72
    struct darshan_dz_state dz;
};

73 74 75 76 77 78 79 80 81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83 84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85 86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87 88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90 91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92 93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94 95 96 97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100 101 102 103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104 105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106 107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108 109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111 112 113 114 115 116 117 118
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);


/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
119 120 121

/* darshan_log_open()
 *
122
 * open an existing darshan log file for reading only
123
 *
124
 * returns file descriptor on success, NULL on failure
125
 */
126
darshan_fd darshan_log_open(const char *name)
127
{
128
    darshan_fd tmp_fd;
129
    int ret;
130

131
    /* allocate a darshan file descriptor */
132 133 134 135
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
136 137 138 139 140 141 142
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
143

144
    /* open the log file in read mode */
145 146
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd->state);
150 151 152
        free(tmp_fd);
        return(NULL);
    }
153
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
154 155

    /* read the header from the log file to init fd data structures */
156
    ret = darshan_log_get_header(tmp_fd);
157 158
    if(ret < 0)
    {
159
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
160 161
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
162
        free(tmp_fd);
163 164 165 166
        return(NULL);
    }

    /* initialize compression data structures */
167
    ret = darshan_log_dzinit(tmp_fd);
168 169 170
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
171 172
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
173 174
        free(tmp_fd);
        return(NULL);
175 176
    }

177 178 179 180 181 182 183
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
184
 * returns file descriptor on success, NULL on failure
185
 */
186 187
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
188 189
{
    darshan_fd tmp_fd;
190
    int ret;
191

192
    /* allocate a darshan file descriptor */
193 194 195 196
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
197 198 199 200 201 202 203
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
204
    tmp_fd->comp_type = comp_type;
205

206 207 208
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
209
    {
210
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
211
        free(tmp_fd->state);
212
        free(tmp_fd);
213
        return(NULL);
214
    }
215
    tmp_fd->state->creat_flag = 1;
216
    tmp_fd->partial_flag = partial_flag;
217
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
218

219 220 221 222 223
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
224
    if(ret < 0)
225 226
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
227 228
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
229 230 231
        free(tmp_fd);
        unlink(name);
        return(NULL);
232 233
    }

234
    /* initialize compression data structures */
235
    ret = darshan_log_dzinit(tmp_fd);
236 237
    if(ret < 0)
    {
238
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
239 240
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
241 242 243
        free(tmp_fd);
        unlink(name);
        return(NULL);
244 245
    }

246
    return(tmp_fd);
247 248
}

249
/* darshan_log_get_job()
250 251
 *
 * read job level metadata from the darshan log file
252
 *
253
 * returns 0 on success, -1 on failure
254
 */
255
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
256
{
257
    struct darshan_fd_int_state *state = fd->state;
258 259
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
260
    int ret;
261

262
    assert(state);
263 264
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

265
    /* read the compressed job data from the log file */
266
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
267
    if(ret <= (int)sizeof(*job))
268
    {
269
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
270 271
        return(-1);
    }
272 273

    memcpy(job, job_buf, sizeof(*job));
274

275
    if(fd->swap_flag)
276
    {
277 278 279 280 281 282
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
283 284
    }

285
    /* save trailing exe & mount information, so it can be retrieved later */
286 287 288
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
289
        return(-1);
290
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
291

292 293 294
    return(0);
}

295
/* darshan_log_put_job()
296
 *
297
 * write job level metadata to darshan log file
298 299 300
 *
 * returns 0 on success, -1 on failure
 */
301
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
302
{
303
    struct darshan_fd_int_state *state = fd->state;
304 305 306 307
    struct darshan_job job_copy;
    int len;
    int ret;

308 309
    assert(state);

310 311 312 313 314 315 316 317 318 319 320 321 322 323
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

324 325
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
326
    if(ret != sizeof(*job))
327
    {
328
        state->err = -1;
329 330 331 332 333 334 335
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

336
/* darshan_log_get_exe()
337 338 339 340 341
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
342
int darshan_log_get_exe(darshan_fd fd, char *buf)
343
{
344
    struct darshan_fd_int_state *state = fd->state;
345
    char *newline;
346
    int ret;
347

348 349
    assert(state);

350
    /* if the exe/mount data has not been saved yet, read in the job info */
351
    if(!(state->exe_mnt_data))
352
    {
353
        struct darshan_job job;
354
        ret = darshan_log_get_job(fd, &job);
355

356
        if(ret < 0 || !(state->exe_mnt_data))
357
            return(-1);
358
    }
359

360
    /* exe string is located before the first line break */
361
    newline = strchr(state->exe_mnt_data, '\n');
362 363

    /* copy over the exe string */
364
    if(newline)
365
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
366 367 368 369

    return (0);
}

370
/* darshan_log_put_exe()
371 372
 *
 * wrties the application exe name to darshan log file
373
 * NOTE: this needs to be called immediately following put_job as it
374
 * expects the file pointer to be positioned immediately following
375
 * the darshan job information
376 377 378
 *
 * returns 0 on success, -1 on failure 
 */
379
int darshan_log_put_exe(darshan_fd fd, char *buf)
380
{
381 382
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
383
    int ret;
384

385
    assert(fd->state);
386

387 388
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
389
    {
390
        state->err = -1;
391 392 393 394 395 396 397
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
/* darshan_log_get_mounts()
399
 * 
400 401 402
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
403 404
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
407
    int* count)
408
{
409
    struct darshan_fd_int_state *state = fd->state;
410 411
    char *pos;
    int array_index = 0;
412
    int ret;
413

414 415
    assert(state);

416
    /* if the exe/mount data has not been saved yet, read in the job info */
417
    if(!(state->exe_mnt_data))
418
    {
419
        struct darshan_job job;
420
        ret = darshan_log_get_job(fd, &job);
421

422
        if(ret < 0 || !(state->exe_mnt_data))
423
            return(-1);
424
    }
425

426
    /* count entries */
427
    *count = 0;
428
    pos = state->exe_mnt_data;
429 430 431 432 433 434 435 436 437 438 439 440 441
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
442 443
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
444

445
    /* work through the table and parse each line (except for
446 447
     * first, which holds command line information)
     */
448 449
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
450
    {
451 452
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
453
        if(ret != 2)
454
        {
455
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
456 457 458 459 460
            return(-1);
        }
        array_index++;
    }

461 462
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

463 464 465
    return(0);
}

466
/* darshan_log_put_mounts()
467 468 469 470 471 472 473 474
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
475
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
476
    int count)
477
{
478
    struct darshan_fd_int_state *state = fd->state;
479 480
    int i;
    char line[1024];
481 482 483
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
484 485
    int ret;

486 487
    assert(state);

488
    /* write each mount entry to file */
489
    tmp = mnt_dat;
490 491
    for(i=count-1; i>=0; i--)
    {
492
        sprintf(line, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
493 494 495 496 497 498

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

499 500
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
501
    {
502
        state->err = -1;
503
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
504 505 506 507 508 509
        return(-1);
    }

    return(0);
}

510
/* darshan_log_get_namehash()
511
 *
512 513
 * read the set of name records from the darshan log file and add to the
 * given hash table
514 515 516
 *
 * returns 0 on success, -1 on failure
 */
517
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
518
{
519
    struct darshan_fd_int_state *state = fd->state;
520 521
    char *name_rec_buf;
    int name_rec_buf_sz;
522 523
    int read;
    int read_req_sz;
524 525
    int buf_len = 0;
    int buf_processed;
526

527 528
    assert(state);

529 530
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
531 532 533 534 535
    {
        *hash = NULL;
        return(0);
    }

536 537 538 539
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
540
        return(-1);
541
    memset(name_rec_buf, 0, name_rec_buf_sz);
542

543
    do
544
    {
545
        /* read chunks of the darshan record id -> name mapping from log file,
546
         * constructing a hash table in the process
547
         */
548
        read_req_sz = name_rec_buf_sz - buf_len;
549
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
550
            name_rec_buf + buf_len, read_req_sz);
551
        if(read < 0)
552
        {
553 554
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
555
            return(-1);
556
        }
557
        buf_len += read;
558

559 560
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
561

562
        /* copy any leftover data to beginning of buffer to parse next */
563 564
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
565 566 567 568 569

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
570
    assert(buf_len == 0);
571

572
    free(name_rec_buf);
573 574 575
    return(0);
}

576
/* darshan_log_put_namehash()
577
 *
578
 * writes the hash table of name records to the darshan log file
579 580 581
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
582 583 584
 *
 * returns 0 on success, -1 on failure
 */
585
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
586
{
587
    struct darshan_fd_int_state *state = fd->state;
588
    struct darshan_name_record_ref *ref, *tmp;
589
    struct darshan_name_record *name_rec;
590
    int name_rec_len;
591
    int wrote;
592

593 594
    assert(state);

595
    /* allocate memory for largest possible hash record */
596
    name_rec = malloc(sizeof(darshan_record_id) + PATH_MAX + 1);
597
    if(!name_rec)
598
        return(-1);
599
    memset(name_rec, 0, sizeof(darshan_record_id) + PATH_MAX + 1);
600

601
    /* individually serialize each hash record and write to log file */
602 603
    HASH_ITER(hlink, hash, ref, tmp)
    {
604
        name_rec_len = sizeof(darshan_record_id) + strlen(ref->name_record->name) + 1;
605
        memcpy(name_rec, ref->name_record, name_rec_len);
606

607
        /* write this hash entry to log file */
608 609 610
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
611
        {
612
            state->err = -1;
613 614
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
615 616
            return(-1);
        }
617 618
    }

619
    free(name_rec);
620 621 622
    return(0);
}

623
/* darshan_log_get_mod()
624
 *
625 626
 * get a chunk of module data from the darshan log file
 *
627
 * returns number of bytes read on success, -1 on failure
628
 */
629
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
630
    void *mod_buf, int mod_buf_sz)
631
{
632
    struct darshan_fd_int_state *state = fd->state;
633
    int ret;
634

635 636
    assert(state);

637
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
638
    {
639 640
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
641 642
    }

643 644 645
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

646 647 648 649 650 651 652 653 654 655 656
    /* assume module will support backwards compatibility, but we obviously
     * can't provide any sort of "forwards" compatibility
     */
    if(fd->mod_ver[mod_id] > darshan_module_versions[mod_id])
    {
        fprintf(stderr, "Error: invalid %s module log format version "
                "(expected %d, got %d)\n", darshan_module_names[mod_id],
                darshan_module_versions[mod_id], fd->mod_ver[mod_id]);
        return(-1);
    }

657
    /* read this module's data from the log file */
658
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
659
    if(ret < 0)
660
    {
661
        fprintf(stderr,
662
            "Error: failed to read module %s data from darshan log file.\n",
663
            darshan_module_names[mod_id]);
664 665 666
        return(-1);
    }

667
    return(ret);
668 669
}

670
/* darshan_log_put_mod()
671 672
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
673 674 675 676 677 678
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
679
 *
680
 * returns number of bytes written on success, -1 on failure
681
 */
682
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
683
    void *mod_buf, int mod_buf_sz, int ver)
684
{
685
    struct darshan_fd_int_state *state = fd->state;
686 687
    int ret;

688 689
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
690
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
691
    {
692
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
693 694
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
695 696 697
    }

    /* write the module chunk to the log file */
698 699
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
700
    {
701
        state->err = -1;
702 703 704 705 706 707
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

708 709 710
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

711 712 713
    return(0);
}

714 715
/* darshan_log_close()
 *
716
 * close an open darshan file descriptor, freeing any resources
717 718 719
 *
 */
void darshan_log_close(darshan_fd fd)
720
{
721
    struct darshan_fd_int_state *state = fd->state;
722 723
    int ret;

724 725 726 727
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
728 729
    {
        /* flush the last region of the log to file */
730
        switch(fd->comp_type)
731 732
        {
            case DARSHAN_ZLIB_COMP:
733
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
734 735
                if(ret == 0)
                    break;
736 737
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
738
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
739 740 741
                if(ret == 0)
                    break;
#endif 
742 743
            default:
                /* if flush fails, remove the output log file */
744
                state->err = -1;
745 746 747 748 749
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
750
        if(state->err != -1)
751
        {
752
            ret = darshan_log_put_header(fd);
753
            if(ret < 0)
754
                state->err = -1;
755 756 757
        }
    }

758
    close(state->fildes);
759 760

    /* remove output log file if error writing to it */
761
    if((state->creat_flag) && (state->err == -1))
762 763
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
764 765
            state->logfile_path);
        unlink(state->logfile_path);
766
    }
767

768
    darshan_log_dzdestroy(fd);
769 770 771
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
772
    free(fd);
773 774

    return;
775 776
}

777 778 779 780 781
void darshan_log_print_version_warnings(const char *version_string)
{
    return;
}

782 783 784
/********************************************************
 *             internal helper functions                *
 ********************************************************/
785

786 787 788 789 790 791 792 793 794 795 796 797 798
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash)
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
                return(-1);

            ref->name_record = malloc(rec_len);
            if(!ref->name_record)
            {
                free(ref);
                return(-1);
            }

            /* copy the name record over from the hash buffer */
            memcpy(ref->name_record, name_rec, rec_len);

            /* add this record to the hash */
            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
        }

        tmp_p = (char *)name_rec + rec_len;
        name_rec = (struct darshan_name_record *)tmp_p;
        buf_len -= rec_len;
        buf_processed += rec_len;
    }

    return(buf_processed);
}

862 863 864 865 866
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
867
static int darshan_log_get_header(darshan_fd fd)
868 869 870 871 872 873 874 875 876 877 878 879
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

880 881 882 883 884 885 886 887 888
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
889 890 891 892
    if(strcmp(fd->version, "3.00") == 0)
    {
        fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
    }
893
    else if(strcmp(fd->version, "3.10") == 0)
894 895 896 897
    {
        fd->state->get_namerecs = darshan_log_get_namerecs;
    }
    else
898 899
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
900 901
        fprintf(stderr, "Error: expected version %s, but got %s\n",
            DARSHAN_LOG_VERSION, fd->version);
902 903 904 905 906 907 908 909 910 911 912
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

913 914
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
915
    if(ret != (int)sizeof(header))
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
935 936
            DARSHAN_BSWAP64(&(header.name_map.off));
            DARSHAN_BSWAP64(&(header.name_map.len));
937 938 939 940
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
941
                DARSHAN_BSWAP32(&(header.mod_ver[i]));
942 943 944 945 946 947 948 949 950 951
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

952
    /* set some fd fields based on what's stored in the header */
953
    fd->comp_type = header.comp_type;
954
    fd->partial_flag = header.partial_flag;
955
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
956 957

    /* save the mapping of data within log file to this file descriptor */
958
    memcpy(&fd->name_map, &(header.name_map), sizeof(struct darshan_log_map));
959 960
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

961 962
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
963
    if(fd->name_map.off == 0)
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
987
        fd->job_map.len = fd->name_map.off - fd->job_map.off;
988 989
    }

990 991 992 993 994 995 996
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
997
static int darshan_log_put_header(darshan_fd fd)
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
1012
    header.comp_type = fd->comp_type;
1013
    header.partial_flag = fd->partial_flag;
1014
    memcpy(&header.name_map, &fd->name_map, sizeof(struct darshan_log_map));
1015 1016
    memcpy(header.mod_map, fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
    memcpy(header.mod_ver, fd->mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
1017 1018 1019

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
1020
    if(ret != (int)sizeof(header))
1021 1022 1023 1024 1025 1026 1027 1028
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

1029 1030 1031 1032
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
1033
    struct darshan_fd_int_state *state = fd->state;
1034
    off_t ret_off;
1035

1036
    if(state->pos == offset)
1037 1038
        return(0);

1039
    ret_off = lseek(state->fildes, offset, SEEK_SET);
1040
    if(ret_off == offset)
1041
    {
1042
        state->pos = offset;
1043
        return(0);
1044 1045 1046 1047 1048
    }

    return(-1);
}

1049
/* return amount read on success, 0 on EOF, -1 on failure.
1050
 */
1051
static int darshan_log_read(darshan_fd fd, void* buf, int len)
1052
{
1053
    struct darshan_fd_int_state *state = fd->state;
1054
    int ret;
1055
    unsigned int read_so_far = 0;
1056

1057 1058 1059 1060 1061 1062 1063 1064 1065
    do
    {
        ret = read(state->fildes, buf + read_so_far, len - read_so_far);
        if(ret <= 0)
            break;
        read_so_far += ret;
    } while(read_so_far < len);
    if(ret < 0)
        return(-1);
1066

1067 1068
    state->pos += read_so_far;
    return(read_so_far);
1069 1070
}

1071
/* return amount written on success, -1 on failure.
1072
 */
1073
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1074
{
1075
    struct darshan_fd_int_state *state = fd->state;
1076
    int ret;
1077
    unsigned int wrote_so_far = 0;
1078

1079 1080 1081 1082 1083 1084 1085 1086 1087
    do
    {
        ret = write(state->fildes, buf + wrote_so_far, len - wrote_so_far);
        if(ret <= 0)
            break;
        wrote_so_far += ret;
    } while(wrote_so_far < len);
    if(ret < 0)
        return(-1);
1088

1089 1090
    state->pos += wrote_so_far;
    return(wrote_so_far);
1091 1092
}

1093
static int darshan_log_dzinit(darshan_fd fd)
1094
{
1095
    struct darshan_fd_int_state *state = fd->state;
1096 1097
    int ret;

1098 1099 1100
    /* initialize buffers for staging compressed data
     * to/from log file
     */
1101 1102
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1103
        return(-1);
1104
    state->dz.size = 0;
1105
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1106

1107
    switch(fd->comp_type)
1108 1109
    {
        case DARSHAN_ZLIB_COMP:
1110 1111 1112 1113
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1114
                free(state->dz.buf);
1115 1116 1117 1118 1119 1120 1121 1122 1123
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1124
            if(!(state->creat_flag))
1125 1126 1127 1128 1129 1130 1131 1132 1133
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1134
                tmp_zstrm->next_out = state->dz.buf;
1135 1136 1137 1138
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1139
                free(state->dz.buf);
1140 1141
                return(-1);
            }
1142
            state->dz.comp_dat = tmp_zstrm;
1143
            break;
1144
        }
1145 1146 1147 1148 1149 1150
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1151
                free(state->dz.buf);
1152 1153 1154 1155 1156 1157
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
1158
            tmp_bzstrm->next_in = NULL;
1159

1160
            if(!(state->creat_flag))
1161 1162 1163 1164 1165 1166 1167 1168 1169
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1170
                tmp_bzstrm->next_out = (char *)state->dz.buf;
1171 1172 1173 1174
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
1175
                free(state->dz.buf);
1176 1177
                return(-1);
            }
1178
            state->dz.comp_dat = tmp_bzstrm;
1179 1180 1181
            break;
        }
#endif
1182 1183 1184 1185 1186 1187 1188 1189
        case DARSHAN_NO_COMP:
        {
            /* we just track an offset into the staging buffers for no_comp */
            int *buf_off = malloc(sizeof(int));
            *buf_off = 0;
            state->dz.comp_dat = buf_off;
            break;
        }
1190 1191 1192 1193 1194 1195 1196 1197
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

1198
static void darshan_log_dzdestroy(darshan_fd fd)
1199
{
1200 1201 1202
    struct darshan_fd_int_state *state = fd->state;

    switch(fd->comp_type)
1203 1204
    {
        case DARSHAN_ZLIB_COMP:
1205
            if(!(state->creat_flag))
1206
                inflateEnd((z_stream *)state->dz.comp_dat);
1207
            else
1208
                deflateEnd((z_stream *)state->dz.comp_dat);
1209
            break;
1210 1211
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1212
            if(!(state->creat_flag))
1213
                BZ2_bzDecompressEnd((bz_stream *)state->dz.comp_dat);
1214
            else
1215
                BZ2_bzCompressEnd((bz_stream *)state->dz.comp_dat);
1216 1217
            break;
#endif
1218 1219 1220
        case DARSHAN_NO_COMP:
            /* do nothing */
            break;
1221 1222 1223 1224
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

1225
    free(state->dz.comp_dat);
1226
    free(state->dz.buf);
1227 1228 1229 1230 1231
    return;
}

static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len)
{
1232
    struct darshan_fd_int_state *state = fd->state;
1233 1234
    struct darshan_log_map map;
    int reset_strm_flag = 0;
1235 1236
    int ret;

1237 1238 1239 1240
    /* if new log region, we reload buffers and clear eor flag */
    if(region_id !=