darshan-logutils.c 52.7 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2 3 4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5 6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9 10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15 16 17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20 21
#include "darshan-logutils.h"

22 23 24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26 27 28 29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32 33
struct darshan_dz_state
{
34 35 36 37 38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39 40 41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43 44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;
64 65 66 67
    /* log format version-specific function calls for getting
     * data from the log file
     */
    int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
68

69
    /* compression/decompression stream read/write state */
70 71 72
    struct darshan_dz_state dz;
};

73 74 75 76 77 78 79 80 81
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c, d) d,
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
    DARSHAN_MODULE_IDS
};
#undef X

/* internal helper functions */
82
static int darshan_mnt_info_cmp(const void *a, const void *b);
83 84
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);
85 86
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
87 88
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
89
static int darshan_log_write(darshan_fd fd, void *buf, int len);
90 91
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
92 93
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
94 95 96 97
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
98
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
99
#ifdef HAVE_LIBBZ2
100 101 102 103
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
104 105
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
106 107
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
108 109
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
110

111 112 113 114 115 116 117 118
/* backwards compatibility functions */
int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash);


/********************************************************
 *        publically exposed logutil functions          *
 ********************************************************/
119 120 121

/* darshan_log_open()
 *
122
 * open an existing darshan log file for reading only
123
 *
124
 * returns file descriptor on success, NULL on failure
125
 */
126
darshan_fd darshan_log_open(const char *name)
127
{
128
    darshan_fd tmp_fd;
129
    int ret;
130

131
    /* allocate a darshan file descriptor */
132 133 134 135
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
136 137 138 139 140 141 142
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
143

144
    /* open the log file in read mode */
145 146
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
147
    {
148
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
149
        free(tmp_fd->state);
150 151 152
        free(tmp_fd);
        return(NULL);
    }
153
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
154 155

    /* read the header from the log file to init fd data structures */
156
    ret = darshan_log_get_header(tmp_fd);
157 158
    if(ret < 0)
    {
159
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
160 161
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
162
        free(tmp_fd);
163 164 165 166
        return(NULL);
    }

    /* initialize compression data structures */
167
    ret = darshan_log_dzinit(tmp_fd);
168 169 170
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
171 172
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
173 174
        free(tmp_fd);
        return(NULL);
175 176
    }

177 178 179 180 181 182 183
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
184
 * returns file descriptor on success, NULL on failure
185
 */
186 187
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
188 189
{
    darshan_fd tmp_fd;
190
    int ret;
191

192
    /* allocate a darshan file descriptor */
193 194 195 196
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
197 198 199 200 201 202 203
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
204
    tmp_fd->comp_type = comp_type;
205

206 207 208
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
209
    {
210
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
211
        free(tmp_fd->state);
212
        free(tmp_fd);
213
        return(NULL);
214
    }
215
    tmp_fd->state->creat_flag = 1;
216
    tmp_fd->partial_flag = partial_flag;
217
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
218

219 220 221 222 223
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
224
    if(ret < 0)
225 226
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
227 228
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
229 230 231
        free(tmp_fd);
        unlink(name);
        return(NULL);
232 233
    }

234
    /* initialize compression data structures */
235
    ret = darshan_log_dzinit(tmp_fd);
236 237
    if(ret < 0)
    {
238
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
239 240
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
241 242 243
        free(tmp_fd);
        unlink(name);
        return(NULL);
244 245
    }

246
    return(tmp_fd);
247 248
}

249
/* darshan_log_get_job()
250 251
 *
 * read job level metadata from the darshan log file
252
 *
253
 * returns 0 on success, -1 on failure
254
 */
255
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
256
{
257
    struct darshan_fd_int_state *state = fd->state;
258 259
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
260
    int ret;
261

262
    assert(state);
263 264
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

265
    /* read the compressed job data from the log file */
266
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
267
    if(ret <= (int)sizeof(*job))
268
    {
269
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
270 271
        return(-1);
    }
272 273

    memcpy(job, job_buf, sizeof(*job));
274

275
    if(fd->swap_flag)
276
    {
277 278 279 280 281 282
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
283 284
    }

285
    /* save trailing exe & mount information, so it can be retrieved later */
286 287 288
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
289
        return(-1);
290
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
291

292 293 294
    return(0);
}

295
/* darshan_log_put_job()
296
 *
297
 * write job level metadata to darshan log file
298 299 300
 *
 * returns 0 on success, -1 on failure
 */
301
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
302
{
303
    struct darshan_fd_int_state *state = fd->state;
304 305 306 307
    struct darshan_job job_copy;
    int len;
    int ret;

308 309
    assert(state);

310 311 312 313 314 315 316 317 318 319 320 321 322 323
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

324 325
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
326
    if(ret != sizeof(*job))
327
    {
328
        state->err = -1;
329 330 331 332 333 334 335
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

336
/* darshan_log_get_exe()
337 338 339 340 341
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
342
int darshan_log_get_exe(darshan_fd fd, char *buf)
343
{
344
    struct darshan_fd_int_state *state = fd->state;
345
    char *newline;
346
    int ret;
347

348 349
    assert(state);

350
    /* if the exe/mount data has not been saved yet, read in the job info */
351
    if(!(state->exe_mnt_data))
352
    {
353
        struct darshan_job job;
354
        ret = darshan_log_get_job(fd, &job);
355

356
        if(ret < 0 || !(state->exe_mnt_data))
357
            return(-1);
358
    }
359

360
    /* exe string is located before the first line break */
361
    newline = strchr(state->exe_mnt_data, '\n');
362 363

    /* copy over the exe string */
364
    if(newline)
365
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
366 367 368 369

    return (0);
}

370
/* darshan_log_put_exe()
371 372
 *
 * wrties the application exe name to darshan log file
373
 * NOTE: this needs to be called immediately following put_job as it
374
 * expects the file pointer to be positioned immediately following
375
 * the darshan job information
376 377 378
 *
 * returns 0 on success, -1 on failure 
 */
379
int darshan_log_put_exe(darshan_fd fd, char *buf)
380
{
381 382
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
383
    int ret;
384

385
    assert(fd->state);
386

387 388
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
389
    {
390
        state->err = -1;
391 392 393 394 395 396 397
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
/* darshan_log_get_mounts()
399
 * 
400 401 402
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
403 404
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
407
    int* count)
408
{
409
    struct darshan_fd_int_state *state = fd->state;
410 411
    char *pos;
    int array_index = 0;
412
    int ret;
413

414 415
    assert(state);

416
    /* if the exe/mount data has not been saved yet, read in the job info */
417
    if(!(state->exe_mnt_data))
418
    {
419
        struct darshan_job job;
420
        ret = darshan_log_get_job(fd, &job);
421

422
        if(ret < 0 || !(state->exe_mnt_data))
423
            return(-1);
424
    }
425

426
    /* count entries */
427
    *count = 0;
428
    pos = state->exe_mnt_data;
429 430 431 432 433 434 435 436 437 438 439 440 441
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
442 443
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
444

445
    /* work through the table and parse each line (except for
446 447
     * first, which holds command line information)
     */
448 449
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
450
    {
451 452
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
453
        if(ret != 2)
454
        {
455
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
456 457 458 459 460
            return(-1);
        }
        array_index++;
    }

461 462
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

463 464 465
    return(0);
}

466
/* darshan_log_put_mounts()
467 468 469 470 471 472 473 474
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
475
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
476
    int count)
477
{
478
    struct darshan_fd_int_state *state = fd->state;
479
    int i;
480
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
481
    int ret;
482 483
    int left = DARSHAN_EXE_LEN;
    int pos = 0;
484

485 486
    assert(state);

487 488 489
    /* write each mount entry to file */
    for(i=count-1; i>=0; i--)
    {
490 491 492 493 494 495 496 497 498 499 500
        if((strlen(mnt_data_array[i].mnt_type) + strlen(mnt_data_array[i].mnt_path) + 2) < left)
        {
            ret = snprintf(&mnt_dat[pos], left, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
            left -= ret;
            assert(left >= 0);
            pos += ret;
        }
        else
        {
            break;
        }
501
    }
502
    pos++;
503

504 505
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, pos);
    if (ret != pos)
506
    {
507
        state->err = -1;
508
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
509 510 511 512 513 514
        return(-1);
    }

    return(0);
}

515
/* darshan_log_get_namehash()
516
 *
517 518
 * read the set of name records from the darshan log file and add to the
 * given hash table
519 520 521
 *
 * returns 0 on success, -1 on failure
 */
522
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
523
{
524
    struct darshan_fd_int_state *state = fd->state;
525 526
    char *name_rec_buf;
    int name_rec_buf_sz;
527 528
    int read;
    int read_req_sz;
529 530
    int buf_len = 0;
    int buf_processed;
531

532 533
    assert(state);

534 535
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
536 537 538 539 540
    {
        *hash = NULL;
        return(0);
    }

541 542 543 544
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
545
        return(-1);
546
    memset(name_rec_buf, 0, name_rec_buf_sz);
547

548
    do
549
    {
550
        /* read chunks of the darshan record id -> name mapping from log file,
551
         * constructing a hash table in the process
552
         */
553
        read_req_sz = name_rec_buf_sz - buf_len;
554
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
555
            name_rec_buf + buf_len, read_req_sz);
556
        if(read < 0)
557
        {
558 559
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
560
            return(-1);
561
        }
562
        buf_len += read;
563

564 565
        /* extract any name records in the buffer */
        buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
566

567
        /* copy any leftover data to beginning of buffer to parse next */
568 569
        memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
        buf_len -= buf_processed;
570 571 572 573 574

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
575
    assert(buf_len == 0);
576

577
    free(name_rec_buf);
578 579 580
    return(0);
}

581
/* darshan_log_put_namehash()
582
 *
583
 * writes the hash table of name records to the darshan log file
584 585 586
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
587 588 589
 *
 * returns 0 on success, -1 on failure
 */
590
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
591
{
592
    struct darshan_fd_int_state *state = fd->state;
593
    struct darshan_name_record_ref *ref, *tmp;
594
    struct darshan_name_record *name_rec;
595
    int name_rec_len;
596
    int wrote;
597

598 599
    assert(state);

600
    /* allocate memory for largest possible hash record */
601
    name_rec = malloc(sizeof(darshan_record_id) + PATH_MAX + 1);
602
    if(!name_rec)
603
        return(-1);
604
    memset(name_rec, 0, sizeof(darshan_record_id) + PATH_MAX + 1);
605

606
    /* individually serialize each hash record and write to log file */
607 608
    HASH_ITER(hlink, hash, ref, tmp)
    {
609
        name_rec_len = sizeof(darshan_record_id) + strlen(ref->name_record->name) + 1;
610
        memcpy(name_rec, ref->name_record, name_rec_len);
611

612
        /* write this hash entry to log file */
613 614 615
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
616
        {
617
            state->err = -1;
618 619
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
620 621
            return(-1);
        }
622 623
    }

624
    free(name_rec);
625 626 627
    return(0);
}

628
/* darshan_log_get_mod()
629
 *
630 631
 * get a chunk of module data from the darshan log file
 *
632
 * returns number of bytes read on success, -1 on failure
633
 */
634
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
635
    void *mod_buf, int mod_buf_sz)
636
{
637
    struct darshan_fd_int_state *state = fd->state;
638
    int ret;
639

640 641
    assert(state);

642
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
643
    {
644 645
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
646 647
    }

648 649 650
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

651 652 653 654 655 656 657 658 659 660 661
    /* assume module will support backwards compatibility, but we obviously
     * can't provide any sort of "forwards" compatibility
     */
    if(fd->mod_ver[mod_id] > darshan_module_versions[mod_id])
    {
        fprintf(stderr, "Error: invalid %s module log format version "
                "(expected %d, got %d)\n", darshan_module_names[mod_id],
                darshan_module_versions[mod_id], fd->mod_ver[mod_id]);
        return(-1);
    }

662
    /* read this module's data from the log file */
663
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
664
    if(ret < 0)
665
    {
666
        fprintf(stderr,
667
            "Error: failed to read module %s data from darshan log file.\n",
668
            darshan_module_names[mod_id]);
669 670 671
        return(-1);
    }

672
    return(ret);
673 674
}

675
/* darshan_log_put_mod()
676 677
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
678 679 680 681 682 683
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
684
 *
685
 * returns number of bytes written on success, -1 on failure
686
 */
687
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
688
    void *mod_buf, int mod_buf_sz, int ver)
689
{
690
    struct darshan_fd_int_state *state = fd->state;
691 692
    int ret;

693 694
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
695
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
696
    {
697
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
698 699
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
700 701 702
    }

    /* write the module chunk to the log file */
703 704
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
705
    {
706
        state->err = -1;
707 708 709 710 711 712
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

713 714 715
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

716 717 718
    return(0);
}

719 720
/* darshan_log_close()
 *
721
 * close an open darshan file descriptor, freeing any resources
722 723 724
 *
 */
void darshan_log_close(darshan_fd fd)
725
{
726
    struct darshan_fd_int_state *state = fd->state;
727 728
    int ret;

729 730 731 732
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
733 734
    {
        /* flush the last region of the log to file */
735
        switch(fd->comp_type)
736 737
        {
            case DARSHAN_ZLIB_COMP:
738
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
739 740
                if(ret == 0)
                    break;
741 742
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
743
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
744 745 746
                if(ret == 0)
                    break;
#endif 
747 748
            default:
                /* if flush fails, remove the output log file */
749
                state->err = -1;
750 751 752 753 754
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
755
        if(state->err != -1)
756
        {
757
            ret = darshan_log_put_header(fd);
758
            if(ret < 0)
759
                state->err = -1;
760 761 762
        }
    }

763
    close(state->fildes);
764 765

    /* remove output log file if error writing to it */
766
    if((state->creat_flag) && (state->err == -1))
767 768
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
769 770
            state->logfile_path);
        unlink(state->logfile_path);
771
    }
772

773
    darshan_log_dzdestroy(fd);
774 775 776
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
777
    free(fd);
778 779

    return;
780 781
}

782 783 784 785 786
void darshan_log_print_version_warnings(const char *version_string)
{
    return;
}

787 788 789
/********************************************************
 *             internal helper functions                *
 ********************************************************/
790

791 792 793 794 795 796 797 798 799 800 801 802 803
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
    int swap_flag, struct darshan_name_record_ref **hash)
{
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
    char *tmp_p;
    int buf_processed = 0;
    int rec_len;

    /* work through the name record buffer -- deserialize the record data
     * and add to the output hash table
     * NOTE: these mapping pairs are variable in length, so we have to be able
     * to handle incomplete mappings temporarily here
     */
    name_rec = (struct darshan_name_record *)name_rec_buf;
    while(buf_len > sizeof(darshan_record_id) + 1)
    {
        if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
            (buf_len - sizeof(darshan_record_id)))
        {
            /* if this record name's terminating null character is not
             * present, we need to read more of the buffer before continuing
             */
            break;
        }
        rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;

        if(swap_flag)
        {
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(&(name_rec->id));
        }

        HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
                return(-1);

            ref->name_record = malloc(rec_len);
            if(!ref->name_record)
            {
                free(ref);
                return(-1);
            }

            /* copy the name record over from the hash buffer */
            memcpy(ref->name_record, name_rec, rec_len);

            /* add this record to the hash */
            HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
        }

        tmp_p = (char *)name_rec + rec_len;
        name_rec = (struct darshan_name_record *)tmp_p;
        buf_len -= rec_len;
        buf_processed += rec_len;
    }

    return(buf_processed);
}

867 868 869 870 871
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
872
static int darshan_log_get_header(darshan_fd fd)
873 874 875 876 877 878 879 880 881 882 883 884
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

885 886 887 888 889 890 891 892 893
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
894 895 896 897
    if(strcmp(fd->version, "3.00") == 0)
    {
        fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
    }
898
    else if(strcmp(fd->version, "3.10") == 0)
899 900 901 902
    {
        fd->state->get_namerecs = darshan_log_get_namerecs;
    }
    else
903 904
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
905 906
        fprintf(stderr, "Error: expected version %s, but got %s\n",
            DARSHAN_LOG_VERSION, fd->version);
907 908 909 910 911 912 913 914 915 916 917
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

918 919
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
920
    if(ret != (int)sizeof(header))
921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
940 941
            DARSHAN_BSWAP64(&(header.name_map.off));
            DARSHAN_BSWAP64(&(header.name_map.len));
942 943 944 945
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
946
                DARSHAN_BSWAP32(&(header.mod_ver[i]));
947 948 949 950 951 952 953 954 955 956
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

957
    /* set some fd fields based on what's stored in the header */
958
    fd->comp_type = header.comp_type;
959
    fd->partial_flag = header.partial_flag;
960
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
961 962

    /* save the mapping of data within log file to this file descriptor */
963
    memcpy(&fd->name_map, &(header.name_map), sizeof(struct darshan_log_map));
964 965
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

966 967
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
968
    if(fd->name_map.off == 0)
969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
992
        fd->job_map.len = fd->name_map.off - fd->job_map.off;
993 994
    }

995 996 997 998 999 1000 1001
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
1002
static int darshan_log_put_header(darshan_fd fd)
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
1017
    header.comp_type = fd->comp_type;
1018
    header.partial_flag = fd->partial_flag;
1019
    memcpy(&header.name_map, &fd->name_map, sizeof(struct darshan_log_map));
1020 1021
    memcpy(header.mod_map, fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
    memcpy(header.mod_ver, fd->mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
1022 1023 1024

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
1025
    if(ret != (int)sizeof(header))
1026 1027 1028 1029 1030 1031 1032 1033
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

1034 1035 1036 1037
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
1038
    struct darshan_fd_int_state *state = fd->state;
1039
    off_t ret_off;
1040

1041
    if(state->pos == offset)
1042 1043
        return(0);

1044
    ret_off = lseek(state->fildes, offset, SEEK_SET);
1045
    if(ret_off == offset)
1046
    {
1047
        state->pos = offset;
1048
        return(0);
1049 1050 1051 1052 1053
    }

    return(-1);
}

1054
/* return amount read on success, 0 on EOF, -1 on failure.
1055
 */
1056
static int darshan_log_read(darshan_fd fd, void* buf, int len)
1057
{
1058
    struct darshan_fd_int_state *state = fd->state;
1059
    int ret;
1060
    unsigned int read_so_far = 0;
1061

1062 1063 1064 1065 1066 1067 1068 1069 1070
    do
    {
        ret = read(state->fildes, buf + read_so_far, len - read_so_far);
        if(ret <= 0)
            break;
        read_so_far += ret;
    } while(read_so_far < len);
    if(ret < 0)
        return(-1);
1071

1072 1073
    state->pos += read_so_far;
    return(read_so_far);
1074 1075
}

1076
/* return amount written on success, -1 on failure.
1077
 */
1078
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1079
{
1080
    struct darshan_fd_int_state *state = fd->state;
1081
    int ret;
1082
    unsigned int wrote_so_far = 0;
1083

1084 1085 1086 1087 1088 1089 1090 1091 1092
    do
    {
        ret = write(state->fildes, buf + wrote_so_far, len - wrote_so_far);
        if(ret <= 0)
            break;
        wrote_so_far += ret;
    } while(wrote_so_far < len);
    if(ret < 0)
        return(-1);
1093

1094 1095
    state->pos += wrote_so_far;
    return(wrote_so_far);
1096 1097
}

1098
static int darshan_log_dzinit(darshan_fd fd)
1099
{
1100
    struct darshan_fd_int_state *state = fd->state;
1101 1102
    int ret;

1103 1104 1105
    /* initialize buffers for staging compressed data
     * to/from log file
     */
1106 1107
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1108
        return(-1);
1109
    state->dz.size = 0;
1110
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1111

1112
    switch(fd->comp_type)
1113 1114
    {
        case DARSHAN_ZLIB_COMP:
1115 1116 1117 1118
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1119
                free(state->dz.buf);
1120 1121 1122 1123 1124 1125 1126 1127 1128
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1129
            if(!(state->creat_flag))
1130 1131 1132 1133 1134 1135 1136 1137 1138
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1139
                tmp_zstrm->next_out = state->dz.buf;
1140 1141 1142 1143
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1144
                free(state->dz.buf);
1145 1146
                return(-1);
            }
1147
            state->dz.comp_dat = tmp_zstrm;
1148
            break;
1149
        }
1150 1151 1152 1153 1154 1155
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1156
                free(state->dz.buf);
1157 1158 1159 1160 1161 1162
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
1163
            tmp_bzstrm->next_in = NULL;
1164

1165
            if(!(state->creat_flag))
1166 1167 1168 1169 1170 1171 1172 1173 1174
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */
                ret = BZ2_bzCompressInit(tmp_bzstrm, 9, 1, 30);
                tmp_bzstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1175
                tmp_bzstrm->next_out = (char *)state->dz.buf;
1176 1177 1178 1179
            }
            if(ret != BZ_OK)
            {
                free(tmp_bzstrm);
1180
                free(state->dz.buf);
1181 1182
                return(-1);
            }
1183
            state->dz.comp_dat = tmp_bzstrm;
1184 1185 1186
            break;
        }
#endif
1187 1188 1189 1190 1191 1192 1193 1194
        case DARSHAN_NO_COMP:
        {
            /* we just track an offset into the staging buffers for no_comp */
            int *buf_off = malloc(sizeof(int));
            *buf_off = 0;
            state->dz.comp_dat = buf_off;
            break;
        }
1195 1196 1197 1198 1199 1200 1201 1202
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
            return(-1);
    }

    return(0);
}

1203
static void darshan_log_dzdestroy(darshan_fd fd)
1204
{
1205 1206 1207
    struct darshan_fd_int_state *state = fd->state;

    switch(fd->comp_type)
1208 1209
    {
        case DARSHAN_ZLIB_COMP:
1210
            if(!(state->creat_flag))
1211
                inflateEnd((z_stream *)state->dz.comp_dat);
1212
            else
1213
                deflateEnd((z_stream *)state->dz.comp_dat);
1214
            break;
1215 1216
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
1217
            if(!(state->creat_flag))
1218
                BZ2_bzDecompressEnd((bz_stream *)state->dz.comp_dat);
1219
            else
1220
                BZ2_bzCompressEnd((bz_stream *)state->dz.comp_dat);
1221 1222
            break;
#endif
1223 1224 1225
        case DARSHAN_NO_COMP:
            /* do nothing */
            break;
1226 1227 1228 1229
        default:
            fprintf(stderr, "Error: invalid compression type.\n");
    }

1230
    free(state->dz.comp_dat);
1231
    free(state->dz.buf);