darshan-logutils.c 48.2 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
30
31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32
33
struct darshan_dz_state
{
34
35
36
37
38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39
40
41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43
44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

65
    /* compression/decompression stream read/write state */
66
67
68
    struct darshan_dz_state dz;
};

69
70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71
72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74
75
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
76
77
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
78
79
80
81
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
82
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
83
#ifdef HAVE_LIBBZ2
84
85
86
87
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
88
89
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
90
91
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
92
93
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
94

Shane Snyder's avatar
Shane Snyder committed
95
/* each module's implementation of the darshan logutil functions */
96
#define X(a, b, c, d) d,
97
98
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
99
    DARSHAN_MODULE_IDS
100
};
Shane Snyder's avatar
Shane Snyder committed
101
#undef X
102
103
104

/* darshan_log_open()
 *
105
 * open an existing darshan log file for reading only
106
 *
107
 * returns file descriptor on success, NULL on failure
108
 */
109
darshan_fd darshan_log_open(const char *name)
110
{
111
    darshan_fd tmp_fd;
112
    int ret;
113

114
    /* allocate a darshan file descriptor */
115
116
117
118
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
119
120
121
122
123
124
125
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
126

127
    /* open the log file in read mode */
128
129
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
130
    {
131
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
132
        free(tmp_fd->state);
133
134
135
        free(tmp_fd);
        return(NULL);
    }
136
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
137
138
139
140
141

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
142
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
143
144
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
145
        free(tmp_fd);
146
147
148
149
        return(NULL);
    }

    /* initialize compression data structures */
150
    ret = darshan_log_dzinit(tmp_fd);
151
152
153
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
154
155
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
156
157
        free(tmp_fd);
        return(NULL);
158
159
    }

160
161
162
163
164
165
166
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
167
 * returns file descriptor on success, NULL on failure
168
 */
169
170
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
171
172
{
    darshan_fd tmp_fd;
173
    int ret;
174

175
    /* allocate a darshan file descriptor */
176
177
178
179
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
180
181
182
183
184
185
186
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
187
    tmp_fd->comp_type = comp_type;
188

189
190
191
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
192
    {
193
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
194
        free(tmp_fd->state);
195
        free(tmp_fd);
196
        return(NULL);
197
    }
198
    tmp_fd->state->creat_flag = 1;
199
    tmp_fd->partial_flag = partial_flag;
200
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
201

202
203
204
205
206
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
207
    if(ret < 0)
208
209
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
210
211
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
212
213
214
        free(tmp_fd);
        unlink(name);
        return(NULL);
215
216
    }

217
    /* initialize compression data structures */
218
    ret = darshan_log_dzinit(tmp_fd);
219
220
    if(ret < 0)
    {
221
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
222
223
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
224
225
226
        free(tmp_fd);
        unlink(name);
        return(NULL);
227
228
    }

229
    return(tmp_fd);
230
231
}

232
/* darshan_log_getjob()
233
234
 *
 * read job level metadata from the darshan log file
235
 *
236
 * returns 0 on success, -1 on failure
237
 */
238
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
239
{
240
    struct darshan_fd_int_state *state = fd->state;
241
242
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
243
    int ret;
244

245
    assert(state);
246
247
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

248
    /* read the compressed job data from the log file */
249
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
250
    if(ret <= (int)sizeof(*job))
251
    {
252
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
253
254
        return(-1);
    }
255
256

    memcpy(job, job_buf, sizeof(*job));
257

258
    if(fd->swap_flag)
259
    {
260
261
262
263
264
265
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
266
267
    }

268
    /* save trailing exe & mount information, so it can be retrieved later */
269
270
271
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
272
        return(-1);
273
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
274

275
276
277
    return(0);
}

278
279
/* darshan_log_putjob()
 *
280
 * write job level metadata to darshan log file
281
282
283
284
285
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
286
    struct darshan_fd_int_state *state = fd->state;
287
288
289
290
    struct darshan_job job_copy;
    int len;
    int ret;

291
292
    assert(state);

293
294
295
296
297
298
299
300
301
302
303
304
305
306
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

307
308
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
309
    if(ret != sizeof(*job))
310
    {
311
        state->err = -1;
312
313
314
315
316
317
318
319
320
321
322
323
324
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
325
326
int darshan_log_getexe(darshan_fd fd, char *buf)
{
327
    struct darshan_fd_int_state *state = fd->state;
328
    char *newline;
329
    int ret;
330

331
332
    assert(state);

333
    /* if the exe/mount data has not been saved yet, read in the job info */
334
    if(!(state->exe_mnt_data))
335
    {
336
337
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
338

339
        if(ret < 0 || !(state->exe_mnt_data))
340
            return(-1);
341
    }
342

343
    /* exe string is located before the first line break */
344
    newline = strchr(state->exe_mnt_data, '\n');
345
346

    /* copy over the exe string */
347
    if(newline)
348
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
349
350
351
352

    return (0);
}

353
354
355
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
356
 * NOTE: this needs to be called immediately following put_job as it
357
 * expects the file pointer to be positioned immediately following
358
 * the darshan job information
359
360
361
362
363
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
364
365
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
366
    int ret;
367

368
    assert(fd->state);
369

370
371
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
372
    {
373
        state->err = -1;
374
375
376
377
378
379
380
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

381
382
/* darshan_log_getmounts()
 * 
383
 * retrieves mount table information from the log. Note that mnt_pts and
384
 * fs_types are arrays that will be allocated by the function and must be
385
386
387
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
388
 */
389
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
390
391
    char*** fs_types, int* count)
{
392
    struct darshan_fd_int_state *state = fd->state;
393
394
    char *pos;
    int array_index = 0;
395
    int ret;
396

397
398
    assert(state);

399
    /* if the exe/mount data has not been saved yet, read in the job info */
400
    if(!(state->exe_mnt_data))
401
    {
402
403
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
404

405
        if(ret < 0 || !(state->exe_mnt_data))
406
            return(-1);
407
    }
408

409
    /* count entries */
410
    *count = 0;
411
    pos = state->exe_mnt_data;
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
433
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
434
435
    {
        /* overestimate string lengths */
436
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
437
        assert((*mnt_pts)[array_index]);
438
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
439
440
        assert((*fs_types)[array_index]);

441
442
443
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
444
        {
445
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
446
447
448
449
450
451
452
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

453
454
455
    return(0);
}

456
457
458
459
460
461
462
463
464
465
466
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
467
    struct darshan_fd_int_state *state = fd->state;
468
469
    int i;
    char line[1024];
470
471
472
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
473
474
    int ret;

475
476
    assert(state);

477
    /* write each mount entry to file */
478
    tmp = mnt_dat;
479
480
481
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
482
483
484
485
486
487

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

488
489
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
490
    {
491
        state->err = -1;
492
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
493
494
495
496
497
498
        return(-1);
    }

    return(0);
}

499
/* darshan_log_gethash()
500
 *
501
 * read the hash of records from the darshan log file
502
503
504
 *
 * returns 0 on success, -1 on failure
 */
505
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
506
{
507
    struct darshan_fd_int_state *state = fd->state;
508
    char *hash_buf;
509
    int hash_buf_sz;
510
    char *buf_ptr;
511
512
    darshan_record_id *rec_id_ptr;
    char *path_ptr;
513
    char *tmp_p;
514
    struct darshan_record_ref *ref;
515
516
    int read;
    int read_req_sz;
517
    int buf_rem = 0;
518

519
520
    assert(state);

521
522
523
524
525
526
527
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

528
529
530
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
531
        return(-1);
532
533
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
534

535
    do
536
    {
537
538
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
539
         */
540
        read_req_sz = hash_buf_sz - buf_rem;
541
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
542
            hash_buf + buf_rem, read_req_sz);
543
        if(read < 0)
544
        {
545
546
547
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
548
549
        }

550
551
552
553
554
555
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
556
557
        buf_rem += read;
        while(buf_rem > (sizeof(darshan_record_id) + 1))
558
        {
559
560
561
562
563
564
565
566
567
            tmp_p = buf_ptr + sizeof(darshan_record_id);
            while(tmp_p < (buf_ptr + buf_rem))
            {
                /* look for terminating null character for record name */
                if(*tmp_p == '\0')
                    break;
                tmp_p++;
            }
            if(*tmp_p != '\0')
568
569
570
571
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
572
             *          ... darshan_record_id | path '\0' ...
573
574
575
576
577
578
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
579
            {
580
581
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
582
            }
583
584
585

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
586
            {
587
588
589
590
591
592
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
593
594
                ref->name = malloc(strlen(path_ptr) + 1);
                if(!ref->name)
595
596
597
598
599
600
601
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
602
603
                ref->id = *rec_id_ptr;
                strcpy(ref->name, path_ptr);
604
605

                /* add this record to the hash */
606
                HASH_ADD(hlink, *hash, id, sizeof(darshan_record_id), ref);
607
608
            }

609
610
            buf_ptr += strlen(path_ptr) + 1;
            buf_rem -= (sizeof(darshan_record_id) + strlen(path_ptr) + 1);
611
        }
612

613
        /* copy any leftover data to beginning of buffer to parse next */
614
        memcpy(hash_buf, buf_ptr, buf_rem);
615
616
617
618
619

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
620
    assert(buf_rem == 0);
621

622
    free(hash_buf);
623
624
625
    return(0);
}

626
627
628
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
629
630
631
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
632
633
634
635
636
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
637
    struct darshan_fd_int_state *state = fd->state;
638
    char *hash_buf;
639
    int hash_buf_sz;
640
    struct darshan_record_ref *ref, *tmp;
641
642
    char *buf_ptr;
    int wrote;
643

644
645
    assert(state);

646
    /* allocate memory for largest possible hash record */
647
    hash_buf_sz = sizeof(darshan_record_id) + PATH_MAX + 1;
648
649
650
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
651
    memset(hash_buf, 0, hash_buf_sz);
652

653
    /* individually serialize each hash record and write to log file */
654
655
    HASH_ITER(hlink, hash, ref, tmp)
    {
656
        buf_ptr = hash_buf;
657

658
        /* the hash buffer has space to serialize this record
659
         * NOTE: darshan record hash serialization method: 
660
         *          ... darshan_record_id | path '\0' ...
661
         */
662
        *((darshan_record_id *)buf_ptr) = ref->id;
663
        buf_ptr += sizeof(darshan_record_id);
664
665
        strcpy(buf_ptr, ref->name);
        buf_ptr += strlen(ref->name) + 1;
666

667
668
669
670
671
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
672
            state->err = -1;
673
674
675
676
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
677
678
679
680
681
682
683
684
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
685
686
 * get a chunk of module data from the darshan log file
 *
687
 * returns number of bytes read on success, -1 on failure
688
689
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
690
    void *mod_buf, int mod_buf_sz)
691
{
692
    struct darshan_fd_int_state *state = fd->state;
693
    int ret;
694

695
696
    assert(state);

697
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
698
    {
699
700
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
701
702
    }

703
704
705
706
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
707
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
708
    if(ret < 0)
709
    {
710
        fprintf(stderr,
711
            "Error: failed to read module %s data from darshan log file.\n",
712
            darshan_module_names[mod_id]);
713
714
715
        return(-1);
    }

716
    return(ret);
717
718
}

719
720
721
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
722
723
724
725
726
727
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
728
 *
729
 * returns number of bytes written on success, -1 on failure
730
731
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
732
    void *mod_buf, int mod_buf_sz, int ver)
733
{
734
    struct darshan_fd_int_state *state = fd->state;
735
736
    int ret;

737
738
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
739
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
740
    {
741
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
742
743
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
744
745
746
    }

    /* write the module chunk to the log file */
747
748
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
749
    {
750
        state->err = -1;
751
752
753
754
755
756
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

757
758
759
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

760
761
762
    return(0);
}

763
764
/* darshan_log_close()
 *
765
 * close an open darshan file descriptor, freeing any resources
766
767
768
 *
 */
void darshan_log_close(darshan_fd fd)
769
{
770
    struct darshan_fd_int_state *state = fd->state;
771
772
    int ret;

773
774
775
776
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
777
778
    {
        /* flush the last region of the log to file */
779
        switch(fd->comp_type)
780
781
        {
            case DARSHAN_ZLIB_COMP:
782
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
783
784
                if(ret == 0)
                    break;
785
786
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
787
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
788
789
790
                if(ret == 0)
                    break;
#endif 
791
792
            default:
                /* if flush fails, remove the output log file */
793
                state->err = -1;
794
795
796
797
798
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
799
        if(state->err != -1)
800
801
802
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
803
                state->err = -1;
804
805
806
        }
    }

807
    close(state->fildes);
808
809

    /* remove output log file if error writing to it */
810
    if((state->creat_flag) && (state->err == -1))
811
812
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
813
814
            state->logfile_path);
        unlink(state->logfile_path);
815
    }
816

817
    darshan_log_dzdestroy(fd);
818
819
820
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
821
    free(fd);
822
823

    return;
824
825
}

826
/* **************************************************** */
827

828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
    if(strcmp(fd->version, "3.00"))
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
        fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

870
871
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
872
    if(ret != (int)sizeof(header))
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

908
    /* set some fd fields based on what's stored in the header */
909
    fd->comp_type = header.comp_type;
910
    fd->partial_flag = header.partial_flag;
911
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
912
913
914
915
916

    /* save the mapping of data within log file to this file descriptor */
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
    if(fd->rec_map.off == 0)
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
        fd->job_map.len = fd->rec_map.off - fd->job_map.off;
    }

946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
968
    header.comp_type = fd->comp_type;
969
    header.partial_flag = fd->partial_flag;
970
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
971
972
    memcpy(header.mod_map, fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
    memcpy(header.mod_ver, fd->mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
973
974
975

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
976
    if(ret != (int)sizeof(header))
977
978
979
980
981
982
983
984
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

985
986
987
988
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
989
    struct darshan_fd_int_state *state = fd->state;
990
    off_t ret_off;
991

992
    if(state->pos == offset)
993
994
        return(0);

995
    ret_off = lseek(state->fildes, offset, SEEK_SET);
996
    if(ret_off == offset)
997
    {
998
        state->pos = offset;
999
        return(0);
1000
1001
1002
1003
1004
    }

    return(-1);
}

1005
/* return amount read on success, 0 on EOF, -1 on failure.
1006
 */
1007
static int dar