darshan-logutils.c 45.7 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
30
31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
struct darshan_dz_state
{
    /* (libz/bzip2) stream data structure for managing
     * compression and decompression state */
    void *strm;
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
    int size;
    /* for reading logs, flag indicating end of log file region */
    int eor;
    /* the region we last tried reading/writing */
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* compression type used on log file (libz or bzip2) */
    enum darshan_comp_type comp_type;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

    /* compression/decompression state */
    struct darshan_dz_state dz;
};

69
70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71
72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74
75
static int darshan_log_dzinit(struct darshan_fd_int_state *state);
static void darshan_log_dzdestroy(struct darshan_fd_int_state *state);
76
77
78
79
80
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
81
82
83
84
85
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
86
87
88
89
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

/* TODO: check comments on functions to make sure they are right /cleanup */
90
91

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
92
/* see gzip interface for ideas */
93
94
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
95
    NULL,               /* NULL */
96
    &posix_logutils,    /* POSIX */
97
98
99
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
100
    &bgq_logutils,      /* BG/Q */
101
102
103
104
105
106
107
108
109
110
111
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
112
113
114

/* darshan_log_open()
 *
115
 * open an existing darshan log file for reading only
116
 *
117
 * returns file descriptor on success, NULL on failure
118
 */
119
darshan_fd darshan_log_open(const char *name)
120
{
121
    darshan_fd tmp_fd;
122
    int ret;
123

124
    /* allocate a darshan file descriptor */
125
126
127
128
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
129
130
131
132
133
134
135
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
136

137
    /* open the log file in read mode */
138
139
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
140
    {
141
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
142
        free(tmp_fd->state);
143
144
145
        free(tmp_fd);
        return(NULL);
    }
146
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
147
148
149
150
151

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
152
153
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
154
        free(tmp_fd);
155
156
157
158
        return(NULL);
    }

    /* initialize compression data structures */
159
    ret = darshan_log_dzinit(tmp_fd->state);
160
161
162
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
163
164
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
165
166
        free(tmp_fd);
        return(NULL);
167
168
    }

169
170
171
172
173
174
175
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
176
 * returns file descriptor on success, NULL on failure
177
178
179
180
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;
181
    int ret;
182

183
    /* allocate a darshan file descriptor */
184
185
186
187
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
188
189
190
191
192
193
194
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
195

196
197
198
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
199
    {
200
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
201
        free(tmp_fd->state);
202
        free(tmp_fd);
203
        return(NULL);
204
    }
205
206
207
    tmp_fd->state->creat_flag = 1;
    tmp_fd->state->comp_type = comp_type;
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
208

209
210
211
212
213
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
214
    if(ret < 0)
215
216
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
217
218
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
219
220
221
        free(tmp_fd);
        unlink(name);
        return(NULL);
222
223
    }

224
    /* initialize compression data structures */
225
    ret = darshan_log_dzinit(tmp_fd->state);
226
227
    if(ret < 0)
    {
228
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
229
230
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
231
232
233
        free(tmp_fd);
        unlink(name);
        return(NULL);
234
235
    }

236
    return(tmp_fd);
237
238
}

239
/* darshan_log_getjob()
240
241
 *
 * read job level metadata from the darshan log file
242
 *
243
 * returns 0 on success, -1 on failure
244
 */
245
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
246
{
247
    struct darshan_fd_int_state *state = fd->state;
248
249
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
250
    int ret;
251

252
    assert(state);
253
254
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

255
    /* read the compressed job data from the log file */
256
257
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
258
    {
259
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
260
261
        return(-1);
    }
262
263

    memcpy(job, job_buf, sizeof(*job));
264

265
    if(fd->swap_flag)
266
    {
267
268
269
270
271
272
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
273
274
    }

275
    /* save trailing exe & mount information, so it can be retrieved later */
276
277
278
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
279
        return(-1);
280
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
281

282
283
284
    return(0);
}

285
286
/* darshan_log_putjob()
 *
287
 * write job level metadata to darshan log file
288
289
290
291
292
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
293
    struct darshan_fd_int_state *state = fd->state;
294
295
296
297
    struct darshan_job job_copy;
    int len;
    int ret;

298
299
    assert(state);

300
301
302
303
304
305
306
307
308
309
310
311
312
313
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

314
315
316
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
317
    {
318
        state->err = -1;
319
320
321
322
323
324
325
326
327
328
329
330
331
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
332
333
int darshan_log_getexe(darshan_fd fd, char *buf)
{
334
    struct darshan_fd_int_state *state = fd->state;
335
    char *newline;
336
    int ret;
337

338
339
    assert(state);

340
    /* if the exe/mount data has not been saved yet, read in the job info */
341
    if(!(state->exe_mnt_data))
342
    {
343
344
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
345

346
        if(ret < 0 || !(state->exe_mnt_data))
347
            return(-1);
348
    }
349

350
    /* exe string is located before the first line break */
351
    newline = strchr(state->exe_mnt_data, '\n');
352
353

    /* copy over the exe string */
354
    if(newline)
355
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
356
357
358
359

    return (0);
}

360
361
362
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
363
 * NOTE: this needs to be called immediately following put_job as it
364
 * expects the file pointer to be positioned immediately following
365
 * the darshan job information
366
367
368
369
370
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
371
372
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
373
    int ret;
374

375
    assert(fd->state);
376

377
378
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
379
    {
380
        state->err = -1;
381
382
383
384
385
386
387
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

388
389
/* darshan_log_getmounts()
 * 
390
 * retrieves mount table information from the log. Note that mnt_pts and
391
 * fs_types are arrays that will be allocated by the function and must be
392
393
394
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
395
 */
396
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
397
398
    char*** fs_types, int* count)
{
399
    struct darshan_fd_int_state *state = fd->state;
400
401
    char *pos;
    int array_index = 0;
402
    int ret;
403

404
405
    assert(state);

406
    /* if the exe/mount data has not been saved yet, read in the job info */
407
    if(!(state->exe_mnt_data))
408
    {
409
410
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
411

412
        if(ret < 0 || !(state->exe_mnt_data))
413
            return(-1);
414
    }
415

416
    /* count entries */
417
    *count = 0;
418
    pos = state->exe_mnt_data;
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
440
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
441
442
    {
        /* overestimate string lengths */
443
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
444
        assert((*mnt_pts)[array_index]);
445
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
446
447
        assert((*fs_types)[array_index]);

448
449
450
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
451
        {
452
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
453
454
455
456
457
458
459
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

460
461
462
    return(0);
}

463
464
465
466
467
468
469
470
471
472
473
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
474
    struct darshan_fd_int_state *state = fd->state;
475
476
    int i;
    char line[1024];
477
478
479
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
480
481
    int ret;

482
483
    assert(state);

484
    /* write each mount entry to file */
485
    tmp = mnt_dat;
486
487
488
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
489
490
491
492
493
494

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

495
496
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
497
    {
498
        state->err = -1;
499
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
500
501
502
503
504
505
        return(-1);
    }

    return(0);
}

506
/* darshan_log_gethash()
507
 *
508
 * read the hash of records from the darshan log file
509
510
511
 *
 * returns 0 on success, -1 on failure
 */
512
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
513
{
514
    struct darshan_fd_int_state *state = fd->state;
515
    char *hash_buf;
516
    int hash_buf_sz;
517
    char *buf_ptr;
518
    darshan_record_id *rec_id_ptr;
519
    uint32_t *path_len_ptr, tmp_path_len;
520
521
    char *path_ptr;
    struct darshan_record_ref *ref;
522
523
524
    int read;
    int read_req_sz;
    int buf_remaining = 0;
525

526
527
    assert(state);

528
529
530
531
532
533
534
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

535
536
537
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
538
        return(-1);
539
540
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
541

542
    do
543
    {
544
545
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
546
         */
547
548
549
550
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
551
        {
552
553
554
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
555
556
        }

557
558
559
560
561
562
563
564
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
565
        {
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
587
            {
588
589
590
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
591
            }
592
593
594

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
595
            {
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
617
618
            }

619
620
621
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
622
        }
623

624
625
626
627
628
629
630
631
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
632

633
    free(hash_buf);
634
635
636
    return(0);
}

637
638
639
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
640
641
642
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
643
644
645
646
647
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
648
    struct darshan_fd_int_state *state = fd->state;
649
    char *hash_buf;
650
    int hash_buf_sz;
651
    struct darshan_record_ref *ref, *tmp;
652
653
654
    char *buf_ptr;
    int path_len;
    int wrote;
655

656
657
    assert(state);

658
659
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
660
661
662
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
663
    memset(hash_buf, 0, hash_buf_sz);
664

665
    /* individually serialize each hash record and write to log file */
666
667
    HASH_ITER(hlink, hash, ref, tmp)
    {
668
669
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
670

671
        /* the hash buffer has space to serialize this record
672
673
674
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
675
676
677
678
679
680
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
681

682
683
684
685
686
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
687
            state->err = -1;
688
689
690
691
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
692
693
694
695
696
697
698
699
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
700
701
702
 * get a chunk of module data from the darshan log file
 *
 * returns 0 on success, -1 on failure
703
704
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
705
    void *buf, int len)
706
{
707
    struct darshan_fd_int_state *state = fd->state;
708
    int ret;
709

710
711
    assert(state);

712
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
713
    {
714
715
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
716
717
    }

718
719
720
721
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
722
723
    ret = darshan_log_dzread(fd, mod_id, buf, len);
    if(ret < 0)
724
    {
725
        fprintf(stderr,
726
            "Error: failed to read module %s data from darshan log file.\n",
727
            darshan_module_names[mod_id]);
728
729
730
        return(-1);
    }

731
    return(ret);
732
733
}

734
735
736
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
737
738
739
740
741
742
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
743
744
745
746
747
748
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
749
    struct darshan_fd_int_state *state = fd->state;
750
751
    int ret;

752
753
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
754
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
755
    {
756
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
757
758
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
759
760
761
    }

    /* write the module chunk to the log file */
762
763
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
764
    {
765
        state->err = -1;
766
767
768
769
770
771
772
773
774
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

775
776
/* darshan_log_close()
 *
777
 * close an open darshan file descriptor, freeing any resources
778
779
780
781
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
782
{
783
    struct darshan_fd_int_state *state = fd->state;
784
785
    int ret;

786
787
788
789
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
790
791
    {
        /* flush the last region of the log to file */
792
        switch(state->comp_type)
793
794
        {
            case DARSHAN_ZLIB_COMP:
795
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
796
797
                if(ret == 0)
                    break;
798
799
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
800
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
801
802
803
                if(ret == 0)
                    break;
#endif 
804
805
            default:
                /* if flush fails, remove the output log file */
806
                state->err = -1;
807
808
809
810
811
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
812
        if(state->err != -1)
813
814
815
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
816
                state->err = -1;
817
818
819
        }
    }

820
    close(state->fildes);
821
822

    /* remove output log file if error writing to it */
823
    if((state->creat_flag) && (state->err == -1))
824
825
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
826
827
            state->logfile_path);
        unlink(state->logfile_path);
828
    }
829

830
831
832
833
    darshan_log_dzdestroy(state);
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
834
    free(fd);
835
836

    return;
837
838
}

839
/* **************************************************** */
840

841
842
843
844
845
846
847
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
848
    struct darshan_fd_int_state *state = fd->state;
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

901
    state->comp_type = header.comp_type;
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
918
    struct darshan_fd_int_state *state = fd->state;
919
920
921
922
923
924
925
926
927
928
929
930
931
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
932
    header.comp_type = state->comp_type;
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

949
950
951
952
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
953
    struct darshan_fd_int_state *state = fd->state;
954
    off_t ret_off;
955

956
    if(state->pos == offset)
957
958
        return(0);

959
    ret_off = lseek(state->fildes, offset, SEEK_SET);
960
    if(ret_off == offset)
961
    {
962
        state->pos = offset;
963
        return(0);
964
965
966
967
968
    }

    return(-1);
}

969
/* return amount read on success, 0 on EOF, -1 on failure.
970
 */
971
static int darshan_log_read(darshan_fd fd, void* buf, int len)
972
{
973
    struct darshan_fd_int_state *state = fd->state;
974
975
    int ret;

976
    /* read data from the log file using the given map */
977
    ret = read(state->fildes, buf, len);
978
    if(ret > 0)
979
        state->pos += ret;
980

981
    return(ret);
982
983
}

984
/* return amount written on success, -1 on failure.
985
 */
986
static int darshan_log_write(darshan_fd fd, void* buf, int len)
987
{
988
    struct darshan_fd_int_state *state = fd->state;
989
990
    int ret;

991
    ret = write(state->fildes, buf, len);
992
    if(ret > 0)
993
        state->pos += ret;
994
995
996
997

    return(ret);
}

998
static int darshan_log_dzinit(struct darshan_fd_int_state *state)
999
1000
1001
{
    int ret;

1002
    /* initialize buffers for staging compressed data to/from log file */
1003
1004
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1005
1006
        return(-1);

1007
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1008

1009
    switch(state->comp_type)
1010
1011
    {
        case DARSHAN_ZLIB_COMP:
1012
1013
1014
1015
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1016
                free(state->dz.buf);
1017
1018
1019
1020
1021
1022
1023
1024
1025
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1026
            if(!(state->creat_flag))
1027
1028
1029
1030
1031
1032
1033
1034
1035
            {
                /* read only file, init inflate algorithm */
                ret = inflateInit(tmp_zstrm);
            }
            else
            {
                /* write only file, init deflate algorithm */
                ret = deflateInit(tmp_zstrm, Z_DEFAULT_COMPRESSION);
                tmp_zstrm->avail_out = DARSHAN_DEF_COMP_BUF_SZ;
1036
                tmp_zstrm->next_out = state->dz.buf;
1037
1038
1039
1040
            }
            if(ret != Z_OK)
            {
                free(tmp_zstrm);
1041
                free(state->dz.buf);
1042
1043
                return(-1);
            }
1044
            state->dz.strm = tmp_zstrm;
1045
            break;
1046
        }
1047
1048
1049
1050
1051
1052
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
        {
            bz_stream *tmp_bzstrm = malloc(sizeof(*tmp_bzstrm));
            if(!tmp_bzstrm)
            {
1053
                free(state->dz.buf);
1054
1055
1056
1057
1058
1059
1060
1061
                return(-1);
            }
            tmp_bzstrm->bzalloc = NULL;
            tmp_bzstrm->bzfree = NULL;
            tmp_bzstrm->opaque = NULL;
            tmp_bzstrm->avail_in = 0;
            tmp_bzstrm->next_in = Z_NULL;

1062
            if(state->creat_flag)
1063
1064
1065
1066
1067
1068
1069
1070
1071
            {
                /* read only file, init decompress algorithm */
                ret = BZ2_bzDecompressInit(tmp_bzstrm, 1, 0);
            }
            else
            {
                /* write only file, init compress algorithm */