darshan-logutils.c 46.2 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
30
31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
struct darshan_dz_state
{
    /* (libz/bzip2) stream data structure for managing
     * compression and decompression state */
    void *strm;
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
    int size;
    /* for reading logs, flag indicating end of log file region */
    int eor;
    /* the region we last tried reading/writing */
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* compression type used on log file (libz or bzip2) */
    enum darshan_comp_type comp_type;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

    /* compression/decompression state */
    struct darshan_dz_state dz;
};

69
70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71
72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74
75
static int darshan_log_dzinit(struct darshan_fd_int_state *state);
static void darshan_log_dzdestroy(struct darshan_fd_int_state *state);
76
77
78
79
80
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
81
82
83
84
85
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
86
87
88
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

Shane Snyder's avatar
Shane Snyder committed
89
90
/* each module's implementation of the darshan logutil functions */
#define X(a, b, c) c,
91
92
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
93
    DARSHAN_MODULE_IDS
94
};
Shane Snyder's avatar
Shane Snyder committed
95
#undef X
96
97
98

/* darshan_log_open()
 *
99
 * open an existing darshan log file for reading only
100
 *
101
 * returns file descriptor on success, NULL on failure
102
 */
103
darshan_fd darshan_log_open(const char *name)
104
{
105
    darshan_fd tmp_fd;
106
    int ret;
107

108
    /* allocate a darshan file descriptor */
109
110
111
112
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
113
114
115
116
117
118
119
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
120

121
    /* open the log file in read mode */
122
123
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
124
    {
125
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
126
        free(tmp_fd->state);
127
128
129
        free(tmp_fd);
        return(NULL);
    }
130
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
131
132
133
134
135

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
136
137
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
138
        free(tmp_fd);
139
140
141
142
        return(NULL);
    }

    /* initialize compression data structures */
143
    ret = darshan_log_dzinit(tmp_fd->state);
144
145
146
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
147
148
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
149
150
        free(tmp_fd);
        return(NULL);
151
152
    }

153
154
155
156
157
158
159
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
160
 * returns file descriptor on success, NULL on failure
161
 */
162
163
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
164
165
{
    darshan_fd tmp_fd;
166
    int ret;
167

168
    /* allocate a darshan file descriptor */
169
170
171
172
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
173
174
175
176
177
178
179
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
180

181
182
183
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
184
    {
185
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
186
        free(tmp_fd->state);
187
        free(tmp_fd);
188
        return(NULL);
189
    }
190
191
    tmp_fd->state->creat_flag = 1;
    tmp_fd->state->comp_type = comp_type;
192
    tmp_fd->partial_flag = partial_flag;
193
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
194

195
196
197
198
199
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
200
    if(ret < 0)
201
202
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
203
204
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
205
206
207
        free(tmp_fd);
        unlink(name);
        return(NULL);
208
209
    }

210
    /* initialize compression data structures */
211
    ret = darshan_log_dzinit(tmp_fd->state);
212
213
    if(ret < 0)
    {
214
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
215
216
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
217
218
219
        free(tmp_fd);
        unlink(name);
        return(NULL);
220
221
    }

222
    return(tmp_fd);
223
224
}

225
/* darshan_log_getjob()
226
227
 *
 * read job level metadata from the darshan log file
228
 *
229
 * returns 0 on success, -1 on failure
230
 */
231
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
232
{
233
    struct darshan_fd_int_state *state = fd->state;
234
235
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
236
    int ret;
237

238
    assert(state);
239
240
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

241
    /* read the compressed job data from the log file */
242
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
243
    if(ret <= (int)sizeof(*job))
244
    {
245
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
246
247
        return(-1);
    }
248
249

    memcpy(job, job_buf, sizeof(*job));
250

251
    if(fd->swap_flag)
252
    {
253
254
255
256
257
258
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
259
260
    }

261
    /* save trailing exe & mount information, so it can be retrieved later */
262
263
264
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
265
        return(-1);
266
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
267

268
269
270
    return(0);
}

271
272
/* darshan_log_putjob()
 *
273
 * write job level metadata to darshan log file
274
275
276
277
278
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
279
    struct darshan_fd_int_state *state = fd->state;
280
281
282
283
    struct darshan_job job_copy;
    int len;
    int ret;

284
285
    assert(state);

286
287
288
289
290
291
292
293
294
295
296
297
298
299
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

300
301
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
302
    if(ret != (int)sizeof(*job))
303
    {
304
        state->err = -1;
305
306
307
308
309
310
311
312
313
314
315
316
317
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
318
319
int darshan_log_getexe(darshan_fd fd, char *buf)
{
320
    struct darshan_fd_int_state *state = fd->state;
321
    char *newline;
322
    int ret;
323

324
325
    assert(state);

326
    /* if the exe/mount data has not been saved yet, read in the job info */
327
    if(!(state->exe_mnt_data))
328
    {
329
330
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
331

332
        if(ret < 0 || !(state->exe_mnt_data))
333
            return(-1);
334
    }
335

336
    /* exe string is located before the first line break */
337
    newline = strchr(state->exe_mnt_data, '\n');
338
339

    /* copy over the exe string */
340
    if(newline)
341
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
342
343
344
345

    return (0);
}

346
347
348
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
349
 * NOTE: this needs to be called immediately following put_job as it
350
 * expects the file pointer to be positioned immediately following
351
 * the darshan job information
352
353
354
355
356
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
357
358
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
359
    int ret;
360

361
    assert(fd->state);
362

363
364
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
365
    {
366
        state->err = -1;
367
368
369
370
371
372
373
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

374
375
/* darshan_log_getmounts()
 * 
376
 * retrieves mount table information from the log. Note that mnt_pts and
377
 * fs_types are arrays that will be allocated by the function and must be
378
379
380
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
381
 */
382
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
383
384
    char*** fs_types, int* count)
{
385
    struct darshan_fd_int_state *state = fd->state;
386
387
    char *pos;
    int array_index = 0;
388
    int ret;
389

390
391
    assert(state);

392
    /* if the exe/mount data has not been saved yet, read in the job info */
393
    if(!(state->exe_mnt_data))
394
    {
395
396
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
397

398
        if(ret < 0 || !(state->exe_mnt_data))
399
            return(-1);
400
    }
401

402
    /* count entries */
403
    *count = 0;
404
    pos = state->exe_mnt_data;
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
426
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
427
428
    {
        /* overestimate string lengths */
429
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
430
        assert((*mnt_pts)[array_index]);
431
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
432
433
        assert((*fs_types)[array_index]);

434
435
436
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
437
        {
438
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
439
440
441
442
443
444
445
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

446
447
448
    return(0);
}

449
450
451
452
453
454
455
456
457
458
459
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
460
    struct darshan_fd_int_state *state = fd->state;
461
462
    int i;
    char line[1024];
463
464
465
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
466
467
    int ret;

468
469
    assert(state);

470
    /* write each mount entry to file */
471
    tmp = mnt_dat;
472
473
474
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
475
476
477
478
479
480

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

481
482
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
483
    {
484
        state->err = -1;
485
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
486
487
488
489
490
491
        return(-1);
    }

    return(0);
}

492
/* darshan_log_gethash()
493
 *
494
 * read the hash of records from the darshan log file
495
496
497
 *
 * returns 0 on success, -1 on failure
 */
498
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
499
{
500
    struct darshan_fd_int_state *state = fd->state;
501
    char *hash_buf;
502
    int hash_buf_sz;
503
    char *buf_ptr;
504
    darshan_record_id *rec_id_ptr;
505
    uint32_t *path_len_ptr, tmp_path_len;
506
507
    char *path_ptr;
    struct darshan_record_ref *ref;
508
509
510
    int read;
    int read_req_sz;
    int buf_remaining = 0;
511

512
513
    assert(state);

514
515
516
517
518
519
520
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

521
522
523
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
524
        return(-1);
525
526
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
527

528
    do
529
    {
530
531
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
532
         */
533
534
535
536
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
537
        {
538
539
540
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
541
542
        }

543
544
545
546
547
548
549
550
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
551
        {
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
573
            {
574
575
576
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
577
            }
578
579
580

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
581
            {
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
603
604
            }

605
606
607
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
608
        }
609

610
611
612
613
614
615
616
617
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
618

619
    free(hash_buf);
620
621
622
    return(0);
}

623
624
625
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
626
627
628
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
629
630
631
632
633
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
634
    struct darshan_fd_int_state *state = fd->state;
635
    char *hash_buf;
636
    int hash_buf_sz;
637
    struct darshan_record_ref *ref, *tmp;
638
639
640
    char *buf_ptr;
    int path_len;
    int wrote;
641

642
643
    assert(state);

644
645
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
646
647
648
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
649
    memset(hash_buf, 0, hash_buf_sz);
650

651
    /* individually serialize each hash record and write to log file */
652
653
    HASH_ITER(hlink, hash, ref, tmp)
    {
654
655
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
656

657
        /* the hash buffer has space to serialize this record
658
659
660
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
661
662
663
664
665
666
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
667

668
669
670
671
672
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
673
            state->err = -1;
674
675
676
677
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
678
679
680
681
682
683
684
685
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
686
687
 * get a chunk of module data from the darshan log file
 *
688
 * returns number of bytes read on success, -1 on failure
689
690
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
691
    void *mod_buf, int mod_buf_sz)
692
{
693
    struct darshan_fd_int_state *state = fd->state;
694
    int ret;
695

696
697
    assert(state);

698
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
699
    {
700
701
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
702
703
    }

704
705
706
707
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
708
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
709
    if(ret < 0)
710
    {
711
        fprintf(stderr,
712
            "Error: failed to read module %s data from darshan log file.\n",
713
            darshan_module_names[mod_id]);
714
715
716
        return(-1);
    }

717
    return(ret);
718
719
}

720
721
722
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
723
724
725
726
727
728
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
729
 *
730
 * returns number of bytes written on success, -1 on failure
731
732
733
734
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
735
    struct darshan_fd_int_state *state = fd->state;
736
737
    int ret;

738
739
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
740
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
741
    {
742
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
743
744
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
745
746
747
    }

    /* write the module chunk to the log file */
748
749
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
750
    {
751
        state->err = -1;
752
753
754
755
756
757
758
759
760
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

761
762
/* darshan_log_close()
 *
763
 * close an open darshan file descriptor, freeing any resources
764
765
766
 *
 */
void darshan_log_close(darshan_fd fd)
767
{
768
    struct darshan_fd_int_state *state = fd->state;
769
770
    int ret;

771
772
773
774
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
775
776
    {
        /* flush the last region of the log to file */
777
        switch(state->comp_type)
778
779
        {
            case DARSHAN_ZLIB_COMP:
780
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
781
782
                if(ret == 0)
                    break;
783
784
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
785
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
786
787
788
                if(ret == 0)
                    break;
#endif 
789
790
            default:
                /* if flush fails, remove the output log file */
791
                state->err = -1;
792
793
794
795
796
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
797
        if(state->err != -1)
798
799
800
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
801
                state->err = -1;
802
803
804
        }
    }

805
    close(state->fildes);
806
807

    /* remove output log file if error writing to it */
808
    if((state->creat_flag) && (state->err == -1))
809
810
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
811
812
            state->logfile_path);
        unlink(state->logfile_path);
813
    }
814

815
816
817
818
    darshan_log_dzdestroy(state);
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
819
    free(fd);
820
821

    return;
822
823
}

824
/* **************************************************** */
825

826
827
828
829
830
831
832
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
833
    struct darshan_fd_int_state *state = fd->state;
834
835
836
837
838
839
840
841
842
843
844
845
846
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
847
    if(ret != (int)sizeof(header))
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    /* save the version string */
    strncpy(fd->version, header.version_string, 8);

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

886
    state->comp_type = header.comp_type;
887
    fd->partial_flag = header.partial_flag;
888
889
890
891
892

    /* save the mapping of data within log file to this file descriptor */
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
    /* there may be nothing following the job data, so safety check map */
    fd->job_map.off = sizeof(struct darshan_header);
    if(fd->rec_map.off == 0)
    {
        for(i = 0; i < DARSHAN_MAX_MODS; i++)
        {
            if(fd->mod_map[i].off != 0)
            {
                fd->job_map.len = fd->mod_map[i].off - fd->job_map.off;
                break;
            }
        }

        if(fd->job_map.len == 0)
        {
            struct stat sbuf;
            if(fstat(fd->state->fildes, &sbuf) != 0)
            {
                fprintf(stderr, "Error: unable to stat darshan log file.\n");
                return(-1);
            }
            fd->job_map.len = sbuf.st_size - fd->job_map.off;
        }
    }
    else
    {
        fd->job_map.len = fd->rec_map.off - fd->job_map.off;
    }

922
923
924
925
926
927
928
929
930
    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
931
    struct darshan_fd_int_state *state = fd->state;
932
933
934
935
936
937
938
939
940
941
942
943
944
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
945
    header.comp_type = state->comp_type;
946
    header.partial_flag = fd->partial_flag;
947
948
949
950
951
952
953

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
954
    if(ret != (int)sizeof(header))
955
956
957
958
959
960
961
962
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

963
964
965
966
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
967
    struct darshan_fd_int_state *state = fd->state;
968
    off_t ret_off;
969

970
    if(state->pos == offset)
971
972
        return(0);

973
    ret_off = lseek(state->fildes, offset, SEEK_SET);
974
    if(ret_off == offset)
975
    {
976
        state->pos = offset;
977
        return(0);
978
979
980
981
982
    }

    return(-1);
}

983
/* return amount read on success, 0 on EOF, -1 on failure.
984
 */
985
static int darshan_log_read(darshan_fd fd, void* buf, int len)
986
{
987
    struct darshan_fd_int_state *state = fd->state;
988
989
    int ret;

990
    /* read data from the log file using the given map */
991
    ret = read(state->fildes, buf, len);
992
    if(ret > 0)
993
        state->pos += ret;
994

995
    return(ret);
996
997
}

998
/* return amount written on success, -1 on failure.
999
 */
1000
static int darshan_log_write(darshan_fd fd, void* buf, int len)
For faster browsing, not all history is shown. View entire blame