darshan-logutils.c 46.5 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
30
31
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
#define DARSHAN_REC_MAP_REGION_ID   (-1)

32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
struct darshan_dz_state
{
    /* (libz/bzip2) stream data structure for managing
     * compression and decompression state */
    void *strm;
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
    int size;
    /* for reading logs, flag indicating end of log file region */
    int eor;
    /* the region we last tried reading/writing */
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* compression type used on log file (libz or bzip2) */
    enum darshan_comp_type comp_type;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

    /* compression/decompression state */
    struct darshan_dz_state dz;
};

69
70
static int darshan_log_getheader(darshan_fd fd);
static int darshan_log_putheader(darshan_fd fd);
71
72
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
73
static int darshan_log_write(darshan_fd fd, void *buf, int len);
74
75
static int darshan_log_dzinit(struct darshan_fd_int_state *state);
static void darshan_log_dzdestroy(struct darshan_fd_int_state *state);
76
77
78
79
80
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
81
82
83
84
85
#ifdef HAVE_LIBBZ2
static int darshan_log_bzip2_read(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_write(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
86
87
88
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);

Shane Snyder's avatar
Shane Snyder committed
89
/* each module's implementation of the darshan logutil functions */
90
#define X(a, b, c, d) d,
91
92
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
93
    DARSHAN_MODULE_IDS
94
};
Shane Snyder's avatar
Shane Snyder committed
95
#undef X
96
97
98

/* darshan_log_open()
 *
99
 * open an existing darshan log file for reading only
100
 *
101
 * returns file descriptor on success, NULL on failure
102
 */
103
darshan_fd darshan_log_open(const char *name)
104
{
105
    darshan_fd tmp_fd;
106
    int ret;
107

108
    /* allocate a darshan file descriptor */
109
110
111
112
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
113
114
115
116
117
118
119
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
120

121
    /* open the log file in read mode */
122
123
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
124
    {
125
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
126
        free(tmp_fd->state);
127
128
129
        free(tmp_fd);
        return(NULL);
    }
130
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
131
132
133
134
135

    /* read the header from the log file to init fd data structures */
    ret = darshan_log_getheader(tmp_fd);
    if(ret < 0)
    {
136
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
137
138
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
139
        free(tmp_fd);
140
141
142
143
        return(NULL);
    }

    /* initialize compression data structures */
144
    ret = darshan_log_dzinit(tmp_fd->state);
145
146
147
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
148
149
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
150
151
        free(tmp_fd);
        return(NULL);
152
153
    }

154
155
156
157
158
159
160
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
161
 * returns file descriptor on success, NULL on failure
162
 */
163
164
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
165
166
{
    darshan_fd tmp_fd;
167
    int ret;
168

169
    /* allocate a darshan file descriptor */
170
171
172
173
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
174
175
176
177
178
179
180
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
181

182
183
184
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
185
    {
186
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
187
        free(tmp_fd->state);
188
        free(tmp_fd);
189
        return(NULL);
190
    }
191
192
    tmp_fd->state->creat_flag = 1;
    tmp_fd->state->comp_type = comp_type;
193
    tmp_fd->partial_flag = partial_flag;
194
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
195

196
197
198
199
200
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
201
    if(ret < 0)
202
203
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
204
205
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
206
207
208
        free(tmp_fd);
        unlink(name);
        return(NULL);
209
210
    }

211
    /* initialize compression data structures */
212
    ret = darshan_log_dzinit(tmp_fd->state);
213
214
    if(ret < 0)
    {
215
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
216
217
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
218
219
220
        free(tmp_fd);
        unlink(name);
        return(NULL);
221
222
    }

223
    return(tmp_fd);
224
225
}

226
/* darshan_log_getjob()
227
228
 *
 * read job level metadata from the darshan log file
229
 *
230
 * returns 0 on success, -1 on failure
231
 */
232
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
233
{
234
    struct darshan_fd_int_state *state = fd->state;
235
236
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
237
    int ret;
238

239
    assert(state);
240
241
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

242
    /* read the compressed job data from the log file */
243
244
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
    if(ret <= sizeof(*job))
245
    {
246
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
247
248
        return(-1);
    }
249
250

    memcpy(job, job_buf, sizeof(*job));
251

252
    if(fd->swap_flag)
253
    {
254
255
256
257
258
259
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
260
261
    }

262
    /* save trailing exe & mount information, so it can be retrieved later */
263
264
265
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
266
        return(-1);
267
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
268

269
270
271
    return(0);
}

272
273
/* darshan_log_putjob()
 *
274
 * write job level metadata to darshan log file
275
276
277
278
279
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
280
    struct darshan_fd_int_state *state = fd->state;
281
282
283
284
    struct darshan_job job_copy;
    int len;
    int ret;

285
286
    assert(state);

287
288
289
290
291
292
293
294
295
296
297
298
299
300
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

301
302
303
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
    if(ret != sizeof(*job))
304
    {
305
        state->err = -1;
306
307
308
309
310
311
312
313
314
315
316
317
318
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
319
320
int darshan_log_getexe(darshan_fd fd, char *buf)
{
321
    struct darshan_fd_int_state *state = fd->state;
322
    char *newline;
323
    int ret;
324

325
326
    assert(state);

327
    /* if the exe/mount data has not been saved yet, read in the job info */
328
    if(!(state->exe_mnt_data))
329
    {
330
331
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
332

333
        if(ret < 0 || !(state->exe_mnt_data))
334
            return(-1);
335
    }
336

337
    /* exe string is located before the first line break */
338
    newline = strchr(state->exe_mnt_data, '\n');
339
340

    /* copy over the exe string */
341
    if(newline)
342
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
343
344
345
346

    return (0);
}

347
348
349
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
350
 * NOTE: this needs to be called immediately following put_job as it
351
 * expects the file pointer to be positioned immediately following
352
 * the darshan job information
353
354
355
356
357
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
358
359
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
360
    int ret;
361

362
    assert(fd->state);
363

364
365
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
366
    {
367
        state->err = -1;
368
369
370
371
372
373
374
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

375
376
/* darshan_log_getmounts()
 * 
377
 * retrieves mount table information from the log. Note that mnt_pts and
378
 * fs_types are arrays that will be allocated by the function and must be
379
380
381
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
382
 */
383
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
384
385
    char*** fs_types, int* count)
{
386
    struct darshan_fd_int_state *state = fd->state;
387
388
    char *pos;
    int array_index = 0;
389
    int ret;
390

391
392
    assert(state);

393
    /* if the exe/mount data has not been saved yet, read in the job info */
394
    if(!(state->exe_mnt_data))
395
    {
396
397
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
398

399
        if(ret < 0 || !(state->exe_mnt_data))
400
            return(-1);
401
    }
402

403
    /* count entries */
404
    *count = 0;
405
    pos = state->exe_mnt_data;
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
427
    while((pos = strrchr(state->exe_mnt_data, '\n')) != NULL)
428
429
    {
        /* overestimate string lengths */
430
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
431
        assert((*mnt_pts)[array_index]);
432
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
433
434
        assert((*fs_types)[array_index]);

435
436
437
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
438
        {
439
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
440
441
442
443
444
445
446
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

447
448
449
    return(0);
}

450
451
452
453
454
455
456
457
458
459
460
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
461
    struct darshan_fd_int_state *state = fd->state;
462
463
    int i;
    char line[1024];
464
465
466
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
467
468
    int ret;

469
470
    assert(state);

471
    /* write each mount entry to file */
472
    tmp = mnt_dat;
473
474
475
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
476
477
478
479
480
481

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

482
483
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
484
    {
485
        state->err = -1;
486
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
487
488
489
490
491
492
        return(-1);
    }

    return(0);
}

493
/* darshan_log_gethash()
494
 *
495
 * read the hash of records from the darshan log file
496
497
498
 *
 * returns 0 on success, -1 on failure
 */
499
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
500
{
501
    struct darshan_fd_int_state *state = fd->state;
502
    char *hash_buf;
503
    int hash_buf_sz;
504
    char *buf_ptr;
505
    darshan_record_id *rec_id_ptr;
506
    uint32_t *path_len_ptr, tmp_path_len;
507
508
    char *path_ptr;
    struct darshan_record_ref *ref;
509
510
511
    int read;
    int read_req_sz;
    int buf_remaining = 0;
512

513
514
    assert(state);

515
516
517
518
519
520
521
    /* just return if there is no record mapping data */
    if(fd->rec_map.len == 0)
    {
        *hash = NULL;
        return(0);
    }

522
523
524
    /* default to hash buffer twice as big as default compression buf */
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ * 2);
    if(!hash_buf)
525
        return(-1);
526
527
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ * 2);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
528

529
    do
530
    {
531
532
        /* read chunks of the darshan record id -> file name mapping from log file,
         * constructing a hash table in the process
533
         */
534
535
536
537
        read_req_sz = hash_buf_sz - buf_remaining;
        read = darshan_log_dzread(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf + buf_remaining, read_req_sz);
        if(read < 0)
538
        {
539
540
541
            fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
            free(hash_buf);
            return(-1);
542
543
        }

544
545
546
547
548
549
550
551
        /* work through the hash buffer -- deserialize the mapping data and
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
        buf_ptr = hash_buf;
        buf_remaining += read;
        while(buf_remaining > (sizeof(darshan_record_id) + sizeof(uint32_t)))
552
        {
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
            /* see if we have enough buf space to read in the next full record */
            tmp_path_len = *(uint32_t *)(buf_ptr + sizeof(darshan_record_id));
            if(fd->swap_flag)
                DARSHAN_BSWAP32(&tmp_path_len);

            /* we need to read more before we continue deserializing */
            if(buf_remaining <
                (sizeof(darshan_record_id) + sizeof(uint32_t) + tmp_path_len))
                break;

            /* get pointers for each field of this darshan record */
            /* NOTE: darshan record hash serialization method: 
             *          ... darshan_record_id | (uint32_t) path_len | path ...
             */
            rec_id_ptr = (darshan_record_id *)buf_ptr;
            buf_ptr += sizeof(darshan_record_id);
            path_len_ptr = (uint32_t *)buf_ptr;
            buf_ptr += sizeof(uint32_t);
            path_ptr = (char *)buf_ptr;

            if(fd->swap_flag)
574
            {
575
576
577
                /* we need to sort out endianness issues before deserializing */
                DARSHAN_BSWAP64(rec_id_ptr);
                DARSHAN_BSWAP32(path_len_ptr);
578
            }
579
580
581

            HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
            if(!ref)
582
            {
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
                    free(hash_buf);
                    return(-1);
                }
                ref->rec.name = malloc(*path_len_ptr + 1);
                if(!ref->rec.name)
                {
                    free(ref);
                    free(hash_buf);
                    return(-1);
                }

                /* set the fields for this record */
                ref->rec.id = *rec_id_ptr;
                memcpy(ref->rec.name, path_ptr, *path_len_ptr);
                ref->rec.name[*path_len_ptr] = '\0';

                /* add this record to the hash */
                HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
604
605
            }

606
607
608
            buf_ptr += *path_len_ptr;
            buf_remaining -=
                (sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr);
609
        }
610

611
612
613
614
615
616
617
618
        /* copy any leftover data to beginning of buffer to parse next */
        memcpy(hash_buf, buf_ptr, buf_remaining);

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
    assert(buf_remaining == 0);
619

620
    free(hash_buf);
621
622
623
    return(0);
}

624
625
626
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
627
628
629
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
630
631
632
633
634
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
635
    struct darshan_fd_int_state *state = fd->state;
636
    char *hash_buf;
637
    int hash_buf_sz;
638
    struct darshan_record_ref *ref, *tmp;
639
640
641
    char *buf_ptr;
    int path_len;
    int wrote;
642

643
644
    assert(state);

645
646
    /* allocate memory for largest possible hash record */
    hash_buf_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + PATH_MAX;
647
648
649
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
        return(-1);
650
    memset(hash_buf, 0, hash_buf_sz);
651

652
    /* individually serialize each hash record and write to log file */
653
654
    HASH_ITER(hlink, hash, ref, tmp)
    {
655
656
        buf_ptr = hash_buf;
        path_len = strlen(ref->rec.name);
657

658
        /* the hash buffer has space to serialize this record
659
660
661
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
662
663
664
665
666
667
        *((darshan_record_id *)buf_ptr) = ref->rec.id;
        buf_ptr += sizeof(darshan_record_id);
        *((uint32_t *)buf_ptr) = path_len;
        buf_ptr += sizeof(uint32_t);
        memcpy(buf_ptr, ref->rec.name, path_len);
        buf_ptr += path_len;
668

669
670
671
672
673
        /* write this hash entry to log file */
        wrote = darshan_log_dzwrite(fd, DARSHAN_REC_MAP_REGION_ID,
            hash_buf, (buf_ptr - hash_buf));
        if(wrote != (buf_ptr - hash_buf))
        {
674
            state->err = -1;
675
676
677
678
            fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
            free(hash_buf);
            return(-1);
        }
679
680
681
682
683
684
685
686
    }

    free(hash_buf);
    return(0);
}

/* darshan_log_getmod()
 *
687
688
 * get a chunk of module data from the darshan log file
 *
689
 * returns number of bytes read on success, -1 on failure
690
691
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
692
    void *mod_buf, int mod_buf_sz)
693
{
694
    struct darshan_fd_int_state *state = fd->state;
695
    int ret;
696

697
698
    assert(state);

699
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
700
    {
701
702
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
703
704
    }

705
706
707
708
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
709
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
710
    if(ret < 0)
711
    {
712
        fprintf(stderr,
713
            "Error: failed to read module %s data from darshan log file.\n",
714
            darshan_module_names[mod_id]);
715
716
717
        return(-1);
    }

718
    return(ret);
719
720
}

721
722
723
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
724
725
726
727
728
729
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
730
 *
731
 * returns number of bytes written on success, -1 on failure
732
733
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
734
    void *mod_buf, int mod_buf_sz, int ver)
735
{
736
    struct darshan_fd_int_state *state = fd->state;
737
738
    int ret;

739
740
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
741
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
742
    {
743
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
744
745
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
746
747
748
    }

    /* write the module chunk to the log file */
749
750
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
751
    {
752
        state->err = -1;
753
754
755
756
757
758
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

759
760
761
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

762
763
764
    return(0);
}

765
766
/* darshan_log_close()
 *
767
 * close an open darshan file descriptor, freeing any resources
768
769
770
 *
 */
void darshan_log_close(darshan_fd fd)
771
{
772
    struct darshan_fd_int_state *state = fd->state;
773
774
    int ret;

775
776
777
778
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
779
780
    {
        /* flush the last region of the log to file */
781
        switch(state->comp_type)
782
783
        {
            case DARSHAN_ZLIB_COMP:
784
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
785
786
                if(ret == 0)
                    break;
787
788
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
789
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
790
791
792
                if(ret == 0)
                    break;
#endif 
793
794
            default:
                /* if flush fails, remove the output log file */
795
                state->err = -1;
796
797
798
799
800
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
801
        if(state->err != -1)
802
803
804
        {
            ret = darshan_log_putheader(fd);
            if(ret < 0)
805
                state->err = -1;
806
807
808
        }
    }

809
    close(state->fildes);
810
811

    /* remove output log file if error writing to it */
812
    if((state->creat_flag) && (state->err == -1))
813
814
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
815
816
            state->logfile_path);
        unlink(state->logfile_path);
817
    }
818

819
820
821
822
    darshan_log_dzdestroy(state);
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
823
    free(fd);
824
825

    return;
826
827
}

828
/* **************************************************** */
829

830
831
832
833
834
835
836
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_getheader(darshan_fd fd)
{
837
    struct darshan_fd_int_state *state = fd->state;
838
839
840
841
842
843
844
845
846
847
848
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
    if(strcmp(fd->version, "3.00"))
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
        fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&(header.rec_map.off));
            DARSHAN_BSWAP64(&(header.rec_map.len));
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

911
    /* set some fd fields based on what's stored in the header */
912
    state->comp_type = header.comp_type;
913
    fd->partial_flag = header.partial_flag;
914
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930

    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header.rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &(header.rec_map), sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &(header.mod_map), DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    return(0);
}

/* write a darshan header to log file
 *
 * returns 0 on success, -1 on failure
 */
static int darshan_log_putheader(darshan_fd fd)
{
931
    struct darshan_fd_int_state *state = fd->state;
932
933
934
935
936
937
938
939
940
941
942
943
944
    struct darshan_header header;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
945
    header.comp_type = state->comp_type;
946
    header.partial_flag = fd->partial_flag;
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

    /* write header to file */
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

963
964
965
966
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
967
    struct darshan_fd_int_state *state = fd->state;
968
    off_t ret_off;
969

970
    if(state->pos == offset)
971
972
        return(0);

973
    ret_off = lseek(state->fildes, offset, SEEK_SET);
974
    if(ret_off == offset)
975
    {
976
        state->pos = offset;
977
        return(0);
978
979
980
981
982
    }

    return(-1);
}

983
/* return amount read on success, 0 on EOF, -1 on failure.
984
 */
985
static int darshan_log_read(darshan_fd fd, void* buf, int len)
986
{
987
    struct darshan_fd_int_state *state = fd->state;
988
989
    int ret;

990
    /* read data from the log file using the given map */
991
    ret = read(state->fildes, buf, len);
992
    if(ret > 0)
993
        state->pos += ret;
994

995
    return(ret);
996
997
}

998
/* return amount written on success, -1 on failure.
999
 */
1000
static int darshan_log_write(darshan_fd fd, void* buf, int len)
1001
{
1002
    struct darshan_fd_int_state *state = fd->state;
1003
1004
    int ret;

1005
    ret = write(state->fildes, buf, len);
1006
    if(ret > 0)
1007
        state->pos += ret;
1008
1009
1010
1011

    return(ret);
}

1012
static int darshan_log_dzinit(struct darshan_fd_int_state *state)
1013
1014
1015
{
    int ret;

1016
    /* initialize buffers for staging compressed data to/from log file */
1017
1018
    state->dz.buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(state->dz.buf == NULL)
1019
1020
        return(-1);

1021
    state->dz.prev_reg_id = DARSHAN_HEADER_REGION_ID;
1022

1023
    switch(state->comp_type)
1024
1025
    {
        case DARSHAN_ZLIB_COMP:
1026
1027
1028
1029
        {
            z_stream *tmp_zstrm = malloc(sizeof(*tmp_zstrm));
            if(!tmp_zstrm)
            {
1030
                free(state->dz.buf);
1031
1032
1033
1034
1035
1036
1037
1038
1039
                return(-1);
            }
            tmp_zstrm->zalloc = Z_NULL;
            tmp_zstrm->zfree = Z_NULL;
            tmp_zstrm->opaque = Z_NULL;
            tmp_zstrm->avail_in = 0;
            tmp_zstrm->next_in = Z_NULL;

            /* TODO: worth using {inflate/deflate}Init2 ?? */
1040
            if(!(state->creat_flag))