darshan-logutils.c 48.1 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
24
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */

25
/* special identifers for referring to header, job, and
26
27
28
29
 * record map regions of the darshan log file
 */
#define DARSHAN_HEADER_REGION_ID    (-3)
#define DARSHAN_JOB_REGION_ID       (-2)
30
#define DARSHAN_NAME_MAP_REGION_ID  (-1)
31

32
33
struct darshan_dz_state
{
34
35
36
37
38
    /* pointer to arbitrary data structure used for managing
     * compression/decompression state (e.g., z_stream
     * structure needed for libz)
     */
    void *comp_dat;
39
40
41
    /* buffer for staging compressed data to/from log file */
    unsigned char *buf;
    /* size of staging buffer */
42
    unsigned int size;
43
44
    /* for reading logs, flag indicating end of log file region */
    int eor;
45
    /* the region id we last tried reading/writing */
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    int prev_reg_id;
};

/* internal fd data structure */
struct darshan_fd_int_state
{
    /* posix file descriptor for the log file */
    int fildes;
    /* file pointer position */
    int64_t pos;
    /* flag indicating whether log file was created (and written) */
    int creat_flag;
    /* log file path name */
    char logfile_path[PATH_MAX];
    /* pointer to exe & mount data in darshan job data structure */
    char *exe_mnt_data;
    /* whether previous file operations have failed */
    int err;

65
    /* compression/decompression stream read/write state */
66
67
68
    struct darshan_dz_state dz;
};

69
static int darshan_mnt_info_cmp(const void *a, const void *b);
70
71
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
72
73
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
74
static int darshan_log_write(darshan_fd fd, void *buf, int len);
75
76
static int darshan_log_dzinit(darshan_fd fd);
static void darshan_log_dzdestroy(darshan_fd fd);
77
78
static int darshan_log_dzread(darshan_fd fd, int region_id, void *buf, int len);
static int darshan_log_dzwrite(darshan_fd fd, int region_id, void *buf, int len);
79
80
81
82
static int darshan_log_libz_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_libz_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
83
static int darshan_log_libz_flush(darshan_fd fd, int region_id);
84
#ifdef HAVE_LIBBZ2
85
86
87
88
static int darshan_log_bzip2_read(darshan_fd fd, struct darshan_log_map map, 
    void *buf, int len, int reset_strm_flag);
static int darshan_log_bzip2_write(darshan_fd fd, struct darshan_log_map *map_p,
    void *buf, int len, int flush_strm_flag);
89
90
static int darshan_log_bzip2_flush(darshan_fd fd, int region_id);
#endif
91
92
static int darshan_log_dzload(darshan_fd fd, struct darshan_log_map map);
static int darshan_log_dzunload(darshan_fd fd, struct darshan_log_map *map_p);
93
94
static int darshan_log_noz_read(darshan_fd fd, struct darshan_log_map map,
    void *buf, int len, int reset_strm_flag);
95

Shane Snyder's avatar
Shane Snyder committed
96
/* each module's implementation of the darshan logutil functions */
97
#define X(a, b, c, d) d,
98
99
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
Shane Snyder's avatar
Shane Snyder committed
100
    DARSHAN_MODULE_IDS
101
};
Shane Snyder's avatar
Shane Snyder committed
102
#undef X
103
104
105

/* darshan_log_open()
 *
106
 * open an existing darshan log file for reading only
107
 *
108
 * returns file descriptor on success, NULL on failure
109
 */
110
darshan_fd darshan_log_open(const char *name)
111
{
112
    darshan_fd tmp_fd;
113
    int ret;
114

115
    /* allocate a darshan file descriptor */
116
117
118
119
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
120
121
122
123
124
125
126
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd->state);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
127

128
    /* open the log file in read mode */
129
130
    tmp_fd->state->fildes = open(name, O_RDONLY);
    if(tmp_fd->state->fildes < 0)
131
    {
132
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
133
        free(tmp_fd->state);
134
135
136
        free(tmp_fd);
        return(NULL);
    }
137
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
138
139

    /* read the header from the log file to init fd data structures */
140
    ret = darshan_log_get_header(tmp_fd);
141
142
    if(ret < 0)
    {
143
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
144
145
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
146
        free(tmp_fd);
147
148
149
150
        return(NULL);
    }

    /* initialize compression data structures */
151
    ret = darshan_log_dzinit(tmp_fd);
152
153
154
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to initialize decompression data structures.\n");
155
156
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
157
158
        free(tmp_fd);
        return(NULL);
159
160
    }

161
162
163
164
165
166
167
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
168
 * returns file descriptor on success, NULL on failure
169
 */
170
171
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
    int partial_flag)
172
173
{
    darshan_fd tmp_fd;
174
    int ret;
175

176
    /* allocate a darshan file descriptor */
177
178
179
180
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));
181
182
183
184
185
186
187
    tmp_fd->state = malloc(sizeof(struct darshan_fd_int_state));
    if(!tmp_fd->state)
    {
        free(tmp_fd);
        return(NULL);
    }
    memset(tmp_fd->state, 0, sizeof(struct darshan_fd_int_state));
188
    tmp_fd->comp_type = comp_type;
189

190
191
192
    /* create the log for writing, making sure to not overwrite existing log */
    tmp_fd->state->fildes = creat(name, 0400);
    if(tmp_fd->state->fildes < 0)
193
    {
194
        fprintf(stderr, "Error: failed to open darshan log file %s.\n", name);
195
        free(tmp_fd->state);
196
        free(tmp_fd);
197
        return(NULL);
198
    }
199
    tmp_fd->state->creat_flag = 1;
200
    tmp_fd->partial_flag = partial_flag;
201
    strncpy(tmp_fd->state->logfile_path, name, PATH_MAX);
202

203
204
205
206
207
    /* position file pointer to prealloc space for the log file header
     * NOTE: the header is written at close time, after all internal data
     * structures have been properly set
     */
    ret = darshan_log_seek(tmp_fd, sizeof(struct darshan_header));
208
    if(ret < 0)
209
210
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
211
212
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
213
214
215
        free(tmp_fd);
        unlink(name);
        return(NULL);
216
217
    }

218
    /* initialize compression data structures */
219
    ret = darshan_log_dzinit(tmp_fd);
220
221
    if(ret < 0)
    {
222
        fprintf(stderr, "Error: failed to initialize compression data structures.\n");
223
224
        close(tmp_fd->state->fildes);
        free(tmp_fd->state);
225
226
227
        free(tmp_fd);
        unlink(name);
        return(NULL);
228
229
    }

230
    return(tmp_fd);
231
232
}

233
/* darshan_log_get_job()
234
235
 *
 * read job level metadata from the darshan log file
236
 *
237
 * returns 0 on success, -1 on failure
238
 */
239
int darshan_log_get_job(darshan_fd fd, struct darshan_job *job)
240
{
241
    struct darshan_fd_int_state *state = fd->state;
242
243
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
244
    int ret;
245

246
    assert(state);
247
248
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

249
    /* read the compressed job data from the log file */
250
    ret = darshan_log_dzread(fd, DARSHAN_JOB_REGION_ID, job_buf, job_buf_sz);
251
    if(ret <= (int)sizeof(*job))
252
    {
253
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
254
255
        return(-1);
    }
256
257

    memcpy(job, job_buf, sizeof(*job));
258

259
    if(fd->swap_flag)
260
    {
261
262
263
264
265
266
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
267
268
    }

269
    /* save trailing exe & mount information, so it can be retrieved later */
270
271
272
    if(!(state->exe_mnt_data))
        state->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!(state->exe_mnt_data))
273
        return(-1);
274
    memcpy(state->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
275

276
277
278
    return(0);
}

279
/* darshan_log_put_job()
280
 *
281
 * write job level metadata to darshan log file
282
283
284
 *
 * returns 0 on success, -1 on failure
 */
285
int darshan_log_put_job(darshan_fd fd, struct darshan_job *job)
286
{
287
    struct darshan_fd_int_state *state = fd->state;
288
289
290
291
    struct darshan_job job_copy;
    int len;
    int ret;

292
293
    assert(state);

294
295
296
297
298
299
300
301
302
303
304
305
306
307
    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

308
309
    /* write the compressed job data to log file */
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, &job_copy, sizeof(*job));
310
    if(ret != sizeof(*job))
311
    {
312
        state->err = -1;
313
314
315
316
317
318
319
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

320
/* darshan_log_get_exe()
321
322
323
324
325
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
326
int darshan_log_get_exe(darshan_fd fd, char *buf)
327
{
328
    struct darshan_fd_int_state *state = fd->state;
329
    char *newline;
330
    int ret;
331

332
333
    assert(state);

334
    /* if the exe/mount data has not been saved yet, read in the job info */
335
    if(!(state->exe_mnt_data))
336
    {
337
        struct darshan_job job;
338
        ret = darshan_log_get_job(fd, &job);
339

340
        if(ret < 0 || !(state->exe_mnt_data))
341
            return(-1);
342
    }
343

344
    /* exe string is located before the first line break */
345
    newline = strchr(state->exe_mnt_data, '\n');
346
347

    /* copy over the exe string */
348
    if(newline)
349
        memcpy(buf, state->exe_mnt_data, (newline - state->exe_mnt_data));
350
351
352
353

    return (0);
}

354
/* darshan_log_put_exe()
355
356
 *
 * wrties the application exe name to darshan log file
357
 * NOTE: this needs to be called immediately following put_job as it
358
 * expects the file pointer to be positioned immediately following
359
 * the darshan job information
360
361
362
 *
 * returns 0 on success, -1 on failure 
 */
363
int darshan_log_put_exe(darshan_fd fd, char *buf)
364
{
365
366
    struct darshan_fd_int_state *state = fd->state;
    int len = strlen(buf);
367
    int ret;
368

369
    assert(fd->state);
370

371
372
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, buf, len);
    if(ret != len)
373
    {
374
        state->err = -1;
375
376
377
378
379
380
381
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

382
/* darshan_log_get_mounts()
383
 * 
384
385
386
 * retrieves mount table information from the log. Note that mnt_data_array
 * is an array that will be allocated by the function and must be
 * freed by the caller. count will indicate the size of the array
387
388
 *
 * returns 0 on success, -1 on failure
389
 */
390
int darshan_log_get_mounts(darshan_fd fd, struct darshan_mnt_info **mnt_data_array,
391
    int* count)
392
{
393
    struct darshan_fd_int_state *state = fd->state;
394
395
    char *pos;
    int array_index = 0;
396
    int ret;
397

398
399
    assert(state);

400
    /* if the exe/mount data has not been saved yet, read in the job info */
401
    if(!(state->exe_mnt_data))
402
    {
403
        struct darshan_job job;
404
        ret = darshan_log_get_job(fd, &job);
405

406
        if(ret < 0 || !(state->exe_mnt_data))
407
            return(-1);
408
    }
409

410
    /* count entries */
411
    *count = 0;
412
    pos = state->exe_mnt_data;
413
414
415
416
417
418
419
420
421
422
423
424
425
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
426
427
    *mnt_data_array = malloc((*count)*sizeof(**mnt_data_array));
    assert(*mnt_data_array);
428

429
    /* work through the table and parse each line (except for
430
431
     * first, which holds command line information)
     */
432
433
    pos = state->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
434
    {
435
436
        ret = sscanf(++pos, "%s\t%s", (*mnt_data_array)[array_index].mnt_type,
            (*mnt_data_array)[array_index].mnt_path);
437
        if(ret != 2)
438
        {
439
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
440
441
442
443
444
            return(-1);
        }
        array_index++;
    }

445
446
    qsort(*mnt_data_array, *count, sizeof(**mnt_data_array), darshan_mnt_info_cmp);

447
448
449
    return(0);
}

450
/* darshan_log_put_mounts()
451
452
453
454
455
456
457
458
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
459
int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_array,
460
    int count)
461
{
462
    struct darshan_fd_int_state *state = fd->state;
463
464
    int i;
    char line[1024];
465
466
467
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char *tmp;
468
469
    int ret;

470
471
    assert(state);

472
    /* write each mount entry to file */
473
    tmp = mnt_dat;
474
475
    for(i=count-1; i>=0; i--)
    {
476
        sprintf(line, "\n%s\t%s", mnt_data_array[i].mnt_type, mnt_data_array[i].mnt_path);
477
478
479
480
481
482

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

483
484
    ret = darshan_log_dzwrite(fd, DARSHAN_JOB_REGION_ID, mnt_dat, mnt_dat_sz);
    if (ret != mnt_dat_sz)
485
    {
486
        state->err = -1;
487
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
488
489
490
491
492
493
        return(-1);
    }

    return(0);
}

494
/* darshan_log_get_namehash()
495
 *
496
 * read the hash of name records from the darshan log file
497
498
499
 *
 * returns 0 on success, -1 on failure
 */
500
int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **hash)
501
{
502
    struct darshan_fd_int_state *state = fd->state;
503
    char *name_rec_buf;
504
    char *tmp_p;
505
    int name_rec_buf_sz;
506
507
    int read;
    int read_req_sz;
508
509
    struct darshan_name_record_ref *ref;
    struct darshan_name_record *name_rec;
510
    int buf_rem = 0;
511
    int rec_len;
512

513
514
    assert(state);

515
516
    /* just return if there is no name record mapping data */
    if(fd->name_map.len == 0)
517
518
519
520
521
    {
        *hash = NULL;
        return(0);
    }

522
523
524
525
    /* default to buffer twice as big as default compression buf */
    name_rec_buf_sz = DARSHAN_DEF_COMP_BUF_SZ * 2;
    name_rec_buf = malloc(name_rec_buf_sz);
    if(!name_rec_buf)
526
        return(-1);
527
    memset(name_rec_buf, 0, name_rec_buf_sz);
528

529
    do
530
    {
531
        /* read chunks of the darshan record id -> name mapping from log file,
532
         * constructing a hash table in the process
533
         */
534
535
536
        read_req_sz = name_rec_buf_sz - buf_rem;
        read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec_buf + buf_rem, read_req_sz);
537
        if(read < 0)
538
        {
539
540
            fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
            free(name_rec_buf);
541
            return(-1);
542
        }
543
        buf_rem += read;
544

545
        /* work through the name record buffer -- deserialize the mapping data and
546
547
548
549
         * add to the output hash table
         * NOTE: these mapping pairs are variable in length, so we have to be able
         * to handle incomplete mappings temporarily here
         */
550
551
        name_rec = (struct darshan_name_record *)name_rec_buf;
        while(buf_rem > sizeof(darshan_record_id) + 1)
552
        {
553
554
            if(strnlen(name_rec->name, buf_rem - sizeof(darshan_record_id)) ==
                (buf_rem - sizeof(darshan_record_id)))
555
            {
556
557
558
                /* if this record name's terminating null character is not
                 * present, we need to read more of the buffer before continuing
                 */
559
                break;
560
            }
561
562

            if(fd->swap_flag)
563
            {
564
                /* we need to sort out endianness issues before deserializing */
565
                DARSHAN_BSWAP64(&(name_rec->id));
566
            }
567

568
            HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
569
            if(!ref)
570
            {
571
                rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
572
573
574
                ref = malloc(sizeof(*ref));
                if(!ref)
                {
575
                    free(name_rec_buf);
576
577
                    return(-1);
                }
578
579
                ref->name_record = malloc(rec_len);
                if(!ref->name_record)
580
581
                {
                    free(ref);
582
                    free(name_rec_buf);
583
584
585
                    return(-1);
                }

586
587
                /* copy the name record over from the hash buffer */
                memcpy(ref->name_record, name_rec, rec_len);
588
589

                /* add this record to the hash */
590
                HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
591
592
            }

593
594
595
            tmp_p = (char *)name_rec + rec_len;
            name_rec = (struct darshan_name_record *)tmp_p;
            buf_rem -= rec_len;
596
        }
597

598
        /* copy any leftover data to beginning of buffer to parse next */
599
        memcpy(name_rec_buf, name_rec, buf_rem);
600
601
602
603
604

        /* we keep reading until we get a short read informing us we have
         * read all of the record hash
         */
    } while(read == read_req_sz);
605
    assert(buf_rem == 0);
606

607
    free(name_rec_buf);
608
609
610
    return(0);
}

611
/* darshan_log_put_namehash()
612
 *
613
 * writes the hash table of name records to the darshan log file
614
615
616
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
617
618
619
 *
 * returns 0 on success, -1 on failure
 */
620
int darshan_log_put_namehash(darshan_fd fd, struct darshan_name_record_ref *hash)
621
{
622
    struct darshan_fd_int_state *state = fd->state;
623
624
625
    struct darshan_name_record_ref *ref, *tmp;
    struct darshan_name_record_ref *name_rec;
    int name_rec_len;
626
    int wrote;
627

628
629
    assert(state);

630
    /* allocate memory for largest possible hash record */
631
632
    name_rec = malloc(sizeof(struct darshan_name_record) + PATH_MAX);
    if(!name_rec)
633
        return(-1);
634
    memset(name_rec, 0, sizeof(struct darshan_name_record) + PATH_MAX);
635

636
    /* individually serialize each hash record and write to log file */
637
638
    HASH_ITER(hlink, hash, ref, tmp)
    {
639
640
        name_rec_len = sizeof(struct darshan_name_record) + strlen(ref->name_record->name);
        memcpy(name_rec, ref->name_record, name_rec_len);
641

642
        /* write this hash entry to log file */
643
644
645
        wrote = darshan_log_dzwrite(fd, DARSHAN_NAME_MAP_REGION_ID,
            name_rec, name_rec_len);
        if(wrote != name_rec_len)
646
        {
647
            state->err = -1;
648
649
            fprintf(stderr, "Error: failed to write name hash to darshan log file.\n");
            free(name_rec);
650
651
            return(-1);
        }
652
653
    }

654
    free(name_rec);
655
656
657
    return(0);
}

658
/* darshan_log_get_mod()
659
 *
660
661
 * get a chunk of module data from the darshan log file
 *
662
 * returns number of bytes read on success, -1 on failure
663
 */
664
int darshan_log_get_mod(darshan_fd fd, darshan_module_id mod_id,
665
    void *mod_buf, int mod_buf_sz)
666
{
667
    struct darshan_fd_int_state *state = fd->state;
668
    int ret;
669

670
671
    assert(state);

672
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
673
    {
674
675
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
676
677
    }

678
679
680
681
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    /* read this module's data from the log file */
682
    ret = darshan_log_dzread(fd, mod_id, mod_buf, mod_buf_sz);
683
    if(ret < 0)
684
    {
685
        fprintf(stderr,
686
            "Error: failed to read module %s data from darshan log file.\n",
687
            darshan_module_names[mod_id]);
688
689
690
        return(-1);
    }

691
    return(ret);
692
693
}

694
/* darshan_log_put_mod()
695
696
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
697
698
699
700
701
702
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
703
 *
704
 * returns number of bytes written on success, -1 on failure
705
 */
706
int darshan_log_put_mod(darshan_fd fd, darshan_module_id mod_id,
707
    void *mod_buf, int mod_buf_sz, int ver)
708
{
709
    struct darshan_fd_int_state *state = fd->state;
710
711
    int ret;

712
713
    assert(state);

Shane Snyder's avatar
Shane Snyder committed
714
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
715
    {
716
        state->err = -1;
Shane Snyder's avatar
Shane Snyder committed
717
718
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
719
720
721
    }

    /* write the module chunk to the log file */
722
723
    ret = darshan_log_dzwrite(fd, mod_id, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
724
    {
725
        state->err = -1;
726
727
728
729
730
731
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

732
733
734
    /* set the version number for this module's data */
    fd->mod_ver[mod_id] = ver;

735
736
737
    return(0);
}

738
739
/* darshan_log_close()
 *
740
 * close an open darshan file descriptor, freeing any resources
741
742
743
 *
 */
void darshan_log_close(darshan_fd fd)
744
{
745
    struct darshan_fd_int_state *state = fd->state;
746
747
    int ret;

748
749
750
751
    assert(state);

    /* if the file was created for writing */
    if(state->creat_flag)
752
753
    {
        /* flush the last region of the log to file */
754
        switch(fd->comp_type)
755
756
        {
            case DARSHAN_ZLIB_COMP:
757
                ret = darshan_log_libz_flush(fd, state->dz.prev_reg_id);
758
759
                if(ret == 0)
                    break;
760
761
#ifdef HAVE_LIBBZ2
            case DARSHAN_BZIP2_COMP:
762
                ret = darshan_log_bzip2_flush(fd, state->dz.prev_reg_id);
763
764
765
                if(ret == 0)
                    break;
#endif 
766
767
            default:
                /* if flush fails, remove the output log file */
768
                state->err = -1;
769
770
771
772
773
                fprintf(stderr, "Error: final flush to log file failed.\n");
                break;
        }

        /* if no errors flushing, write the log header before closing */
774
        if(state->err != -1)
775
        {
776
            ret = darshan_log_put_header(fd);
777
            if(ret < 0)
778
                state->err = -1;
779
780
781
        }
    }

782
    close(state->fildes);
783
784

    /* remove output log file if error writing to it */
785
    if((state->creat_flag) && (state->err == -1))
786
787
    {
        fprintf(stderr, "Unlinking darshan log file %s ...\n",
788
789
            state->logfile_path);
        unlink(state->logfile_path);
790
    }
791

792
    darshan_log_dzdestroy(fd);
793
794
795
    if(state->exe_mnt_data)
        free(state->exe_mnt_data);
    free(state);
796
    free(fd);
797
798

    return;
799
800
}

801
/* **************************************************** */
802

803
804
805
806
807
808
809
810
811
812
813
814
815
static int darshan_mnt_info_cmp(const void *a, const void *b)
{
    struct darshan_mnt_info *m_a = (struct darshan_mnt_info *)a;
    struct darshan_mnt_info *m_b = (struct darshan_mnt_info *)b;

    if(strlen(m_a->mnt_path) > strlen(m_b->mnt_path))
        return(-1);
    else if(strlen(m_a->mnt_path) < strlen(m_b->mnt_path))
        return(1);
    else
        return(0);
}

816
817
818
819
820
/* read the header of the darshan log and set internal fd data structures
 * NOTE: this is the only portion of the darshan log that is uncompressed
 *
 * returns 0 on success, -1 on failure
 */
821
static int darshan_log_get_header(darshan_fd fd)
822
823
824
825
826
827
828
829
830
831
832
833
{
    struct darshan_header header;
    int i;
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
    /* read the version number so we know how to process this log */
    ret = darshan_log_read(fd, &fd->version, 8);
    if(ret < 8)
    {
        fprintf(stderr, "Error: invalid log file (failed to read version).\n");
        return(-1);
    }

    /* other log file versions can be detected and handled here */
    if(strcmp(fd->version, "3.00"))
    {
        fprintf(stderr, "Error: incompatible darshan file.\n");
        fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
        return(-1);
    }

    /* seek back so we can read the entire header */
    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

858
859
    /* read uncompressed header from log file */
    ret = darshan_log_read(fd, &header, sizeof(header));
860
    if(ret != (int)sizeof(header))
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
    {
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
        return(-1);
    }

    if(header.magic_nr == DARSHAN_MAGIC_NR)
    {
        /* no byte swapping needed, this file is in host format already */
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&(header.magic_nr));
        if(header.magic_nr == DARSHAN_MAGIC_NR)
        {
            fd->swap_flag = 1;

            /* swap the log map variables in the header */
880
881
            DARSHAN_BSWAP64(&(header.name_map.off));
            DARSHAN_BSWAP64(&(header.name_map.len));
882
883
884
885
886
887
888
889
890
891
892
893
894
895
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
            {
                DARSHAN_BSWAP64(&(header.mod_map[i].off));
                DARSHAN_BSWAP64(&(header.mod_map[i].len));
            }
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
    }

896
    /* set some fd fields based on what's stored in the header */
897
    fd->comp_type = header.comp_type;
898
    fd->partial_flag = header.partial_flag;
899
    memcpy(fd->mod_ver, header.mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
900