darshan-logutils.c 26.6 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
/* TODO: for log reads, we need to make sure the header has been read prior */

24
25
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
26
static int darshan_log_write(darshan_fd fd, void *buf, int len);
27
28
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
29
30
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz);
31
32
33
34

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
35
    NULL,               /* NULL */
36
    &posix_logutils,    /* POSIX */
37
38
39
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
40
41
42
43
44
45
46
47
48
49
50
51
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
52
53
54
55
56
57
58

/* darshan_log_open()
 *
 * open a darshan log file for reading/writing
 *
 * returns 0 on success, -1 on failure
 */
59
darshan_fd darshan_log_open(const char *name, const char *mode)
60
{
61
    darshan_fd tmp_fd;
62

63
    /* we only allow "w" or "r" modes, nothing fancy */
64
65
66
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');

67
68
69
70
71
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

72
73
    if(mode[0] == 'r')
    {
74
        tmp_fd->fildes = open(name, O_RDONLY);
75
76
77
    }
    else if (mode[0] == 'w')
    {
78
        /* TODO: permissions when creating?  umask */
79
80
81
        /* when writing, we create the log file making sure not to overwrite
         * an existing log
         */
82
        tmp_fd->fildes = open(name, O_WRONLY | O_CREAT | O_EXCL, 0400);
83
84
85
    }

    if(tmp_fd->fildes < 0)
86
    {
87
        perror("darshan_log_open: ");
88
        free(tmp_fd);
89
        tmp_fd = NULL;
90
    }
91
92

    return(tmp_fd);
93
94
}

95
96
97
98
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
99
 * NOTE: this is the only portion of the darshan log which is uncompressed
100
101
102
103
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
104
{
105
    int i;
106
    int ret;
107

108
    ret = darshan_log_seek(fd, 0);
109
    if(ret < 0)
110
111
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
112
        return(-1);
113
    }
114

115
    /* read uncompressed header from log file */
116
    ret = darshan_log_read(fd, header, sizeof(*header));
117
    if(ret != sizeof(*header))
118
    {
119
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
120
121
122
        return(-1);
    }

123
124
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
125

126
    if(header->magic_nr == DARSHAN_MAGIC_NR)
127
    {
128
        /* no byte swapping needed, this file is in host format already */
129
130
131
132
133
134
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
135
        if(header->magic_nr == DARSHAN_MAGIC_NR)
136
137
        {
            fd->swap_flag = 1;
138
139
140
141

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
142
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
143
144
145
146
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
147
148
149
150
151
152
153
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
154
155
    }

156
    /* save the mapping of data within log file to this file descriptor */
157
158
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
159
160
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
161
162

    return(0);
163
}
164

165
166
167
/* darshan_log_putheader()
 *
 * write a darshan header to log file
168
169
 * NOTE: the header is not passed in, and is instead built using
 * contents of the given file descriptor
170
171
172
 *
 * returns 0 on success, -1 on failure
 */
173
int darshan_log_putheader(darshan_fd fd)
174
{
175
    struct darshan_header header;
176
177
178
179
180
181
182
183
184
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

185
186
187
188
189
190
191
192
    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

193
    /* write header to file */
194
195
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
196
197
198
199
200
201
202
203
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

204
/* darshan_log_getjob()
205
206
 *
 * read job level metadata from the darshan log file
207
 *
208
 * returns 0 on success, -1 on failure
209
 */
210
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
211
{
212
213
214
    char *comp_buf;
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
215
    int ret;
216

217
218
219
220
221
222
    /* allocate buffer to store compressed job info */
    comp_buf = malloc(fd->job_map.len);
    if(!comp_buf)
        return(-1);

    ret = darshan_log_seek(fd, fd->job_map.off);
223
    if(ret < 0)
224
    {
225
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
226
        free(comp_buf);
227
        return(-1);
228
229
    }

230
231
232
    /* read the compressed job data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
    if(ret != fd->job_map.len)
233
    {
234
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
235
236
237
238
239
240
241
242
243
244
        free(comp_buf);
        return(-1);
    }

    /* decompress the job data */
    ret = darshan_decompress_buf(comp_buf, fd->job_map.len, job_buf, &job_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
245
246
        return(-1);
    }
247
248
249
    free(comp_buf);

    memcpy(job, job_buf, sizeof(*job));
250

251
    if(fd->swap_flag)
252
    {
253
254
255
256
257
258
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
259
260
    }

261
262
263
264
265
266
267
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

268
269
270
    return(0);
}

271
272
273
274
275
276
277
278
279
/* darshan_log_putjob()
 *
 * write job level metadat to darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
280
281
    char *comp_buf;
    int comp_buf_sz;
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
    int len;
    int ret;

    ret = darshan_log_seek(fd, sizeof(struct darshan_header));
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

306
307
308
309
310
311
312
313
314
315
316
317
318
319
    comp_buf = malloc(sizeof(*job));
    if(!comp_buf)
        return(-1);
    comp_buf_sz = sizeof(*job);

    /* compress the job data */
    ret = darshan_compress_buf((char*)&job_copy, sizeof(*job), comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
        return(-1);
    }    

320
    /* write job data to file */
321
322
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    {
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
337
338
339
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
340
    int ret;
341

342
343
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
344
    {
345
346
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
347

348
349
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
350
    }
351

352
353
354
355
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
356
    if(newline)
357
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
358
359
360
361

    return (0);
}

362
363
364
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
365
366
367
 * NOTE: this needs to be called immediately following put_job as it
 * expects the final pointer to be positioned immediately following
 * the darshan job information
368
369
370
371
372
373
374
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
375
376
377
378
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;

    len = strlen(buf);
379

380
381
    /* compress the input exe string */
    ret = darshan_compress_buf(buf, len, comp_buf, &comp_buf_sz);
382
383
    if(ret < 0)
    {
384
        fprintf(stderr, "Error: unable to compress exe string.\n");
385
386
387
        return(-1);
    }

388
389
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
390
391
392
393
394
395
396
397
    {
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

398
399
/* darshan_log_getmounts()
 * 
400
 * retrieves mount table information from the log. Note that mnt_pts and
401
 * fs_types are arrays that will be allocated by the function and must be
402
403
404
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
405
 */
406
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
407
408
409
410
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
411
    int ret;
412

413
414
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
415
    {
416
417
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
418

419
420
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
421
    }
422

423
    /* count entries */
424
    *count = 0;
425
    pos = fd->exe_mnt_data;
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
447
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
448
449
    {
        /* overestimate string lengths */
450
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
451
        assert((*mnt_pts)[array_index]);
452
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
453
454
        assert((*fs_types)[array_index]);

455
456
457
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
458
        {
459
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
460
461
462
463
464
465
466
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

467
468
469
    return(0);
}

470
471
472
473
474
475
476
477
478
479
480
481
482
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
483
484
485
486
487
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;
    char *tmp;
488
489
490
    int ret;

    /* write each mount entry to file */
491
    tmp = mnt_dat;
492
493
494
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
495
496
497
498
499
500
501
502
503
504
505

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

    ret = darshan_compress_buf(mnt_dat, mnt_dat_sz, comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress mount data.\n");
        return(-1);
506
507
    }

508
509
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if (ret != comp_buf_sz)
510
    {
511
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
512
513
514
515
516
517
        return(-1);
    }

    return(0);
}

518
/* darshan_log_gethash()
519
 *
520
 * read the hash of records from the darshan log file
521
522
523
 *
 * returns 0 on success, -1 on failure
 */
524
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
525
{
526
    char *comp_buf;
527
    char *hash_buf;
528
    int hash_buf_sz;
529
    char *buf_ptr;
530
531
532
533
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
534
535
    int ret;

536
537
538
539
540
541
542
543
544
545
    /* allocate buffers to store compressed and decompressed record
     * hash data
     */
    comp_buf = malloc(fd->rec_map.len);
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(!comp_buf || !hash_buf)
        return(-1);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);

546
    ret = darshan_log_seek(fd, fd->rec_map.off);
547
    if(ret < 0)
548
549
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
550
        free(comp_buf);
551
        return(-1);
552
    }
553

554
    /* read the record hash from the log file */
555
    ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
556
    if(ret != fd->rec_map.len)
557
    {
558
        fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
559
560
561
562
563
564
565
566
567
568
569
        free(comp_buf);
        return(-1);
    }

    /* decompress the record hash buffer */
    ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
        hash_buf, &hash_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress darshan job data.\n");
        free(comp_buf);
570
571
        return(-1);
    }
572
    free(comp_buf);
573

574
    buf_ptr = hash_buf;
575
    while(buf_ptr < (hash_buf + hash_buf_sz))
576
    {
577
        /* get pointers for each field of this darshan record */
578
        /* NOTE: darshan record hash serialization method: 
579
580
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
581
582
583
584
585
586
587
588
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        if(fd->swap_flag)
589
        {
590
591
592
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
593
594
        }

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
            {
                return(-1);
            }
            ref->rec.name = malloc(*path_len_ptr + 1);
            if(!ref->rec.name)
            {
                free(ref);
                return(-1);
            }

            /* set the fields for this record */
            ref->rec.id = *rec_id_ptr;
            memcpy(ref->rec.name, path_ptr, *path_len_ptr);
            ref->rec.name[*path_len_ptr] = '\0';
614

615
616
617
            /* add this record to the hash */
            HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
        }
618
619
    }

620
    free(hash_buf);
621
622
623
    return(0);
}

624
625
626
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
627
628
629
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
630
631
632
633
634
635
636
637
638
639
640
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    size_t hash_buf_sz;
    char *hash_buf;
    char *hash_buf_off;
    struct darshan_record_ref *ref, *tmp;
    uint32_t name_len;
    size_t record_sz;
641
642
    char *comp_buf;
    char comp_buf_sz;
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
    int ret;

    /* allocate a buffer to store 2 MiB worth of record data */
    /* NOTE: this buffer may be reallocated if estimate is too small */
    hash_buf_sz = 2 * 1024 * 1024;
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
    {
        return(-1);
    }

    /* serialize the record hash into a buffer for writing */
    hash_buf_off = hash_buf;
    HASH_ITER(hlink, hash, ref, tmp)
    {
        name_len = strlen(ref->rec.name);
        record_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + name_len;
        /* make sure there is room in the buffer for this record */
        if((hash_buf_off + record_sz) > (hash_buf + hash_buf_sz))
        {
            char *tmp_buf;
            size_t old_buf_sz;

            /* if no room, reallocate the hash buffer at twice the current size */
            old_buf_sz = hash_buf_off - hash_buf;
            hash_buf_sz *= 2;
            tmp_buf = malloc(hash_buf_sz);
            if(!tmp_buf)
            {
                free(hash_buf);
                return(-1);
            }

            memcpy(tmp_buf, hash_buf, old_buf_sz);
            free(hash_buf);
            hash_buf = tmp_buf;
            hash_buf_off = hash_buf + old_buf_sz;
        }

        /* now serialize the record into the hash buffer.
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        *((darshan_record_id *)hash_buf_off) = ref->rec.id;
        hash_buf_off += sizeof(darshan_record_id);
        *((uint32_t *)hash_buf_off) = name_len;
        hash_buf_off += sizeof(uint32_t);
        memcpy(hash_buf_off, ref->rec.name, name_len);
        hash_buf_off += name_len;
    }
    hash_buf_sz = hash_buf_off - hash_buf;

695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
    comp_buf = malloc(hash_buf_sz);
    if(!comp_buf)
        return(-1);
    comp_buf_sz = hash_buf_sz;

    /* compress the record hash */
    ret = darshan_compress_buf(hash_buf, hash_buf_sz, comp_buf, &comp_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress darshan record hash.\n");
        return(-1);
    }

    /* set the appropriate mapping info for the record hash in the file descriptor */
    fd->rec_map.off = fd->pos;
    fd->rec_map.len = comp_buf_sz;

712
    /* write the record hash to file */
713
714
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
715
716
717
718
719
720
721
722
723
724
725
726
    {
        fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
        return(-1);
    }

    free(hash_buf);

    return(0);
}

/* darshan_log_getmod()
 *
727
 * returns 1 on successful read of module data, 0 on no data, -1 on failure
728
729
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
730
    void *mod_buf, int *mod_buf_sz)
731
{
732
    char *comp_buf;
733
    int ret;
734

735
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
736
    {
737
738
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
739
740
    }

741
742
743
744
745
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    comp_buf = malloc(fd->mod_map[mod_id].len);
    if(!comp_buf)
746
        return(-1);
747
748
749
750
751
752
753

    ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        free(comp_buf);
        return(ret);
754
755
    }

756
757
758
    /* read this module's data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
    if(ret != fd->mod_map[mod_id].len)
759
    {
760
        fprintf(stderr,
761
            "Error: failed to read module %s data from darshan log file.\n",
762
            darshan_module_names[mod_id]);
763
        free(comp_buf);
764
765
766
        return(-1);
    }

767
768
769
770
771
772
773
774
775
776
777
778
    /* decompress this module's data */
    ret = darshan_decompress_buf(comp_buf, fd->mod_map[mod_id].len,
        mod_buf, mod_buf_sz);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
        return(-1);
    }
    free(comp_buf);

779
    return(1);
780
781
}

782
/* XXX */
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
    int ret;

    /* only seek to start of module data if current log file position 
     * is not within the given mod_id's range. This allows one to
     * repeatedly call this function and put chunks of a module's
     * data piecemeal.
     */
    if((fd->pos < fd->mod_map[mod_id].off) || (fd->pos > mod_buf_end))
    {
        ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
        if(ret < 0)
        {
            fprintf(stderr, "Error: unable to seek in darshan log file.\n");
            return(-1);
        }
    }

    /* if the given data violates stored mapping info, error out */
    if(mod_buf_sz > (mod_buf_end - fd->pos))
    {
        fprintf(stderr, "Error: module data write violates stored mapping information.\n");
        return(-1);
    }

    /* write the module chunk to the log file */
    ret = darshan_log_write(fd, mod_buf, mod_buf_sz);
    if(ret != mod_buf_sz)
    {
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

829
830
831
832
833
834
835
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
836
{
837
838
839
840
841
    if(fd->fildes)
        close(fd->fildes);

    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
842

843
    free(fd);
844
845

    return;
846
847
}

848
/* **************************************************** */
849

850
851
852
853
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
854
    off_t ret_off;
855
856
857
858

    if(fd->pos == offset)
        return(0);

859
860
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
861
    {
862
863
        fd->pos = offset;
        return(0);
864
865
866
867
868
    }

    return(-1);
}

869
/* return amount read on success, 0 on EOF, -1 on failure.
870
 */
871
static int darshan_log_read(darshan_fd fd, void* buf, int len)
872
873
874
{
    int ret;

875
876
877
878
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
879

880
    return(ret);
881
882
}

883
/* return amount written on success, -1 on failure.
884
 */
885
static int darshan_log_write(darshan_fd fd, void* buf, int len)
886
887
888
{
    int ret;

889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

/* return 0 on successful decompression, -1 on failure
 */
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz)
{
    int ret;
    int total_out = 0;
    z_stream tmp_stream;

    memset(&tmp_stream, 0, sizeof(tmp_stream));
    tmp_stream.zalloc = Z_NULL;
    tmp_stream.zfree = Z_NULL;
    tmp_stream.opaque = Z_NULL;
    tmp_stream.next_in = (unsigned char*)comp_buf;
    tmp_stream.avail_in = comp_buf_sz;
    tmp_stream.next_out = (unsigned char*)decomp_buf;
    tmp_stream.avail_out = *inout_decomp_buf_sz;

    /* initialize the zlib decompression parameters */
    /* TODO: check these parameters? */
    //ret = inflateInit2(&tmp_stream, 31);
    ret = inflateInit(&tmp_stream);
    if(ret != Z_OK)
919
    {
920
        return(-1);
921
922
    }

923
924
925
926
927
    /* while we have not finished consuming all of the compressed input data */
    while(tmp_stream.avail_in)
    {
        if(tmp_stream.avail_out == 0)
        {
928
            /* We ran out of buffer space for decompression.  In theory,
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
             * we could just alloc more space, but probably just easier
             * to bump up the default size of the output buffer.
             */
            inflateEnd(&tmp_stream);
            return(-1);
        }

        /* decompress data */
        ret = inflate(&tmp_stream, Z_FINISH);
        if(ret != Z_STREAM_END)
        {
            inflateEnd(&tmp_stream);
            return(-1);
        }

        total_out += tmp_stream.total_out;
        if(tmp_stream.avail_in)
            inflateReset(&tmp_stream);
    }
    inflateEnd(&tmp_stream);

    *inout_decomp_buf_sz = total_out;
    return(0);
}

static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz)
{
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
    int ret;
    int total_out = 0;
    z_stream tmp_stream;

    memset(&tmp_stream, 0, sizeof(tmp_stream));
    tmp_stream.zalloc = Z_NULL;
    tmp_stream.zfree = Z_NULL;
    tmp_stream.opaque = Z_NULL;
    tmp_stream.next_in = (unsigned char*)decomp_buf;
    tmp_stream.avail_in = decomp_buf_sz;
    tmp_stream.next_out = (unsigned char*)comp_buf;
    tmp_stream.avail_out = *inout_comp_buf_sz;

    ret = deflateInit(&tmp_stream, Z_DEFAULT_COMPRESSION);
    if(ret != Z_OK)
    {
        return(-1);
    }

    /* while we have not finished consuming all of the uncompressed input data */
    while(tmp_stream.avail_in)
    {
        if(tmp_stream.avail_out == 0)
        {
            /* We ran out of buffer space for compression.  In theory,
             * we could just alloc more space, but probably just easier
             * to bump up the default size of the output buffer.
             */
            deflateEnd(&tmp_stream);
            return(-1);
        }

        /* compress data */
        ret = deflate(&tmp_stream, Z_FINISH);
        if(ret != Z_STREAM_END)
        {
            deflateEnd(&tmp_stream);
            return(-1);
        }

        total_out += tmp_stream.total_out;
        if(tmp_stream.avail_in)
            deflateReset(&tmp_stream);
    }
    deflateEnd(&tmp_stream);
1002

1003
    *inout_comp_buf_sz = total_out;
1004
    return(0);
1005
}
1006
1007
1008
1009
1010
1011
1012
1013
1014

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */