darshan-logutils.c 31.6 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
24
static int darshan_log_write(darshan_fd fd, void *buf, int len);
25
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
26
27
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type);
28
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
29
30
31
32
33
34
35
36
37
38
    char* comp_buf, int* inout_comp_buf_sz,
    enum darshan_comp_type comp_type);
static int darshan_zlib_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_zlib_comp(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz);
#ifdef HAVE_LIBBZ2
static int darshan_bzip2_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_bzip2_comp(char* decomp_buf, int decomp_buf_sz,
39
    char* comp_buf, int* inout_comp_buf_sz);
40
41
#endif

42
43
44
45

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
46
    NULL,               /* NULL */
47
    &posix_logutils,    /* POSIX */
48
49
50
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
51
52
53
54
55
56
57
58
59
60
61
62
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
63
64
65

/* darshan_log_open()
 *
66
 * open an existing darshan log file for reading only
67
68
69
 *
 * returns 0 on success, -1 on failure
 */
70
darshan_fd darshan_log_open(const char *name)
71
{
72
    darshan_fd tmp_fd;
73

74
75
76
77
78
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

79
80
    tmp_fd->fildes = open(name, O_RDONLY);
    if(tmp_fd->fildes < 0)
81
    {
82
83
84
        perror("darshan_log_open: ");
        free(tmp_fd);
        tmp_fd = NULL;
85
86
    }

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
 * returns 0 on success, -1 on failure
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

    /* TODO: permissions when creating?  umask */
    /* when writing, we create the log file making sure not to overwrite
     * an existing log
     */
    tmp_fd->comp_type = comp_type;
    tmp_fd->fildes = open(name, O_WRONLY | O_CREAT | O_EXCL, 0400);
111
    if(tmp_fd->fildes < 0)
112
    {
113
        perror("darshan_log_create: ");
114
        free(tmp_fd);
115
        tmp_fd = NULL;
116
    }
117
118

    return(tmp_fd);
119
120
}

121
122
123
124
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
125
 * NOTE: this is the only portion of the darshan log which is uncompressed
126
127
128
129
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
130
{
131
    int i;
132
    int ret;
133

134
    ret = darshan_log_seek(fd, 0);
135
    if(ret < 0)
136
137
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
138
        return(-1);
139
    }
140

141
    /* read uncompressed header from log file */
142
    ret = darshan_log_read(fd, header, sizeof(*header));
143
    if(ret != sizeof(*header))
144
    {
145
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
146
147
148
        return(-1);
    }

149
150
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
151

152
    if(header->magic_nr == DARSHAN_MAGIC_NR)
153
    {
154
        /* no byte swapping needed, this file is in host format already */
155
156
157
158
159
160
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
161
        if(header->magic_nr == DARSHAN_MAGIC_NR)
162
163
        {
            fd->swap_flag = 1;
164
165
166
167

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
168
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
169
170
171
172
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
173
174
175
176
177
178
179
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
180
181
    }

182
183
    fd->comp_type = header->comp_type;

184
    /* save the mapping of data within log file to this file descriptor */
185
186
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
187
188
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
189
190

    return(0);
191
}
192

193
194
195
/* darshan_log_putheader()
 *
 * write a darshan header to log file
196
197
 * NOTE: the header is not passed in, and is instead built using
 * contents of the given file descriptor
198
199
200
 *
 * returns 0 on success, -1 on failure
 */
201
int darshan_log_putheader(darshan_fd fd)
202
{
203
    struct darshan_header header;
204
205
206
207
208
209
210
211
212
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

213
214
215
    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
Shane Snyder's avatar
Shane Snyder committed
216
    header.comp_type = fd->comp_type;
217
218
219
220
221

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

222
    /* write header to file */
223
224
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
225
226
227
228
229
230
231
232
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

233
/* darshan_log_getjob()
234
235
 *
 * read job level metadata from the darshan log file
236
 *
237
 * returns 0 on success, -1 on failure
238
 */
239
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
240
{
241
242
243
    char *comp_buf;
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
244
    int ret;
245

246
247
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

248
249
250
251
252
253
    /* allocate buffer to store compressed job info */
    comp_buf = malloc(fd->job_map.len);
    if(!comp_buf)
        return(-1);

    ret = darshan_log_seek(fd, fd->job_map.off);
254
    if(ret < 0)
255
    {
256
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
257
        free(comp_buf);
258
        return(-1);
259
260
    }

261
262
263
    /* read the compressed job data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
    if(ret != fd->job_map.len)
264
    {
265
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
266
267
268
269
270
        free(comp_buf);
        return(-1);
    }

    /* decompress the job data */
271
272
    ret = darshan_decompress_buf(comp_buf, fd->job_map.len,
        job_buf, &job_buf_sz, fd->comp_type);
273
274
275
276
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
277
278
        return(-1);
    }
279
280
281
    free(comp_buf);

    memcpy(job, job_buf, sizeof(*job));
282

283
    if(fd->swap_flag)
284
    {
285
286
287
288
289
290
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
291
292
    }

293
294
295
296
297
298
299
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

300
301
302
    return(0);
}

303
304
305
306
307
308
309
310
311
/* darshan_log_putjob()
 *
 * write job level metadat to darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
312
313
    char *comp_buf;
    int comp_buf_sz;
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    int len;
    int ret;

    ret = darshan_log_seek(fd, sizeof(struct darshan_header));
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

338
339
340
341
342
343
    comp_buf = malloc(sizeof(*job));
    if(!comp_buf)
        return(-1);
    comp_buf_sz = sizeof(*job);

    /* compress the job data */
344
345
    ret = darshan_compress_buf((char*)&job_copy, sizeof(*job),
        comp_buf, &comp_buf_sz, fd->comp_type);
346
347
348
349
350
351
352
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
        return(-1);
    }    

353
    /* write job data to file */
354
355
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
356
357
358
359
360
361
362
363
364
365
366
367
368
369
    {
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
        return(-1);
    }

    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
370
371
372
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
373
    int ret;
374

375
376
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
377
    {
378
379
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
380

381
382
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
383
    }
384

385
386
387
388
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
389
    if(newline)
390
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
391
392
393
394

    return (0);
}

395
396
397
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
398
399
400
 * NOTE: this needs to be called immediately following put_job as it
 * expects the final pointer to be positioned immediately following
 * the darshan job information
401
402
403
404
405
406
407
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
408
409
410
411
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;

    len = strlen(buf);
412

413
    /* compress the input exe string */
414
    ret = darshan_compress_buf(buf, len, comp_buf, &comp_buf_sz, fd->comp_type);
415
416
    if(ret < 0)
    {
417
        fprintf(stderr, "Error: unable to compress exe string.\n");
418
419
420
        return(-1);
    }

421
422
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
423
424
425
426
427
428
429
430
    {
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

431
432
/* darshan_log_getmounts()
 * 
433
 * retrieves mount table information from the log. Note that mnt_pts and
434
 * fs_types are arrays that will be allocated by the function and must be
435
436
437
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
438
 */
439
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
440
441
442
443
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
444
    int ret;
445

446
447
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
448
    {
449
450
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
451

452
453
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
454
    }
455

456
    /* count entries */
457
    *count = 0;
458
    pos = fd->exe_mnt_data;
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
480
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
481
482
    {
        /* overestimate string lengths */
483
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
484
        assert((*mnt_pts)[array_index]);
485
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
486
487
        assert((*fs_types)[array_index]);

488
489
490
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
491
        {
492
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
493
494
495
496
497
498
499
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

500
501
502
    return(0);
}

503
504
505
506
507
508
509
510
511
512
513
514
515
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
516
517
518
519
520
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;
    char *tmp;
521
522
523
    int ret;

    /* write each mount entry to file */
524
    tmp = mnt_dat;
525
526
527
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
528
529
530
531
532
533

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

534
    ret = darshan_compress_buf(mnt_dat, mnt_dat_sz, comp_buf, &comp_buf_sz, fd->comp_type);
535
536
537
538
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress mount data.\n");
        return(-1);
539
540
    }

541
542
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if (ret != comp_buf_sz)
543
    {
544
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
545
546
547
548
549
550
        return(-1);
    }

    return(0);
}

551
/* darshan_log_gethash()
552
 *
553
 * read the hash of records from the darshan log file
554
555
556
 *
 * returns 0 on success, -1 on failure
 */
557
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
558
{
559
    char *comp_buf;
560
    char *hash_buf;
561
    int hash_buf_sz;
562
    char *buf_ptr;
563
564
565
566
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
567
568
    int ret;

569
570
571
572
573
574
575
576
577
578
    /* allocate buffers to store compressed and decompressed record
     * hash data
     */
    comp_buf = malloc(fd->rec_map.len);
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(!comp_buf || !hash_buf)
        return(-1);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);

579
    ret = darshan_log_seek(fd, fd->rec_map.off);
580
    if(ret < 0)
581
582
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
583
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
584
        free(hash_buf);
585
        return(-1);
586
    }
587

588
    /* read the record hash from the log file */
589
    ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
590
    if(ret != fd->rec_map.len)
591
    {
592
        fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
593
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
594
        free(hash_buf);
595
596
597
598
599
        return(-1);
    }

    /* decompress the record hash buffer */
    ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
600
        hash_buf, &hash_buf_sz, fd->comp_type);
601
602
603
604
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress darshan job data.\n");
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
605
        free(hash_buf);
606
607
        return(-1);
    }
608
    free(comp_buf);
609

610
    buf_ptr = hash_buf;
611
    while(buf_ptr < (hash_buf + hash_buf_sz))
612
    {
613
        /* get pointers for each field of this darshan record */
614
        /* NOTE: darshan record hash serialization method: 
615
616
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
617
618
619
620
621
622
623
624
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        if(fd->swap_flag)
625
        {
626
627
628
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
629
630
        }

631
632
633
634
635
636
        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
            {
Shane Snyder's avatar
Shane Snyder committed
637
                free(hash_buf);
638
639
640
641
642
643
                return(-1);
            }
            ref->rec.name = malloc(*path_len_ptr + 1);
            if(!ref->rec.name)
            {
                free(ref);
Shane Snyder's avatar
Shane Snyder committed
644
                free(hash_buf);
645
646
647
648
649
650
651
                return(-1);
            }

            /* set the fields for this record */
            ref->rec.id = *rec_id_ptr;
            memcpy(ref->rec.name, path_ptr, *path_len_ptr);
            ref->rec.name[*path_len_ptr] = '\0';
652

653
654
655
            /* add this record to the hash */
            HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
        }
656
657
    }

658
    free(hash_buf);
659
660
661
    return(0);
}

662
663
664
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
665
666
667
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
668
669
670
671
672
673
674
675
676
677
678
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    size_t hash_buf_sz;
    char *hash_buf;
    char *hash_buf_off;
    struct darshan_record_ref *ref, *tmp;
    uint32_t name_len;
    size_t record_sz;
679
    char *comp_buf;
Shane Snyder's avatar
Shane Snyder committed
680
    int comp_buf_sz;
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
    int ret;

    /* allocate a buffer to store 2 MiB worth of record data */
    /* NOTE: this buffer may be reallocated if estimate is too small */
    hash_buf_sz = 2 * 1024 * 1024;
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
    {
        return(-1);
    }

    /* serialize the record hash into a buffer for writing */
    hash_buf_off = hash_buf;
    HASH_ITER(hlink, hash, ref, tmp)
    {
        name_len = strlen(ref->rec.name);
        record_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + name_len;
        /* make sure there is room in the buffer for this record */
        if((hash_buf_off + record_sz) > (hash_buf + hash_buf_sz))
        {
            char *tmp_buf;
            size_t old_buf_sz;

            /* if no room, reallocate the hash buffer at twice the current size */
            old_buf_sz = hash_buf_off - hash_buf;
            hash_buf_sz *= 2;
            tmp_buf = malloc(hash_buf_sz);
            if(!tmp_buf)
            {
                free(hash_buf);
                return(-1);
            }

            memcpy(tmp_buf, hash_buf, old_buf_sz);
            free(hash_buf);
            hash_buf = tmp_buf;
            hash_buf_off = hash_buf + old_buf_sz;
        }

        /* now serialize the record into the hash buffer.
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        *((darshan_record_id *)hash_buf_off) = ref->rec.id;
        hash_buf_off += sizeof(darshan_record_id);
        *((uint32_t *)hash_buf_off) = name_len;
        hash_buf_off += sizeof(uint32_t);
        memcpy(hash_buf_off, ref->rec.name, name_len);
        hash_buf_off += name_len;
    }
    hash_buf_sz = hash_buf_off - hash_buf;

Shane Snyder's avatar
Shane Snyder committed
733
    comp_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
734
735
    if(!comp_buf)
        return(-1);
Shane Snyder's avatar
Shane Snyder committed
736
    comp_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
737
738

    /* compress the record hash */
739
740
    ret = darshan_compress_buf(hash_buf, hash_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
741
742
743
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress darshan record hash.\n");
Shane Snyder's avatar
Shane Snyder committed
744
745
        free(comp_buf);
        free(hash_buf);
746
747
748
749
750
751
752
        return(-1);
    }

    /* set the appropriate mapping info for the record hash in the file descriptor */
    fd->rec_map.off = fd->pos;
    fd->rec_map.len = comp_buf_sz;

753
    /* write the record hash to file */
754
755
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
756
757
    {
        fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
Shane Snyder's avatar
Shane Snyder committed
758
759
        free(comp_buf);
        free(hash_buf);
760
761
762
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
763
    free(comp_buf);
764
765
766
767
768
769
770
    free(hash_buf);

    return(0);
}

/* darshan_log_getmod()
 *
771
 * returns 1 on successful read of module data, 0 on no data, -1 on failure
772
773
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
774
    void *mod_buf, int *mod_buf_sz)
775
{
776
    char *comp_buf;
777
    int ret;
778

779
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
780
    {
781
782
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
783
784
    }

785
786
787
788
789
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    comp_buf = malloc(fd->mod_map[mod_id].len);
    if(!comp_buf)
790
        return(-1);
791
792
793
794
795
796
797

    ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        free(comp_buf);
        return(ret);
798
799
    }

800
801
802
    /* read this module's data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
    if(ret != fd->mod_map[mod_id].len)
803
    {
804
        fprintf(stderr,
805
            "Error: failed to read module %s data from darshan log file.\n",
806
            darshan_module_names[mod_id]);
807
        free(comp_buf);
808
809
810
        return(-1);
    }

811
812
    /* decompress this module's data */
    ret = darshan_decompress_buf(comp_buf, fd->mod_map[mod_id].len,
813
        mod_buf, mod_buf_sz, fd->comp_type);
814
815
816
817
818
819
820
821
822
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
        return(-1);
    }
    free(comp_buf);

823
    return(1);
824
825
}

826
827
828
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
829
830
831
832
833
834
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
835
836
837
838
839
840
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
Shane Snyder's avatar
Shane Snyder committed
841
842
    char *comp_buf;
    int comp_buf_sz;
843
844
    int ret;

Shane Snyder's avatar
Shane Snyder committed
845
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
846
    {
Shane Snyder's avatar
Shane Snyder committed
847
848
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
849
850
    }

Shane Snyder's avatar
Shane Snyder committed
851
852
853
854
855
856
    comp_buf = malloc(mod_buf_sz);
    if(!comp_buf)
        return(-1);
    comp_buf_sz = mod_buf_sz;

    /* compress the module's data */
857
858
    ret = darshan_compress_buf(mod_buf, mod_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
Shane Snyder's avatar
Shane Snyder committed
859
    if(ret < 0)
860
    {
Shane Snyder's avatar
Shane Snyder committed
861
862
863
        fprintf(stderr, "Error: unable to compress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
864
865
866
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
867
868
869
870
    /* set the appropriate mapping information for this module */
    fd->mod_map[mod_id].off = fd->pos;
    fd->mod_map[mod_id].len = comp_buf_sz;

871
    /* write the module chunk to the log file */
Shane Snyder's avatar
Shane Snyder committed
872
873
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
874
875
876
877
878
879
880
881
882
883
    {
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
        return(-1);
    }

    return(0);
}

884
885
886
887
888
889
890
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
891
{
892
893
894
895
896
    if(fd->fildes)
        close(fd->fildes);

    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
897

898
    free(fd);
899
900

    return;
901
902
}

903
/* **************************************************** */
904

905
906
907
908
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
909
    off_t ret_off;
910
911
912
913

    if(fd->pos == offset)
        return(0);

914
915
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
916
    {
917
918
        fd->pos = offset;
        return(0);
919
920
921
922
923
    }

    return(-1);
}

924
/* return amount read on success, 0 on EOF, -1 on failure.
925
 */
926
static int darshan_log_read(darshan_fd fd, void* buf, int len)
927
928
929
{
    int ret;

930
931
932
933
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
934

935
    return(ret);
936
937
}

938
/* return amount written on success, -1 on failure.
939
 */
940
static int darshan_log_write(darshan_fd fd, void* buf, int len)
941
942
943
{
    int ret;

944
945
946
947
948
949
950
951
952
953
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

/* return 0 on successful decompression, -1 on failure
 */
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type)
{
    int ret;

    switch(comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_zlib_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_bzip2_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#endif
        default:
            fprintf(stderr, "Error: invalid decompression method.\n");
            return(-1);
    }

    return(ret);
}

Shane Snyder's avatar
Shane Snyder committed
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz,
    enum darshan_comp_type comp_type)
{
    int ret;

    switch(comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_zlib_comp(decomp_buf, decomp_buf_sz,
                comp_buf, inout_comp_buf_sz);
            break;
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_bzip2_comp(decomp_buf, decomp_buf_sz,
                comp_buf, inout_comp_buf_sz);
            break;
#endif
        default:
            fprintf(stderr, "Error: invalid compression method.\n");
            return(-1);
    }
For faster browsing, not all history is shown. View entire blame