darshan-logutils.c 31.7 KB
Newer Older
1
/*
Shane Snyder's avatar
Shane Snyder committed
2
3
4
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
5
6
 */

7
#define _GNU_SOURCE
8
#include "darshan-util-config.h"
9
10
#include <stdio.h>
#include <string.h>
11
#include <assert.h>
12
#include <stdlib.h>
13
#include <unistd.h>
14
#include <inttypes.h>
15
16
17
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
18
#include <errno.h>
19

20
21
#include "darshan-logutils.h"

22
23
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
24
static int darshan_log_write(darshan_fd fd, void *buf, int len);
25
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
26
27
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type);
28
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
29
30
31
32
33
34
35
36
37
38
    char* comp_buf, int* inout_comp_buf_sz,
    enum darshan_comp_type comp_type);
static int darshan_zlib_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_zlib_comp(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz);
#ifdef HAVE_LIBBZ2
static int darshan_bzip2_decomp(char* comp_buf, int comp_buf_sz,
    char* decomp_buf, int* inout_decomp_buf_sz);
static int darshan_bzip2_comp(char* decomp_buf, int decomp_buf_sz,
39
    char* comp_buf, int* inout_comp_buf_sz);
40
41
#endif

42
43
44
45

/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
{
46
    NULL,               /* NULL */
47
    &posix_logutils,    /* POSIX */
48
49
50
    &mpiio_logutils,    /* MPI-IO */
    &hdf5_logutils,     /* HDF5 */
    &pnetcdf_logutils,  /* PNETCDF */
51
52
53
54
55
56
57
58
59
60
61
62
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};
63
64
65

/* darshan_log_open()
 *
66
 * open an existing darshan log file for reading only
67
68
69
 *
 * returns 0 on success, -1 on failure
 */
70
darshan_fd darshan_log_open(const char *name)
71
{
72
    darshan_fd tmp_fd;
73

74
75
76
77
78
    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

79
80
    tmp_fd->fildes = open(name, O_RDONLY);
    if(tmp_fd->fildes < 0)
81
    {
82
83
84
        perror("darshan_log_open: ");
        free(tmp_fd);
        tmp_fd = NULL;
85
86
    }

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    return(tmp_fd);
}

/* darshan_log_create()
 *
 * create a darshan log file for writing with the given compression method
 *
 * returns 0 on success, -1 on failure
 */
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type)
{
    darshan_fd tmp_fd;

    tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

    /* TODO: permissions when creating?  umask */
    /* when writing, we create the log file making sure not to overwrite
     * an existing log
     */
    tmp_fd->comp_type = comp_type;
    tmp_fd->fildes = open(name, O_WRONLY | O_CREAT | O_EXCL, 0400);
111
    if(tmp_fd->fildes < 0)
112
    {
113
        perror("darshan_log_create: ");
114
        free(tmp_fd);
115
        tmp_fd = NULL;
116
    }
117
118

    return(tmp_fd);
119
120
}

121
122
123
124
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
125
 * NOTE: this is the only portion of the darshan log which is uncompressed
126
127
128
129
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
130
{
131
    int i;
132
    int ret;
133

134
    ret = darshan_log_seek(fd, 0);
135
    if(ret < 0)
136
137
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
138
        return(-1);
139
    }
140

141
    /* read uncompressed header from log file */
142
    ret = darshan_log_read(fd, header, sizeof(*header));
143
    if(ret != sizeof(*header))
144
    {
145
        fprintf(stderr, "Error: failed to read darshan log file header.\n");
146
147
148
        return(-1);
    }

149
150
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
151

152
    if(header->magic_nr == DARSHAN_MAGIC_NR)
153
    {
154
        /* no byte swapping needed, this file is in host format already */
155
156
157
158
159
160
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
161
        if(header->magic_nr == DARSHAN_MAGIC_NR)
162
163
        {
            fd->swap_flag = 1;
164
165
166
167

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
168
            for(i = 0; i < DARSHAN_MAX_MODS; i++)
169
170
171
172
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
173
174
175
176
177
178
179
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
180
181
    }

182
183
    fd->comp_type = header->comp_type;

184
    /* save the mapping of data within log file to this file descriptor */
185
186
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
187
188
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
189
190

    return(0);
191
}
192

193
194
195
/* darshan_log_putheader()
 *
 * write a darshan header to log file
196
197
 * NOTE: the header is not passed in, and is instead built using
 * contents of the given file descriptor
198
199
200
 *
 * returns 0 on success, -1 on failure
 */
201
int darshan_log_putheader(darshan_fd fd)
202
{
203
    struct darshan_header header;
204
205
206
207
208
209
210
211
212
    int ret;

    ret = darshan_log_seek(fd, 0);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

213
214
215
    memset(&header, 0, sizeof(header));
    strcpy(header.version_string, DARSHAN_LOG_VERSION);
    header.magic_nr = DARSHAN_MAGIC_NR;
Shane Snyder's avatar
Shane Snyder committed
216
    header.comp_type = fd->comp_type;
217
218
219
220
221

    /* copy the mapping information to the header */
    memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
    memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));

222
    /* write header to file */
223
224
    ret = darshan_log_write(fd, &header, sizeof(header));
    if(ret != sizeof(header))
225
226
227
228
229
230
231
232
    {
        fprintf(stderr, "Error: failed to write Darshan log file header.\n");
        return(-1);
    }

    return(0);
}

233
/* darshan_log_getjob()
234
235
 *
 * read job level metadata from the darshan log file
236
 *
237
 * returns 0 on success, -1 on failure
238
 */
239
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
240
{
241
242
243
    char *comp_buf;
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
    int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
244
    int ret;
245

246
247
    assert(fd->job_map.len > 0 && fd->job_map.off > 0);

248
249
250
251
252
253
    /* allocate buffer to store compressed job info */
    comp_buf = malloc(fd->job_map.len);
    if(!comp_buf)
        return(-1);

    ret = darshan_log_seek(fd, fd->job_map.off);
254
    if(ret < 0)
255
    {
256
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
257
        free(comp_buf);
258
        return(-1);
259
260
    }

261
262
263
    /* read the compressed job data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
    if(ret != fd->job_map.len)
264
    {
265
        fprintf(stderr, "Error: failed to read darshan log file job data.\n");
266
267
268
269
270
        free(comp_buf);
        return(-1);
    }

    /* decompress the job data */
271
272
    ret = darshan_decompress_buf(comp_buf, fd->job_map.len,
        job_buf, &job_buf_sz, fd->comp_type);
273
274
275
276
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
277
278
        return(-1);
    }
279
280
281
    free(comp_buf);

    memcpy(job, job_buf, sizeof(*job));
282

283
    if(fd->swap_flag)
284
    {
285
286
287
288
289
290
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
291
292
    }

293
294
295
296
297
298
299
    /* save trailing exe & mount information, so it can be retrieved later */
    if(!fd->exe_mnt_data)
        fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);

300
301
302
    return(0);
}

303
304
305
306
307
308
309
310
311
/* darshan_log_putjob()
 *
 * write job level metadat to darshan log file
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
{
    struct darshan_job job_copy;
312
313
    char *comp_buf;
    int comp_buf_sz;
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    int len;
    int ret;

    ret = darshan_log_seek(fd, sizeof(struct darshan_header));
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(-1);
    }

    memset(&job_copy, 0, sizeof(*job));
    memcpy(&job_copy, job, sizeof(*job));

    /* check for newline in existing metadata, add if needed */
    len = strlen(job_copy.metadata);
    if(len > 0 && len < DARSHAN_JOB_METADATA_LEN)
    {
        if(job_copy.metadata[len-1] != '\n')
        {
            job_copy.metadata[len] = '\n';
            job_copy.metadata[len+1] = '\0';
        }
    }

338
339
340
341
342
343
    comp_buf = malloc(sizeof(*job));
    if(!comp_buf)
        return(-1);
    comp_buf_sz = sizeof(*job);

    /* compress the job data */
344
345
    ret = darshan_compress_buf((char*)&job_copy, sizeof(*job),
        comp_buf, &comp_buf_sz, fd->comp_type);
346
347
348
349
350
351
352
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to decompress darshan job data.\n");
        free(comp_buf);
        return(-1);
    }    

353
    /* write job data to file */
354
355
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
356
357
    {
        fprintf(stderr, "Error: failed to write darshan log file job data.\n");
Shane Snyder's avatar
Shane Snyder committed
358
        free(comp_buf);
359
360
361
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
362
    free(comp_buf);
363
364
365
366
367
368
369
370
371
    return(0);
}

/* darshan_log_getexe()
 *
 * reads the application exe name from darshan log file
 * 
 * returns 0 on success, -1 on failure 
 */
372
373
374
int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;
375
    int ret;
376

377
378
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
379
    {
380
381
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
382

383
384
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
385
    }
386

387
388
389
390
    /* exe string is located before the first line break */
    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
391
    if(newline)
392
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
393
394
395
396

    return (0);
}

397
398
399
/* darshan_log_putexe()
 *
 * wrties the application exe name to darshan log file
400
401
402
 * NOTE: this needs to be called immediately following put_job as it
 * expects the final pointer to be positioned immediately following
 * the darshan job information
403
404
405
406
407
408
409
 *
 * returns 0 on success, -1 on failure 
 */
int darshan_log_putexe(darshan_fd fd, char *buf)
{
    int len;
    int ret;
410
411
412
413
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;

    len = strlen(buf);
414

415
    /* compress the input exe string */
416
    ret = darshan_compress_buf(buf, len, comp_buf, &comp_buf_sz, fd->comp_type);
417
418
    if(ret < 0)
    {
419
        fprintf(stderr, "Error: unable to compress exe string.\n");
420
421
422
        return(-1);
    }

423
424
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
425
426
427
428
429
430
431
432
    {
        fprintf(stderr, "Error: failed to write exe string to darshan log file.\n");
        return(-1);
    }

    return(0);
}

433
434
/* darshan_log_getmounts()
 * 
435
 * retrieves mount table information from the log. Note that mnt_pts and
436
 * fs_types are arrays that will be allocated by the function and must be
437
438
439
 * freed by the caller. count will indicate the size of the arrays
 *
 * returns 0 on success, -1 on failure
440
 */
441
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
442
443
444
445
    char*** fs_types, int* count)
{
    char *pos;
    int array_index = 0;
446
    int ret;
447

448
449
    /* if the exe/mount data has not been saved yet, read in the job info */
    if(!fd->exe_mnt_data)
450
    {
451
452
        struct darshan_job job;
        ret = darshan_log_getjob(fd, &job);
453

454
455
        if(ret < 0 || !fd->exe_mnt_data)
            return(-1);
456
    }
457

458
    /* count entries */
459
    *count = 0;
460
    pos = fd->exe_mnt_data;
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
482
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
483
484
    {
        /* overestimate string lengths */
485
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
486
        assert((*mnt_pts)[array_index]);
487
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
488
489
        assert((*fs_types)[array_index]);

490
491
492
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
493
        {
494
            fprintf(stderr, "Error: poorly formatted mount table in darshan log file.\n");
495
496
497
498
499
500
501
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

502
503
504
    return(0);
}

505
506
507
508
509
510
511
512
513
514
515
516
517
/* darshan_log_putmounts()
 *
 * writes mount information to the darshan log file
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putexe(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the exe string
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int count)
{
    int i;
    char line[1024];
518
519
520
521
522
    char mnt_dat[DARSHAN_EXE_LEN] = {0};
    int mnt_dat_sz = 0;
    char comp_buf[DARSHAN_EXE_LEN] = {0};
    int comp_buf_sz = DARSHAN_EXE_LEN;
    char *tmp;
523
524
525
    int ret;

    /* write each mount entry to file */
526
    tmp = mnt_dat;
527
528
529
    for(i=count-1; i>=0; i--)
    {
        sprintf(line, "\n%s\t%s", fs_types[i], mnt_pts[i]);
530
531
532
533
534
535

        memcpy(tmp, line, strlen(line));
        tmp += strlen(line);
        mnt_dat_sz += strlen(line);
    }

536
    ret = darshan_compress_buf(mnt_dat, mnt_dat_sz, comp_buf, &comp_buf_sz, fd->comp_type);
537
538
539
540
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress mount data.\n");
        return(-1);
541
542
    }

543
544
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if (ret != comp_buf_sz)
545
    {
546
        fprintf(stderr, "Error: failed to write darshan log mount data.\n");
547
548
549
550
551
552
        return(-1);
    }

    return(0);
}

553
/* darshan_log_gethash()
554
 *
555
 * read the hash of records from the darshan log file
556
557
558
 *
 * returns 0 on success, -1 on failure
 */
559
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
560
{
561
    char *comp_buf;
562
    char *hash_buf;
563
    int hash_buf_sz;
564
    char *buf_ptr;
565
566
567
568
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
569
570
    int ret;

571
572
573
574
575
576
577
578
579
580
    /* allocate buffers to store compressed and decompressed record
     * hash data
     */
    comp_buf = malloc(fd->rec_map.len);
    hash_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
    if(!comp_buf || !hash_buf)
        return(-1);
    hash_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
    memset(hash_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);

581
    ret = darshan_log_seek(fd, fd->rec_map.off);
582
    if(ret < 0)
583
584
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
585
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
586
        free(hash_buf);
587
        return(-1);
588
    }
589

590
    /* read the record hash from the log file */
591
    ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
592
    if(ret != fd->rec_map.len)
593
    {
594
        fprintf(stderr, "Error: failed to read record hash from darshan log file.\n");
595
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
596
        free(hash_buf);
597
598
599
600
601
        return(-1);
    }

    /* decompress the record hash buffer */
    ret = darshan_decompress_buf(comp_buf, fd->rec_map.len,
602
        hash_buf, &hash_buf_sz, fd->comp_type);
603
604
605
606
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress darshan job data.\n");
        free(comp_buf);
Shane Snyder's avatar
Shane Snyder committed
607
        free(hash_buf);
608
609
        return(-1);
    }
610
    free(comp_buf);
611

612
    buf_ptr = hash_buf;
613
    while(buf_ptr < (hash_buf + hash_buf_sz))
614
    {
615
        /* get pointers for each field of this darshan record */
616
        /* NOTE: darshan record hash serialization method: 
617
618
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
619
620
621
622
623
624
625
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;

        if(fd->swap_flag)
626
        {
627
628
629
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
630
631
        }

632
633
634
635
636
637
        HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
        if(!ref)
        {
            ref = malloc(sizeof(*ref));
            if(!ref)
            {
Shane Snyder's avatar
Shane Snyder committed
638
                free(hash_buf);
639
640
641
642
643
644
                return(-1);
            }
            ref->rec.name = malloc(*path_len_ptr + 1);
            if(!ref->rec.name)
            {
                free(ref);
Shane Snyder's avatar
Shane Snyder committed
645
                free(hash_buf);
646
647
648
649
650
651
652
                return(-1);
            }

            /* set the fields for this record */
            ref->rec.id = *rec_id_ptr;
            memcpy(ref->rec.name, path_ptr, *path_len_ptr);
            ref->rec.name[*path_len_ptr] = '\0';
653

654
655
656
            /* add this record to the hash */
            HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
        }
657
658

        buf_ptr += *path_len_ptr;
659
660
    }

661
    free(hash_buf);
662
663
664
    return(0);
}

665
666
667
/* darshan_log_puthash()
 *
 * writes the hash table of records to the darshan log file
668
669
670
 * NOTE: this function call should follow immediately after the call
 * to darshan_log_putmounts(), as it assumes the darshan log file pointer
 * is pointing to the offset immediately following the mount information
671
672
673
674
675
676
677
678
679
680
681
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
{
    size_t hash_buf_sz;
    char *hash_buf;
    char *hash_buf_off;
    struct darshan_record_ref *ref, *tmp;
    uint32_t name_len;
    size_t record_sz;
682
    char *comp_buf;
Shane Snyder's avatar
Shane Snyder committed
683
    int comp_buf_sz;
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
    int ret;

    /* allocate a buffer to store 2 MiB worth of record data */
    /* NOTE: this buffer may be reallocated if estimate is too small */
    hash_buf_sz = 2 * 1024 * 1024;
    hash_buf = malloc(hash_buf_sz);
    if(!hash_buf)
    {
        return(-1);
    }

    /* serialize the record hash into a buffer for writing */
    hash_buf_off = hash_buf;
    HASH_ITER(hlink, hash, ref, tmp)
    {
        name_len = strlen(ref->rec.name);
        record_sz = sizeof(darshan_record_id) + sizeof(uint32_t) + name_len;
        /* make sure there is room in the buffer for this record */
        if((hash_buf_off + record_sz) > (hash_buf + hash_buf_sz))
        {
            char *tmp_buf;
            size_t old_buf_sz;

            /* if no room, reallocate the hash buffer at twice the current size */
            old_buf_sz = hash_buf_off - hash_buf;
            hash_buf_sz *= 2;
            tmp_buf = malloc(hash_buf_sz);
            if(!tmp_buf)
            {
                free(hash_buf);
                return(-1);
            }

            memcpy(tmp_buf, hash_buf, old_buf_sz);
            free(hash_buf);
            hash_buf = tmp_buf;
            hash_buf_off = hash_buf + old_buf_sz;
        }

        /* now serialize the record into the hash buffer.
         * NOTE: darshan record hash serialization method: 
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
        *((darshan_record_id *)hash_buf_off) = ref->rec.id;
        hash_buf_off += sizeof(darshan_record_id);
        *((uint32_t *)hash_buf_off) = name_len;
        hash_buf_off += sizeof(uint32_t);
        memcpy(hash_buf_off, ref->rec.name, name_len);
        hash_buf_off += name_len;
    }
    hash_buf_sz = hash_buf_off - hash_buf;

Shane Snyder's avatar
Shane Snyder committed
736
    comp_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
737
738
    if(!comp_buf)
        return(-1);
Shane Snyder's avatar
Shane Snyder committed
739
    comp_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
740
741

    /* compress the record hash */
742
743
    ret = darshan_compress_buf(hash_buf, hash_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
744
745
746
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to compress darshan record hash.\n");
Shane Snyder's avatar
Shane Snyder committed
747
748
        free(comp_buf);
        free(hash_buf);
749
750
751
752
753
754
755
        return(-1);
    }

    /* set the appropriate mapping info for the record hash in the file descriptor */
    fd->rec_map.off = fd->pos;
    fd->rec_map.len = comp_buf_sz;

756
    /* write the record hash to file */
757
758
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
759
760
    {
        fprintf(stderr, "Error: failed to write record hash to darshan log file.\n");
Shane Snyder's avatar
Shane Snyder committed
761
762
        free(comp_buf);
        free(hash_buf);
763
764
765
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
766
    free(comp_buf);
767
768
769
770
771
772
773
    free(hash_buf);

    return(0);
}

/* darshan_log_getmod()
 *
774
 * returns 1 on successful read of module data, 0 on no data, -1 on failure
775
776
 */
int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
777
    void *mod_buf, int *mod_buf_sz)
778
{
779
    char *comp_buf;
780
    int ret;
781

782
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
783
    {
784
785
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
786
787
    }

788
789
790
791
792
    if(fd->mod_map[mod_id].len == 0)
        return(0); /* no data corresponding to this mod_id */

    comp_buf = malloc(fd->mod_map[mod_id].len);
    if(!comp_buf)
793
        return(-1);
794
795
796
797
798
799
800

    ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        free(comp_buf);
        return(ret);
801
802
    }

803
804
805
    /* read this module's data from the log file */
    ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
    if(ret != fd->mod_map[mod_id].len)
806
    {
807
        fprintf(stderr,
808
            "Error: failed to read module %s data from darshan log file.\n",
809
            darshan_module_names[mod_id]);
810
        free(comp_buf);
811
812
813
        return(-1);
    }

814
815
    /* decompress this module's data */
    ret = darshan_decompress_buf(comp_buf, fd->mod_map[mod_id].len,
816
        mod_buf, mod_buf_sz, fd->comp_type);
817
818
819
820
821
822
823
824
825
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to decompress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
        return(-1);
    }
    free(comp_buf);

826
    return(1);
827
828
}

829
830
831
/* darshan_log_putmod()
 *
 * write a chunk of module data to the darshan log file
Shane Snyder's avatar
Shane Snyder committed
832
833
834
835
836
837
 * NOTE: this function call should be called directly after the
 * put_hash() function, as it expects the file pointer to be
 * positioned directly past the record hash location. Also,
 * for a set of modules with data to write to file, this function
 * should be called in order of increasing module identifiers,
 * as the darshan log file format expects this ordering.
838
839
840
841
842
843
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
    void *mod_buf, int mod_buf_sz)
{
Shane Snyder's avatar
Shane Snyder committed
844
845
    char *comp_buf;
    int comp_buf_sz;
846
847
    int ret;

Shane Snyder's avatar
Shane Snyder committed
848
    if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
849
    {
Shane Snyder's avatar
Shane Snyder committed
850
851
        fprintf(stderr, "Error: invalid Darshan module id.\n");
        return(-1);
852
853
    }

Shane Snyder's avatar
Shane Snyder committed
854
855
856
857
858
859
    comp_buf = malloc(mod_buf_sz);
    if(!comp_buf)
        return(-1);
    comp_buf_sz = mod_buf_sz;

    /* compress the module's data */
860
861
    ret = darshan_compress_buf(mod_buf, mod_buf_sz,
        comp_buf, &comp_buf_sz, fd->comp_type);
Shane Snyder's avatar
Shane Snyder committed
862
    if(ret < 0)
863
    {
Shane Snyder's avatar
Shane Snyder committed
864
865
866
        fprintf(stderr, "Error: unable to compress module %s data.\n",
            darshan_module_names[mod_id]);
        free(comp_buf);
867
868
869
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
870
871
872
873
    /* set the appropriate mapping information for this module */
    fd->mod_map[mod_id].off = fd->pos;
    fd->mod_map[mod_id].len = comp_buf_sz;

874
    /* write the module chunk to the log file */
Shane Snyder's avatar
Shane Snyder committed
875
876
    ret = darshan_log_write(fd, comp_buf, comp_buf_sz);
    if(ret != comp_buf_sz)
877
878
879
880
    {
        fprintf(stderr,
            "Error: failed to write module %s data to darshan log file.\n",
            darshan_module_names[mod_id]);
Shane Snyder's avatar
Shane Snyder committed
881
        free(comp_buf);
882
883
884
        return(-1);
    }

Shane Snyder's avatar
Shane Snyder committed
885
    free(comp_buf);
886
887
888
    return(0);
}

889
890
891
892
893
894
895
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
896
{
897
898
899
900
901
    if(fd->fildes)
        close(fd->fildes);

    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);
902

903
    free(fd);
904
905

    return;
906
907
}

908
/* **************************************************** */
909

910
911
912
913
/* return 0 on successful seek to offset, -1 on failure.
 */
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
914
    off_t ret_off;
915
916
917
918

    if(fd->pos == offset)
        return(0);

919
920
    ret_off = lseek(fd->fildes, offset, SEEK_SET);
    if(ret_off == offset)
921
    {
922
923
        fd->pos = offset;
        return(0);
924
925
926
927
928
    }

    return(-1);
}

929
/* return amount read on success, 0 on EOF, -1 on failure.
930
 */
931
static int darshan_log_read(darshan_fd fd, void* buf, int len)
932
933
934
{
    int ret;

935
936
937
938
    /* read data from the log file using the given map */
    ret = read(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;
939

940
    return(ret);
941
942
}

943
/* return amount written on success, -1 on failure.
944
 */
945
static int darshan_log_write(darshan_fd fd, void* buf, int len)
946
947
948
{
    int ret;

949
950
951
952
953
954
955
956
957
958
    ret = write(fd->fildes, buf, len);
    if(ret > 0)
        fd->pos += ret;

    return(ret);
}

/* return 0 on successful decompression, -1 on failure
 */
static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
    char* decomp_buf, int* inout_decomp_buf_sz,
    enum darshan_comp_type comp_type)
{
    int ret;

    switch(comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_zlib_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_bzip2_decomp(comp_buf, comp_buf_sz,
                decomp_buf, inout_decomp_buf_sz);
            break;
#endif
        default:
            fprintf(stderr, "Error: invalid decompression method.\n");
            return(-1);
    }

    return(ret);
}

Shane Snyder's avatar
Shane Snyder committed
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
static int darshan_compress_buf(char* decomp_buf, int decomp_buf_sz,
    char* comp_buf, int* inout_comp_buf_sz,
    enum darshan_comp_type comp_type)
{
    int ret;

    switch(comp_type)
    {
        case DARSHAN_ZLIB_COMP:
            ret = darshan_zlib_comp(decomp_buf, decomp_buf_sz,
                comp_buf, inout_comp_buf_sz);
            break;
#ifdef HAVE_LIBBZ2
        case DARSHAN_BZIP2_COMP:
            ret = darshan_bzip2_comp(decomp_buf, decomp_buf_sz,
                comp_buf, inout_comp_buf_sz);
            break;
For faster browsing, not all history is shown. View entire blame