darshan-logutils.c 11.9 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7
#include "darshan-util-config.h"
8
9
#include <stdio.h>
#include <string.h>
10
#include <assert.h>
11
#include <stdlib.h>
12
#include <unistd.h>
13
#include <inttypes.h>
14
15
16
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
17
#include <errno.h>
18

19
20
21
22
23
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif

24
25
#include "darshan-logutils.h"

26
27
struct darshan_fd_s
{
28
    int pf;
29
    int64_t pos;
30
    char version[8];
31
32
    int swap_flag;
    char *exe_mnt_data;
33
34
35
    struct darshan_log_map job_map;
    struct darshan_log_map rec_map;
    struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
36
37
};

38
39
40
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
41

42
43
44
45
46
47
48

/* darshan_log_open()
 *
 * open a darshan log file for reading/writing
 *
 * returns 0 on success, -1 on failure
 */
49
darshan_fd darshan_log_open(const char *name, const char *mode)
50
{
51
    int o_flags;
52

53
54
55
    /* we only allows "w" or "r" modes, nothing fancy */
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');
56
57
58
59
    if(mode[0] == 'r')
        o_flags = O_RDONLY;
    else
        o_flags = O_WRONLY;
60

61
62
63
64
65
    darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

66
67
    tmp_fd->pf = open(name, o_flags);
    if(tmp_fd->pf < 0)
68
69
    {
        free(tmp_fd);
70
        return(NULL);
71
    }
72
73

    return(tmp_fd);
74
75
}

76
77
78
79
80
81
82
83
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
84
{
85
    int i;
86
    int ret;
87

88
    ret = darshan_log_seek(fd, 0);
89
    if(ret < 0)
90
91
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
92
        return(ret);
93
    }
94

95
    /* read header from log file */
96
    ret = darshan_log_read(fd, header, sizeof(*header));
97
    if(ret < sizeof(*header))
98
    {
99
        fprintf(stderr, "Error: invalid darshan log file (failed to read header).\n");
100
101
102
        return(-1);
    }

103
104
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
105

106
    if(header->magic_nr == DARSHAN_MAGIC_NR)
107
    {
108
        /* no byte swapping needed, this file is in host format already */
109
110
111
112
113
114
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
115
        if(header->magic_nr == DARSHAN_MAGIC_NR)
116
117
        {
            fd->swap_flag = 1;
118
119
120
121
122
123
124
125
126

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
            for(i=0;i<DARSHAN_MAX_MODS;i++)
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
127
128
129
130
131
132
133
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
134
135
    }

136
137
138
139
140
    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
141
142

    return(0);
143
}
144

145
/* darshan_log_getjob()
146
147
 *
 * read job level metadata from the darshan log file
148
 *
149
 * returns 0 on success, -1 on failure
150
 */
151
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
152
{
153
    char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
154
    int ret;
155

156
    ret = darshan_log_seek(fd, fd->job_map.off);
157
    if(ret < 0)
158
    {
159
160
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(ret);
161
162
    }

163
    /* read the job data from the log file */
164
    ret = darshan_log_read(fd, job_buf, fd->job_map.len);
165
    if(ret < fd->job_map.len)
166
    {
167
        fprintf(stderr, "Error: invalid darshan log file (failed to read job data).\n");
168
169
170
        return(-1);
    }

171
172
    memcpy(job, job_buf, sizeof(*job));

173
    if(fd->swap_flag)
174
    {
175
176
177
178
179
180
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
181
182
    }

183
    /* save trailing job data, so exe and mount information can be retrieved later */
184
    fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
185
186
    if(!fd->exe_mnt_data)
        return(-1);
187
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210

    return(0);
}

int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;

    /* TODO: try reading log job one more time to set this buffer up */
    if(!fd->exe_mnt_data)
        return(-1);

    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
    if(newline)
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));

    return (0);
}

/* darshan_log_getmounts()
 * 
211
212
213
 * retrieves mount table information from the log.  Note that mnt_pts and
 * fs_types are arrays that will be allocated by the function and must be
 * freed by the caller.  count will indicate the size of the arrays
214
 */
215
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
    char*** fs_types, int* count)
{
    int ret;
    char *pos;
    int array_index = 0;

    /* TODO: try reading log job one more time to set this buffer up */
    if(!fd->exe_mnt_data)
        return(-1);

    /* count entries */
    *count = 0;
    pos = fd->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
    {
        /* overestimate string lengths */
253
        (*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
254
        assert((*mnt_pts)[array_index]);
255
        (*fs_types)[array_index] = malloc(DARSHAN_EXE_LEN);
256
257
        assert((*fs_types)[array_index]);

258
259
260
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
261
262
263
264
265
266
267
268
269
        {
            fprintf(stderr, "Error: poorly formatted mount table in log file.\n");
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

270
271
272
    return(0);
}

273
/* darshan_log_gethash()
274
 *
275
 * read the hash of records from the darshan log file
276
277
278
 *
 * returns 0 on success, -1 on failure
 */
279
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
280
{
281
    unsigned char *hash_buf;
282
283
284
285
286
    unsigned char *buf_ptr;
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
287
288
    int ret;

289
    ret = darshan_log_seek(fd, fd->rec_map.off);
290
    if(ret < 0)
291
292
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
293
        return(ret);
294
    }
295

296
297
298
    /* allocate a buffer to store the (serialized) darshan record hash */
    hash_buf = malloc(fd->rec_map.len);
    if(!hash_buf)
299
300
        return(-1);

301
    /* read the record map from the log file */
302
303
    ret = darshan_log_read(fd, hash_buf, fd->rec_map.len);
    if(ret < fd->rec_map.len)
304
    {
305
306
        fprintf(stderr, "Error: invalid darshan log file (failed to read record hash).\n");
        free(hash_buf);
307
        return(-1);
308
309
    }

310
311
    buf_ptr = hash_buf;
    while(buf_ptr < (hash_buf + fd->rec_map.len))
312
    {
313
        /* get pointers for each field of this darshan record */
314
        /* NOTE: darshan record hash serialization method: 
315
316
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
317
318
319
320
321
322
323
324
325
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        ref = malloc(sizeof(*ref));
        if(!ref)
326
        {
327
            free(hash_buf);
328
            return(-1);
329
        }
330
331
        ref->rec.name = malloc(*path_len_ptr + 1);
        if(!ref->rec.name)
332
        {
333
            free(hash_buf);
334
335
            free(ref);
            return(-1);
336
337
        }

338
        if(fd->swap_flag)
339
        {
340
341
342
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
343
344
        }

345
346
347
348
        /* set the fields for this record */
        ref->rec.id = *rec_id_ptr;
        memcpy(ref->rec.name, path_ptr, *path_len_ptr);
        ref->rec.name[*path_len_ptr] = '\0';
349

350
        /* add this record to the hash */
351
        HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
352
353
    }

354
    free(hash_buf);
355

356
    return(0);
357
}
358

359
360
/* TODO: hardcoded for posix -- what can we do generally?
 *       different function for each module and a way to map to this function?
361
 */
362
int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
363
364
365
366
{
    int ret;
    const char* err_string;
    int i;
367

368
    if(fd->pos < fd->mod_map[0].off)
369
    {
370
        ret = darshan_log_seek(fd, fd->mod_map[0].off);
371
        if(ret < 0)
372
373
        {
            fprintf(stderr, "Error: unable to seek in darshan log file.\n");
374
            return(ret);
375
        }
376
    }
377
378
379
380

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
381
    memset(file, 0, sizeof(*file));
382

383
    ret = darshan_log_read(fd, file, sizeof(*file));
384
385
386
387
388
389
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
390
            DARSHAN_BSWAP64(&file->f_id);
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

408
    if(ret == 0)
409
410
411
412
413
414
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
415
    err_string = strerror(errno);
416
417
418
419
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

420
421
422
423
424
425
426
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
427
{
428
429
    if(fd->pf)
        close(fd->pf);
430

431
432
433
    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);

434
    free(fd);
435
436
}

437
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
438

439
440
441
442
443
444
/* return amount written on success, -1 on failure.
 */
static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
    int ret;

445
    if(fd->pf)
446
    {
447
        ret = write(fd->pf, buf, len);
448
449
450
451
452
453
454
455
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

456
457
458
459
460
461
/* return amount read on success, 0 on EOF, -1 on failure.
 */
static int darshan_log_read(darshan_fd fd, void* buf, int len)
{
    int ret;

462
    if(fd->pf)
463
    {
464
        ret = read(fd->pf, buf, len);
465
466
467
468
469
470
471
472
473
474
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

/* return 0 on successful seek to offset, -1 on failure.
 */
475
static int darshan_log_seek(darshan_fd fd, off_t offset)
476
{
477
478
    off_t ret_off;

479
480
481
    if(fd->pos == offset)
        return(0);

482
483
    ret_off = lseek(fd->pf, offset, SEEK_SET);
    if(ret_off == offset)
484
    {
485
        fd->pos = offset;
486
487
488
489
490
        return(0);
    }

    return(-1);
}