darshan-logutils.c 12.8 KB
Newer Older
1 2 3 4 5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7
#include "darshan-util-config.h"
8 9
#include <stdio.h>
#include <string.h>
10
#include <assert.h>
11
#include <stdlib.h>
12
#include <unistd.h>
13
#include <inttypes.h>
14 15 16
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
17
#include <errno.h>
18

19 20 21 22 23
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif

24 25
#include "darshan-logutils.h"

26 27
struct darshan_fd_s
{
28
    int pf;
29
    int64_t pos;
30
    char version[8];
31 32
    int swap_flag;
    char *exe_mnt_data;
33 34 35
    struct darshan_log_map job_map;
    struct darshan_log_map rec_map;
    struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
36 37
};

38 39 40
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
41

42 43 44 45 46 47 48

/* darshan_log_open()
 *
 * open a darshan log file for reading/writing
 *
 * returns 0 on success, -1 on failure
 */
49
darshan_fd darshan_log_open(const char *name, const char *mode)
50
{
51
    int o_flags;
52

53 54 55
    /* we only allows "w" or "r" modes, nothing fancy */
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');
56 57 58 59
    if(mode[0] == 'r')
        o_flags = O_RDONLY;
    else
        o_flags = O_WRONLY;
60

61 62 63 64 65
    darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

66 67
    tmp_fd->pf = open(name, o_flags);
    if(tmp_fd->pf < 0)
68 69
    {
        free(tmp_fd);
70
        return(NULL);
71
    }
72 73

    return(tmp_fd);
74 75
}

76 77 78 79 80 81 82 83
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
84
{
85
    int i;
86
    int ret;
87

88
    ret = darshan_log_seek(fd, 0);
89
    if(ret < 0)
90 91
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
92
        return(ret);
93
    }
94

95
    /* read header from log file */
96
    ret = darshan_log_read(fd, header, sizeof(*header));
97
    if(ret < sizeof(*header))
98
    {
99
        fprintf(stderr, "Error: invalid darshan log file (failed to read header).\n");
100 101 102
        return(-1);
    }

103 104
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
105 106

    if(header->magic_nr == CP_MAGIC_NR)
107
    {
108
        /* no byte swapping needed, this file is in host format already */
109 110 111 112 113 114 115 116 117
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
        if(header->magic_nr == CP_MAGIC_NR)
        {
            fd->swap_flag = 1;
118 119 120 121 122 123 124 125 126

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
            for(i=0;i<DARSHAN_MAX_MODS;i++)
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
127 128 129 130 131 132 133
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
134 135
    }

136 137 138 139 140
    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
141 142

    return(0);
143
}
144

145
/* darshan_log_getjob()
146 147
 *
 * read job level metadata from the darshan log file
148
 *
149
 * returns 0 on success, -1 on failure
150
 */
151
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
152
{
153
    char job_buf[CP_JOB_RECORD_SIZE] = {0};
154
    int ret;
155

156
    ret = darshan_log_seek(fd, fd->job_map.off);
157
    if(ret < 0)
158
    {
159 160
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(ret);
161 162
    }

163
    /* read the job data from the log file */
164
    ret = darshan_log_read(fd, job_buf, fd->job_map.len);
165
    if(ret < fd->job_map.len)
166
    {
167
        fprintf(stderr, "Error: invalid darshan log file (failed to read job data).\n");
168 169 170
        return(-1);
    }

171 172
    memcpy(job, job_buf, sizeof(*job));

173
    if(fd->swap_flag)
174
    {
175 176 177 178 179 180
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
181 182
    }

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
    /* save trailing job data, so exe and mount information can be retrieved later */
    fd->exe_mnt_data = malloc(CP_EXE_LEN+1);
    if(!fd->exe_mnt_data)
        return(-1);
    memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], CP_EXE_LEN+1);

    return(0);
}

#if 0
#ifdef HAVE_STRNDUP
    metadata = strndup(job->metadata, sizeof(job->metadata));
#else
    metadata = strdup(job->metadata);
#endif
    char *kv;
    char *key;
    char *value;
    char *save;

    for(kv=strtok_r(metadata, "\n", &save);
        kv != NULL;
        kv=strtok_r(NULL, "\n", &save))
    {
        /* NOTE: we intentionally only split on the first = character.
         * There may be additional = characters in the value portion
         * (for example, when storing mpi-io hints).
         */
        strcpy(buffer, kv);
        key = buffer;
        value = index(buffer, '=');
        if(!value)
            continue;
        /* convert = to a null terminator to split key and value */
        value[0] = '\0';
        value++;
        if (strcmp(key, "prev_ver") == 0)
        {
            strncpy(job->version_string, value, sizeof(job->version_string));
        }
    }
    free(metadata);
#endif

int darshan_log_getexe(darshan_fd fd, char *buf)
{
    char *newline;

    /* TODO: try reading log job one more time to set this buffer up */
    if(!fd->exe_mnt_data)
        return(-1);

    newline = strchr(fd->exe_mnt_data, '\n');

    /* copy over the exe string */
    if(newline)
        memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));

    return (0);
}

/* darshan_log_getmounts()
 * 
246 247 248
 * retrieves mount table information from the log.  Note that mnt_pts and
 * fs_types are arrays that will be allocated by the function and must be
 * freed by the caller.  count will indicate the size of the arrays
249
 */
250
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
    char*** fs_types, int* count)
{
    int ret;
    char *pos;
    int array_index = 0;

    /* TODO: try reading log job one more time to set this buffer up */
    if(!fd->exe_mnt_data)
        return(-1);

    /* count entries */
    *count = 0;
    pos = fd->exe_mnt_data;
    while((pos = strchr(pos, '\n')) != NULL)
    {
        pos++;
        (*count)++;
    }

    if(*count == 0)
    {
        /* no mount entries present */
        return(0);
    }

    /* allocate output arrays */
    *mnt_pts = malloc((*count)*sizeof(char*));
    assert(*mnt_pts);
    *fs_types = malloc((*count)*sizeof(char*));
    assert(*fs_types);

    /* work backwards through the table and parse each line (except for
     * first, which holds command line information)
     */
    while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
    {
        /* overestimate string lengths */
        (*mnt_pts)[array_index] = malloc(CP_EXE_LEN);
        assert((*mnt_pts)[array_index]);
        (*fs_types)[array_index] = malloc(CP_EXE_LEN);
        assert((*fs_types)[array_index]);

293 294 295
        ret = sscanf(++pos, "%s\t%s", (*fs_types)[array_index],
            (*mnt_pts)[array_index]);
        if(ret != 2)
296 297 298 299 300 301 302 303 304
        {
            fprintf(stderr, "Error: poorly formatted mount table in log file.\n");
            return(-1);
        }
        pos--;
        *pos = '\0';
        array_index++;
    }

305 306 307
    return(0);
}

308
/* darshan_log_gethash()
309
 *
310
 * read the hash of records from the darshan log file
311 312 313
 *
 * returns 0 on success, -1 on failure
 */
314
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
315
{
316
    unsigned char *hash_buf;
317 318 319 320 321
    unsigned char *buf_ptr;
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
322 323
    int ret;

324
    ret = darshan_log_seek(fd, fd->rec_map.off);
325
    if(ret < 0)
326 327
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
328
        return(ret);
329
    }
330

331 332 333
    /* allocate a buffer to store the (serialized) darshan record hash */
    hash_buf = malloc(fd->rec_map.len);
    if(!hash_buf)
334 335
        return(-1);

336
    /* read the record map from the log file */
337 338
    ret = darshan_log_read(fd, hash_buf, fd->rec_map.len);
    if(ret < fd->rec_map.len)
339
    {
340 341
        fprintf(stderr, "Error: invalid darshan log file (failed to read record hash).\n");
        free(hash_buf);
342
        return(-1);
343 344
    }

345 346
    buf_ptr = hash_buf;
    while(buf_ptr < (hash_buf + fd->rec_map.len))
347
    {
348
        /* get pointers for each field of this darshan record */
349
        /* NOTE: darshan record hash serialization method: 
350 351
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
352 353 354 355 356 357 358 359 360
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        ref = malloc(sizeof(*ref));
        if(!ref)
361
        {
362
            free(hash_buf);
363
            return(-1);
364
        }
365 366
        ref->rec.name = malloc(*path_len_ptr + 1);
        if(!ref->rec.name)
367
        {
368
            free(hash_buf);
369 370
            free(ref);
            return(-1);
371 372
        }

373
        if(fd->swap_flag)
374
        {
375 376 377
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
378 379
        }

380 381 382 383
        /* set the fields for this record */
        ref->rec.id = *rec_id_ptr;
        memcpy(ref->rec.name, path_ptr, *path_len_ptr);
        ref->rec.name[*path_len_ptr] = '\0';
384

385
        /* add this record to the hash */
386
        HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
387 388
    }

389
    free(hash_buf);
390

391
    return(0);
392
}
393

394 395
/* TODO: hardcoded for posix -- what can we do generally?
 *       different function for each module and a way to map to this function?
396
 */
397
int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
398 399 400 401
{
    int ret;
    const char* err_string;
    int i;
402

403
    if(fd->pos < fd->mod_map[0].off)
404
    {
405
        ret = darshan_log_seek(fd, fd->mod_map[0].off);
406
        if(ret < 0)
407 408
        {
            fprintf(stderr, "Error: unable to seek in darshan log file.\n");
409
            return(ret);
410
        }
411
    }
412 413 414 415

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
416
    memset(file, 0, sizeof(*file));
417

418
    ret = darshan_log_read(fd, file, sizeof(*file));
419 420 421 422 423 424
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
425
            DARSHAN_BSWAP64(&file->f_id);
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

443
    if(ret == 0)
444 445 446 447 448 449
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
450
    err_string = strerror(errno);
451 452 453 454
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

455 456 457 458 459 460 461
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
462
{
463 464
    if(fd->pf)
        close(fd->pf);
465

466 467 468
    if(fd->exe_mnt_data)
        free(fd->exe_mnt_data);

469
    free(fd);
470 471
}

472
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
473

474 475 476 477 478 479
/* return amount written on success, -1 on failure.
 */
static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
    int ret;

480
    if(fd->pf)
481
    {
482
        ret = write(fd->pf, buf, len);
483 484 485 486 487 488 489 490
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

491 492 493 494 495 496
/* return amount read on success, 0 on EOF, -1 on failure.
 */
static int darshan_log_read(darshan_fd fd, void* buf, int len)
{
    int ret;

497
    if(fd->pf)
498
    {
499
        ret = read(fd->pf, buf, len);
500 501 502 503 504 505 506 507 508 509
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

/* return 0 on successful seek to offset, -1 on failure.
 */
510
static int darshan_log_seek(darshan_fd fd, off_t offset)
511
{
512 513
    off_t ret_off;

514 515 516
    if(fd->pos == offset)
        return(0);

517 518
    ret_off = lseek(fd->pf, offset, SEEK_SET);
    if(ret_off == offset)
519
    {
520
        fd->pos = offset;
521 522 523 524 525
        return(0);
    }

    return(-1);
}