darshan-logutils.c 10.5 KB
Newer Older
1 2 3 4 5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7
#include "darshan-util-config.h"
8 9
#include <stdio.h>
#include <string.h>
10
#include <assert.h>
11
#include <stdlib.h>
12
#include <unistd.h>
13
#include <inttypes.h>
14 15 16
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
17
#include <errno.h>
18

19 20 21 22 23
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif

24 25
#include "darshan-logutils.h"

26 27
struct darshan_fd_s
{
28
    int pf;
29 30 31
    int64_t pos;
    char mode[2];
    int swap_flag;
32
    char version[8];
33
    char* name;
34 35 36
    struct darshan_log_map job_map;
    struct darshan_log_map rec_map;
    struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
37 38
};

39 40 41
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
42

43 44 45 46 47 48 49

/* darshan_log_open()
 *
 * open a darshan log file for reading/writing
 *
 * returns 0 on success, -1 on failure
 */
50
darshan_fd darshan_log_open(const char *name, const char *mode)
51
{
52
    int o_flags;
53

54 55 56
    /* we only allows "w" or "r" modes, nothing fancy */
    assert(strlen(mode) == 1);
    assert(mode[0] == 'r' || mode[0] == 'w');
57 58 59 60
    if(mode[0] == 'r')
        o_flags = O_RDONLY;
    else
        o_flags = O_WRONLY;
61

62 63 64 65 66
    darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
    if(!tmp_fd)
        return(NULL);
    memset(tmp_fd, 0, sizeof(*tmp_fd));

67 68
    /* TODO: why is mode needed??? */
    /* TODO: why is name needed??? */
69 70 71 72 73 74 75 76 77
    tmp_fd->mode[0] = mode[0];
    tmp_fd->mode[1] = mode[1];
    tmp_fd->name  = strdup(name);
    if(!tmp_fd->name)
    {
        free(tmp_fd);
        return(NULL);
    }

78 79
    tmp_fd->pf = open(name, o_flags);
    if(tmp_fd->pf < 0)
80
    {
81
        free(tmp_fd->name);
82
        free(tmp_fd);
83
        return(NULL);
84
    }
85 86

    return(tmp_fd);
87 88
}

89 90 91 92 93 94 95 96
/* darshan_log_getheader()
 *
 * read the header of the darshan log and set internal data structures
 * NOTE: this function must be called before reading other portions of the log
 *
 * returns 0 on success, -1 on failure
 */
int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
97
{
98
    int i;
99
    int ret;
100

101
    ret = darshan_log_seek(fd, 0);
102
    if(ret < 0)
103 104
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
105
        return(ret);
106
    }
107

108
    /* read header from log file */
109
    ret = darshan_log_read(fd, header, sizeof(*header));
110
    if(ret < sizeof(*header))
111
    {
112
        fprintf(stderr, "Error: invalid darshan log file (failed to read header).\n");
113 114 115
        return(-1);
    }

116 117
    /* save the version string */
    strncpy(fd->version, header->version_string, 8);
118 119

    if(header->magic_nr == CP_MAGIC_NR)
120
    {
121
        /* no byte swapping needed, this file is in host format already */
122 123 124 125 126 127 128 129 130
        fd->swap_flag = 0;
    }
    else
    {
        /* try byte swapping */
        DARSHAN_BSWAP64(&header->magic_nr);
        if(header->magic_nr == CP_MAGIC_NR)
        {
            fd->swap_flag = 1;
131 132 133 134 135 136 137 138 139

            /* swap the log map variables in the header */
            DARSHAN_BSWAP64(&header->rec_map.off);
            DARSHAN_BSWAP64(&header->rec_map.len);
            for(i=0;i<DARSHAN_MAX_MODS;i++)
            {
                DARSHAN_BSWAP64(&header->mod_map[i].off);
                DARSHAN_BSWAP64(&header->mod_map[i].len);
            }
140 141 142 143 144 145 146
        }
        else
        {
            /* otherwise this file is just broken */
            fprintf(stderr, "Error: bad magic number in darshan log file.\n");
            return(-1);
        }
147 148
    }

149 150 151 152 153
    /* save the mapping of data within log file to this file descriptor */
    fd->job_map.off = sizeof(struct darshan_header);
    fd->job_map.len = header->rec_map.off - fd->job_map.off;
    memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
    memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
154 155

    return(0);
156
}
157

158
/* darshan_log_getjob()
159 160
 *
 * read job level metadata from the darshan log file
161
 *
162
 * returns 0 on success, -1 on failure
163
 */
164
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
165
{
166
    int ret;
167

168
    ret = darshan_log_seek(fd, fd->job_map.off);
169
    if(ret < 0)
170
    {
171 172
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
        return(ret);
173 174
    }

175
    /* read the job data from the log file */
176 177
    ret = darshan_log_read(fd, job, fd->job_map.len);
    if(ret < fd->job_map.len)
178
    {
179
        fprintf(stderr, "Error: invalid darshan log file (failed to read job data).\n");
180 181 182
        return(-1);
    }

183
    if(fd->swap_flag)
184
    {
185 186 187 188 189 190
        /* swap bytes if necessary */
        DARSHAN_BSWAP64(&job->uid);
        DARSHAN_BSWAP64(&job->start_time);
        DARSHAN_BSWAP64(&job->end_time);
        DARSHAN_BSWAP64(&job->nprocs);
        DARSHAN_BSWAP64(&job->jobid);
191 192 193 194 195
    }

    return(0);
}

196
/* darshan_log_gethash()
197
 *
198
 * read the hash of records from the darshan log file
199 200 201
 *
 * returns 0 on success, -1 on failure
 */
202
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
203
{
204
    unsigned char *hash_buf;
205 206 207 208 209
    unsigned char *buf_ptr;
    darshan_record_id *rec_id_ptr;
    uint32_t *path_len_ptr;
    char *path_ptr;
    struct darshan_record_ref *ref;
210 211
    int ret;

212
    ret = darshan_log_seek(fd, fd->rec_map.off);
213
    if(ret < 0)
214 215
    {
        fprintf(stderr, "Error: unable to seek in darshan log file.\n");
216
        return(ret);
217
    }
218

219 220 221
    /* allocate a buffer to store the (serialized) darshan record hash */
    hash_buf = malloc(fd->rec_map.len);
    if(!hash_buf)
222 223
        return(-1);

224
    /* read the record map from the log file */
225 226
    ret = darshan_log_read(fd, hash_buf, fd->rec_map.len);
    if(ret < fd->rec_map.len)
227
    {
228 229
        fprintf(stderr, "Error: invalid darshan log file (failed to read record hash).\n");
        free(hash_buf);
230
        return(-1);
231 232
    }

233 234
    buf_ptr = hash_buf;
    while(buf_ptr < (hash_buf + fd->rec_map.len))
235
    {
236
        /* get pointers for each field of this darshan record */
237
        /* NOTE: darshan record hash serialization method: 
238 239
         *          ... darshan_record_id | (uint32_t) path_len | path ...
         */
240 241 242 243 244 245 246 247 248
        rec_id_ptr = (darshan_record_id *)buf_ptr;
        buf_ptr += sizeof(darshan_record_id);
        path_len_ptr = (uint32_t *)buf_ptr;
        buf_ptr += sizeof(uint32_t);
        path_ptr = (char *)buf_ptr;
        buf_ptr += *path_len_ptr;

        ref = malloc(sizeof(*ref));
        if(!ref)
249
        {
250
            free(hash_buf);
251
            return(-1);
252
        }
253 254
        ref->rec.name = malloc(*path_len_ptr + 1);
        if(!ref->rec.name)
255
        {
256
            free(hash_buf);
257 258
            free(ref);
            return(-1);
259 260
        }

261
        if(fd->swap_flag)
262
        {
263 264 265
            /* we need to sort out endianness issues before deserializing */
            DARSHAN_BSWAP64(rec_id_ptr);
            DARSHAN_BSWAP32(path_len_ptr);
266 267
        }

268 269 270 271
        /* set the fields for this record */
        ref->rec.id = *rec_id_ptr;
        memcpy(ref->rec.name, path_ptr, *path_len_ptr);
        ref->rec.name[*path_len_ptr] = '\0';
272

273
        /* add this record to the hash */
274
        HASH_ADD(hlink, *hash, rec.id, sizeof(darshan_record_id), ref);
275 276
    }

277
    free(hash_buf);
278

279
    return(0);
280
}
281

282 283
/* TODO: hardcoded for posix -- what can we do generally?
 *       different function for each module and a way to map to this function?
284
 */
285
int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
286 287 288 289
{
    int ret;
    const char* err_string;
    int i;
290

291
    if(fd->pos < fd->mod_map[0].off)
292
    {
293
        ret = darshan_log_seek(fd, fd->mod_map[0].off);
294
        if(ret < 0)
295 296
        {
            fprintf(stderr, "Error: unable to seek in darshan log file.\n");
297
            return(ret);
298
        }
299
    }
300 301 302 303

    /* reset file record, so that diff compares against a zero'd out record
     * if file is missing
     */
304
    memset(file, 0, sizeof(*file));
305

306
    ret = darshan_log_read(fd, file, sizeof(*file));
307 308 309 310 311 312
    if(ret == sizeof(*file))
    {
        /* got exactly one, correct size record */
        if(fd->swap_flag)
        {
            /* swap bytes if necessary */
313
            DARSHAN_BSWAP64(&file->f_id);
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
            DARSHAN_BSWAP64(&file->rank);
            for(i=0; i<CP_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->counters[i]);
            for(i=0; i<CP_F_NUM_INDICES; i++)
                DARSHAN_BSWAP64(&file->fcounters[i]);
        }
        return(1);
    }


    if(ret > 0)
    {
        /* got a short read */
        fprintf(stderr, "Error: invalid file record (too small)\n");
        return(-1);
    }

331
    if(ret == 0)
332 333 334 335 336 337
    {
        /* hit end of file */
        return(0);
    }

    /* all other errors */
338
    err_string = strerror(errno);
339 340 341 342
    fprintf(stderr, "Error: %s\n", err_string);
    return(-1);
}

343 344
#if 0
int darshan_log_getexe(darshan_fd fd, char *buf)
Philip Carns's avatar
Philip Carns committed
345 346
{
    int ret;
347
    char* newline;
Philip Carns's avatar
Philip Carns committed
348

349 350 351
    ret = darshan_log_seek(fd, fd->job_struct_size);
    if(ret < 0)
        return(ret);
Philip Carns's avatar
Philip Carns committed
352

353 354
    ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
    if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
Philip Carns's avatar
Philip Carns committed
355
    {
356
        perror("darshan_log_read");
Philip Carns's avatar
Philip Carns committed
357 358 359
        return(-1);
    }

360 361 362 363 364 365 366
    /* this call is only supposed to return the exe string, but starting in
     * log format 1.23 there could be a table of mount entry information
     * after the exe.  Look for newline character and truncate there.
     */
    newline = strchr(buf, '\n');
    if(newline)
        *newline = '\0';
367

368
    return (0);
369
}
370
#endif
371

372 373 374 375 376 377 378
/* darshan_log_close()
 *
 * close an open darshan file descriptor
 *
 * returns 0 on success, -1 on failure
 */
void darshan_log_close(darshan_fd fd)
379
{
380 381
    if(fd->pf)
        close(fd->pf);
382

383 384
    free(fd->name);
    free(fd);
385 386
}

387
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
388

389 390 391 392 393 394
/* return amount written on success, -1 on failure.
 */
static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
    int ret;

395
    if(fd->pf)
396
    {
397
        ret = write(fd->pf, buf, len);
398 399 400 401 402 403 404 405
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

406 407 408 409 410 411
/* return amount read on success, 0 on EOF, -1 on failure.
 */
static int darshan_log_read(darshan_fd fd, void* buf, int len)
{
    int ret;

412
    if(fd->pf)
413
    {
414
        ret = read(fd->pf, buf, len);
415 416 417 418 419 420 421 422 423 424
        if(ret > 0)
            fd->pos += ret;
        return(ret);
    }

    return(-1);
}

/* return 0 on successful seek to offset, -1 on failure.
 */
425
static int darshan_log_seek(darshan_fd fd, off_t offset)
426
{
427 428
    off_t ret_off;

429 430 431
    if(fd->pos == offset)
        return(0);

432 433
    ret_off = lseek(fd->pf, offset, SEEK_SET);
    if(ret_off == offset)
434
    {
435
        fd->pos = offset;
436 437 438 439 440
        return(0);
    }

    return(-1);
}