darshan-posix.c 11.6 KB
Newer Older
1 2 3 4 5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6 7
#define _GNU_SOURCE

8
#include "darshan-runtime-config.h"
9

10 11 12 13 14 15 16 17 18 19 20 21 22 23
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
24
#include <libgen.h>
25
#include <aio.h>
26
#include <pthread.h>
27

28
#include "uthash.h"
29
#include "darshan.h"
30

31
#ifndef HAVE_OFF64_T
32 33
typedef int64_t off64_t;
#endif
34 35 36
#ifndef HAVE_AIOCB64
#define aiocb64 aiocb
#endif
37

38 39
/* TODO these go where ? */

40 41 42 43 44
#define DARSHAN_FORWARD_DECL(name,ret,args) \
  extern ret __real_ ## name args;

#define DARSHAN_DECL(__name) __wrap_ ## __name

45 46
#define MAP_OR_FAIL(func)

47 48
#define POSIX_MOD_NAME "POSIX"

49
struct posix_runtime_file
50
{
51
    struct darshan_posix_file* file_record;
52
    UT_hash_handle hlink;
53
};
54

55
struct posix_runtime_file_ref
56
{
57 58 59 60 61 62 63
    struct posix_runtime_file* file;
    int fd;
    UT_hash_handle hlink;
};

struct posix_runtime
{
64 65
    struct posix_runtime_file* file_runtime_array;
    struct darshan_posix_file* file_record_array;
66
    int file_array_size;
67
    int file_array_ndx;
68 69
    struct posix_runtime_file* file_hash;
    struct posix_runtime_file_ref* fd_hash;
70 71
};

72
static struct posix_runtime *posix_runtime = NULL;
73
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
74
static int my_rank = -1;
75

76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
/* these are paths that we will not trace */
static char* exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

91
DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
92
DARSHAN_FORWARD_DECL(close, int, (int fd));
93

94 95
static void posix_runtime_initialize(void);

96 97 98 99
static struct posix_runtime_file* posix_file_by_name(const char *name);
static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
static void posix_file_close_fd(int fd);

100 101
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size);
static void posix_shutdown(void);
102

103 104
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
105

106
#define POSIX_SET(__file, __counter, __value) do {\
107
    (__file)->file_record->counters[__counter] = __value; \
108 109 110
} while(0)

#define POSIX_F_SET(__file, __counter, __value) do {\
111
    (__file)->file_record->fcounters[__counter] = __value; \
112 113 114
} while(0)

#define POSIX_INC(__file, __counter, __value) do {\
115
    (__file)->file_record->counters[__counter] += __value; \
116 117 118
} while(0)

#define POSIX_F_INC(__file, __counter, __value) do {\
119
    (__file)->file_record->fcounters[__counter] += __value; \
120 121 122 123 124 125 126 127 128 129 130 131
} while(0)

#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
    if(__tm1 > __last) \
        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
    else \
        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
    if(__tm2 > __last) \
        __last = __tm2; \
} while(0)

#define POSIX_VALUE(__file, __counter) \
132
    ((__file)->file_record->counters[__counter])
133 134

#define POSIX_F_VALUE(__file, __counter) \
135
    ((__file)->file_record->fcounters[__counter])
136 137

#define POSIX_MAX(__file, __counter, __value) do {\
138
    if((__file)->file_record->counters[__counter] < __value) \
139
    { \
140
        (__file)->file_record->counters[__counter] = __value; \
141 142 143
    } \
} while(0)

144
#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
145
    struct posix_runtime_file* file; \
146 147 148 149 150 151 152 153 154
    char* exclude; \
    int tmp_index = 0; \
    if(__ret < 0) break; \
    while((exclude = exclusions[tmp_index])) { \
        if(!(strncmp(exclude, __path, strlen(exclude)))) \
            break; \
        tmp_index++; \
    } \
    if(exclude) break; \
155
    file = posix_file_by_name_setfd(__path, __ret); \
156
    if(!file) break; \
157
    file->file_record->rank = my_rank; \
158
    if(__mode) \
159
        POSIX_SET(file, CP_MODE, __mode); \
160
    if(__stream_flag)\
161
        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
162
    else \
163 164 165 166
        POSIX_INC(file, CP_POSIX_OPENS, 1); \
    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
} while(0)
167

168
int DARSHAN_DECL(open)(const char *path, int flags, ...)
169 170 171 172 173
{
    int mode = 0;
    int ret;
    double tm1, tm2;

174 175
    MAP_OR_FAIL(open);

176
    if(flags & O_CREAT) 
177 178 179 180 181 182
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

183
        tm1 = darshan_core_wtime();
184
        ret = __real_open(path, flags, mode);
185
        tm2 = darshan_core_wtime();
186 187 188
    }
    else
    {
189
        tm1 = darshan_core_wtime();
190
        ret = __real_open(path, flags);
191
        tm2 = darshan_core_wtime();
192 193
    }

194 195 196
    POSIX_LOCK();
    posix_runtime_initialize();

197
    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
198

199
    POSIX_UNLOCK();
200 201 202 203

    return(ret);
}

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
int DARSHAN_DECL(close)(int fd)
{
    struct darshan_file_runtime* file;
    int tmp_fd = fd;
    double tm1, tm2;
    int ret;

    MAP_OR_FAIL(close);

    tm1 = darshan_core_wtime();
    ret = __real_close(fd);
    tm2 = darshan_core_wtime();

    POSIX_LOCK();
    posix_runtime_initialize();
219

220
    posix_file_close_fd(tmp_fd);
221 222 223 224 225 226 227

    POSIX_UNLOCK();    

    return(ret);
}

/* ***************************************************** */
228

229
static void posix_runtime_initialize()
230
{
231 232 233 234 235 236 237
    char *alignstr;
    int tmpval;
    int ret;
    int mem_limit;
    struct darshan_module_funcs posix_mod_fns =
    {
        .get_output_data = &posix_get_output_data,
238
        .shutdown = &posix_shutdown
239
    };
240

241
    if(posix_runtime)
242
        return;
243

244 245
    /* register the posix module with darshan core */
    darshan_core_register_module(
246
        DARSHAN_POSIX_MOD,
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
        POSIX_MOD_NAME,
        &posix_mod_fns,
        &mem_limit);

    /* return if no memory assigned by darshan core */
    if(mem_limit == 0)
        return;

    posix_runtime = malloc(sizeof(*posix_runtime));
    if(!posix_runtime)
        return;
    memset(posix_runtime, 0, sizeof(*posix_runtime));

    /* set maximum number of file records according to max memory limit */
    /* NOTE: maximum number of records is based on the size of a posix file record */
    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
263
    posix_runtime->file_array_ndx = 0;
264 265

    /* allocate array of runtime file records */
266 267 268 269 270
    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
                                               sizeof(struct posix_runtime_file));
    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
                                              sizeof(struct darshan_posix_file));
    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
271 272 273 274
    {
        posix_runtime->file_array_size = 0;
        return;
    }
275 276 277 278
    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
           sizeof(struct posix_runtime_file));
    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
           sizeof(struct darshan_posix_file));
279

280 281
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);

282
#if 0
283 284 285 286
    /* set the memory alignment according to config or environment variables */
    #if (__CP_MEM_ALIGNMENT < 1)
        #error Darshan must be configured with a positive value for --with-mem-align
    #endif
287
    alignstr = getenv(CP_MEM_ALIGNMENT_OVERRIDE);
288
    if(alignstr)
289 290 291 292 293 294 295 296
    {
        ret = sscanf(alignstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            darshan_mem_alignment = tmpval;
        }
    }
297
    else
298 299 300
    {
        darshan_mem_alignment = __CP_MEM_ALIGNMENT;
    }
301

302
    /* avoid floating point errors on faulty input */
303
    if(darshan_mem_alignment < 1)
304 305 306
    {
        darshan_mem_alignment = 1;
    }
307
#endif
308

309 310
    return;
}
311

312
static struct posix_runtime_file* posix_file_by_name(const char *name)
313
{
314
    struct posix_runtime_file *file = NULL;
315
    char *newname = NULL;
316
    darshan_record_id file_id;
317

318
    if(!posix_runtime)
319
        return(NULL);
320

321
    newname = darshan_clean_file_path(name);
322
    if(!newname)
323
        newname = (char*)name;
324

325
    /* get a unique id for this file from darshan core */
326
    darshan_core_lookup_record_id(
327 328 329
        (void*)newname,
        strlen(newname),
        1,
330
        &file_id);
331

332
    /* search the hash table for this file record, and return if found */
333
    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
334
    if(file)
335
    {
336
        if(newname != name)
337
            free(newname);
338
        return(file);
339 340
    }

341 342 343 344 345 346
    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
    {
        /* no existing record, assign a new file record from the global array */
        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
        file->file_record->f_id = file_id;
347

348
        /* add new record to file hash table */
349
        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
350

351 352
        posix_runtime->file_array_ndx++;
    }
353 354

    if(newname != name)
355
        free(newname);
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
    return(file);
}

static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
{
    struct posix_runtime_file* file;
    struct posix_runtime_file_ref* ref;

    if(!posix_runtime)
        return(NULL);

    /* find file record by name first */
    file = posix_file_by_name(name);

    if(!file)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
    {
        /* we have a reference.  Make sure it points to the correct file
         * and return it
         */
        ref->file = file;
        return(file);
    }

    /* if we hit this point, then we don't have a reference for this fd
     * in the table yet.  Add it.
     */
    ref = malloc(sizeof(*ref));
    if(!ref)
        return(NULL);
    memset(ref, 0, sizeof(*ref));
    ref->file = file;
    ref->fd = fd;    

    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);

    return(file);
}

static void posix_file_close_fd(int fd)
{
401
    struct posix_runtime_file_ref* ref;
402 403 404 405 406 407

    if(!posix_runtime)
        return;

    /* search hash table for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
408
    if(ref)
409 410 411 412 413 414 415
    {
        /* we have a reference, delete it */
        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
        free(ref);
    }

    return;
416 417
}

418 419
/* ***************************************************** */

420
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size)
421
{
422
    int comm_cmp;
423
    
424 425 426 427 428 429 430 431 432 433 434 435 436 437
    MPI_Comm_compare(MPI_COMM_WORLD, comm, &comm_cmp);

    /* only do shared file reductions if this communicator includes _everyone_ */
    if((comm_cmp == MPI_IDENT) || (comm_cmp == MPI_CONGRUENT))
    {
        /* don't reduce shared files if that feature is disabled, either */
        if(!getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
        {
            /* TODO reduction code */
        }
    }

    *buffer = (void *)(posix_runtime->file_record_array);
    *size = posix_runtime->file_array_ndx * sizeof(struct darshan_posix_file);
438

439
    return;
440 441
}

442
static void posix_shutdown()
443
{
444 445 446 447 448 449
    /* TODO destroy hash tables ?? */

    free(posix_runtime->file_runtime_array);
    free(posix_runtime->file_record_array);
    free(posix_runtime);
    posix_runtime = NULL;
450 451

    return;
452 453
}

454 455 456 457 458 459 460 461
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */