darshan-posix.c 11.6 KB
Newer Older
1 2 3 4 5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6 7
#define _GNU_SOURCE

8
#include "darshan-runtime-config.h"
9

10 11 12 13 14 15 16 17 18 19 20 21 22 23
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
24
#include <libgen.h>
25
#include <aio.h>
26
#include <pthread.h>
27

28
#include "uthash.h"
29
#include "darshan.h"
30
#include "darshan-posix-log-format.h"
31

32
#ifndef HAVE_OFF64_T
33 34
typedef int64_t off64_t;
#endif
35 36 37
#ifndef HAVE_AIOCB64
#define aiocb64 aiocb
#endif
38

39 40
/* TODO these go where ? */

41 42 43 44 45
#define DARSHAN_FORWARD_DECL(name,ret,args) \
  extern ret __real_ ## name args;

#define DARSHAN_DECL(__name) __wrap_ ## __name

46 47
#define MAP_OR_FAIL(func)

48
struct posix_runtime_file
49
{
50
    struct darshan_posix_file* file_record;
51
    UT_hash_handle hlink;
52
};
53

54
struct posix_runtime_file_ref
55
{
56 57 58 59 60 61 62
    struct posix_runtime_file* file;
    int fd;
    UT_hash_handle hlink;
};

struct posix_runtime
{
63 64
    struct posix_runtime_file* file_runtime_array;
    struct darshan_posix_file* file_record_array;
65
    int file_array_size;
66
    int file_array_ndx;
67 68
    struct posix_runtime_file* file_hash;
    struct posix_runtime_file_ref* fd_hash;
69 70
};

71
static struct posix_runtime *posix_runtime = NULL;
72
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
73
static int my_rank = -1;
74

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
/* these are paths that we will not trace */
static char* exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

90
DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
91
DARSHAN_FORWARD_DECL(close, int, (int fd));
92

93 94
static void posix_runtime_initialize(void);

95 96 97 98
static struct posix_runtime_file* posix_file_by_name(const char *name);
static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
static void posix_file_close_fd(int fd);

99 100
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size);
static void posix_shutdown(void);
101

102 103
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
104

105
#define POSIX_SET(__file, __counter, __value) do {\
106
    (__file)->file_record->counters[__counter] = __value; \
107 108 109
} while(0)

#define POSIX_F_SET(__file, __counter, __value) do {\
110
    (__file)->file_record->fcounters[__counter] = __value; \
111 112 113
} while(0)

#define POSIX_INC(__file, __counter, __value) do {\
114
    (__file)->file_record->counters[__counter] += __value; \
115 116 117
} while(0)

#define POSIX_F_INC(__file, __counter, __value) do {\
118
    (__file)->file_record->fcounters[__counter] += __value; \
119 120 121 122 123 124 125 126 127 128 129 130
} while(0)

#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
    if(__tm1 > __last) \
        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
    else \
        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
    if(__tm2 > __last) \
        __last = __tm2; \
} while(0)

#define POSIX_VALUE(__file, __counter) \
131
    ((__file)->file_record->counters[__counter])
132 133

#define POSIX_F_VALUE(__file, __counter) \
134
    ((__file)->file_record->fcounters[__counter])
135 136

#define POSIX_MAX(__file, __counter, __value) do {\
137
    if((__file)->file_record->counters[__counter] < __value) \
138
    { \
139
        (__file)->file_record->counters[__counter] = __value; \
140 141 142
    } \
} while(0)

143
#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
144
    struct posix_runtime_file* file; \
145 146 147 148 149 150 151 152 153
    char* exclude; \
    int tmp_index = 0; \
    if(__ret < 0) break; \
    while((exclude = exclusions[tmp_index])) { \
        if(!(strncmp(exclude, __path, strlen(exclude)))) \
            break; \
        tmp_index++; \
    } \
    if(exclude) break; \
154
    file = posix_file_by_name_setfd(__path, __ret); \
155
    if(!file) break; \
156
    file->file_record->rank = my_rank; \
157
    if(__mode) \
158
        POSIX_SET(file, CP_MODE, __mode); \
159
    if(__stream_flag)\
160
        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
161
    else \
162 163 164 165
        POSIX_INC(file, CP_POSIX_OPENS, 1); \
    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
} while(0)
166

167
int DARSHAN_DECL(open)(const char *path, int flags, ...)
168 169 170 171 172
{
    int mode = 0;
    int ret;
    double tm1, tm2;

173 174
    MAP_OR_FAIL(open);

175
    if(flags & O_CREAT) 
176 177 178 179 180 181
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

182
        tm1 = darshan_core_wtime();
183
        ret = __real_open(path, flags, mode);
184
        tm2 = darshan_core_wtime();
185 186 187
    }
    else
    {
188
        tm1 = darshan_core_wtime();
189
        ret = __real_open(path, flags);
190
        tm2 = darshan_core_wtime();
191 192
    }

193 194 195
    POSIX_LOCK();
    posix_runtime_initialize();

196
    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
197

198
    POSIX_UNLOCK();
199 200 201 202

    return(ret);
}

203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
int DARSHAN_DECL(close)(int fd)
{
    struct darshan_file_runtime* file;
    int tmp_fd = fd;
    double tm1, tm2;
    int ret;

    MAP_OR_FAIL(close);

    tm1 = darshan_core_wtime();
    ret = __real_close(fd);
    tm2 = darshan_core_wtime();

    POSIX_LOCK();
    posix_runtime_initialize();
218

219
    posix_file_close_fd(tmp_fd);
220 221 222 223 224 225 226

    POSIX_UNLOCK();    

    return(ret);
}

/* ***************************************************** */
227

228
static void posix_runtime_initialize()
229
{
230 231 232 233 234 235 236
    char *alignstr;
    int tmpval;
    int ret;
    int mem_limit;
    struct darshan_module_funcs posix_mod_fns =
    {
        .get_output_data = &posix_get_output_data,
237
        .shutdown = &posix_shutdown
238
    };
239

240
    if(posix_runtime)
241
        return;
242

243 244
    /* register the posix module with darshan core */
    darshan_core_register_module(
245
        DARSHAN_POSIX_MOD,
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
        &posix_mod_fns,
        &mem_limit);

    /* return if no memory assigned by darshan core */
    if(mem_limit == 0)
        return;

    posix_runtime = malloc(sizeof(*posix_runtime));
    if(!posix_runtime)
        return;
    memset(posix_runtime, 0, sizeof(*posix_runtime));

    /* set maximum number of file records according to max memory limit */
    /* NOTE: maximum number of records is based on the size of a posix file record */
    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
261
    posix_runtime->file_array_ndx = 0;
262 263

    /* allocate array of runtime file records */
264 265 266 267 268
    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
                                               sizeof(struct posix_runtime_file));
    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
                                              sizeof(struct darshan_posix_file));
    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
269 270 271 272
    {
        posix_runtime->file_array_size = 0;
        return;
    }
273 274 275 276
    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
           sizeof(struct posix_runtime_file));
    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
           sizeof(struct darshan_posix_file));
277

278 279
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);

280
#if 0
281 282 283 284
    /* set the memory alignment according to config or environment variables */
    #if (__CP_MEM_ALIGNMENT < 1)
        #error Darshan must be configured with a positive value for --with-mem-align
    #endif
285
    alignstr = getenv(CP_MEM_ALIGNMENT_OVERRIDE);
286
    if(alignstr)
287 288 289 290 291 292 293 294
    {
        ret = sscanf(alignstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            darshan_mem_alignment = tmpval;
        }
    }
295
    else
296 297 298
    {
        darshan_mem_alignment = __CP_MEM_ALIGNMENT;
    }
299

300
    /* avoid floating point errors on faulty input */
301
    if(darshan_mem_alignment < 1)
302 303 304
    {
        darshan_mem_alignment = 1;
    }
305
#endif
306

307 308
    return;
}
309

310
static struct posix_runtime_file* posix_file_by_name(const char *name)
311
{
312
    struct posix_runtime_file *file = NULL;
313
    char *newname = NULL;
314
    darshan_record_id file_id;
315

316
    if(!posix_runtime)
317
        return(NULL);
318

319
    newname = darshan_clean_file_path(name);
320
    if(!newname)
321
        newname = (char*)name;
322

323
    /* get a unique id for this file from darshan core */
324
    darshan_core_lookup_record_id(
325 326 327
        (void*)newname,
        strlen(newname),
        1,
328
        &file_id);
329

330
    /* search the hash table for this file record, and return if found */
331
    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
332
    if(file)
333
    {
334
        if(newname != name)
335
            free(newname);
336
        return(file);
337 338
    }

339 340 341 342 343 344
    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
    {
        /* no existing record, assign a new file record from the global array */
        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
        file->file_record->f_id = file_id;
345

346
        /* add new record to file hash table */
347
        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
348

349 350
        posix_runtime->file_array_ndx++;
    }
351 352

    if(newname != name)
353
        free(newname);
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
    return(file);
}

static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
{
    struct posix_runtime_file* file;
    struct posix_runtime_file_ref* ref;

    if(!posix_runtime)
        return(NULL);

    /* find file record by name first */
    file = posix_file_by_name(name);

    if(!file)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
    {
        /* we have a reference.  Make sure it points to the correct file
         * and return it
         */
        ref->file = file;
        return(file);
    }

    /* if we hit this point, then we don't have a reference for this fd
     * in the table yet.  Add it.
     */
    ref = malloc(sizeof(*ref));
    if(!ref)
        return(NULL);
    memset(ref, 0, sizeof(*ref));
    ref->file = file;
    ref->fd = fd;    

    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);

    return(file);
}

static void posix_file_close_fd(int fd)
{
399
    struct posix_runtime_file_ref* ref;
400 401 402 403 404 405

    if(!posix_runtime)
        return;

    /* search hash table for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
406
    if(ref)
407 408 409 410 411 412 413
    {
        /* we have a reference, delete it */
        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
        free(ref);
    }

    return;
414 415
}

416 417
/* ***************************************************** */

418
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size)
419
{
420
    int comm_cmp;
421
    
422
    DARSHAN_MPI_CALL(PMPI_Comm_compare)(MPI_COMM_WORLD, comm, &comm_cmp);
423 424 425 426 427 428 429 430 431 432 433 434 435

    /* only do shared file reductions if this communicator includes _everyone_ */
    if((comm_cmp == MPI_IDENT) || (comm_cmp == MPI_CONGRUENT))
    {
        /* don't reduce shared files if that feature is disabled, either */
        if(!getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
        {
            /* TODO reduction code */
        }
    }

    *buffer = (void *)(posix_runtime->file_record_array);
    *size = posix_runtime->file_array_ndx * sizeof(struct darshan_posix_file);
436

437
    return;
438 439
}

440
static void posix_shutdown()
441
{
442 443 444 445 446 447
    /* TODO destroy hash tables ?? */

    free(posix_runtime->file_runtime_array);
    free(posix_runtime->file_record_array);
    free(posix_runtime);
    posix_runtime = NULL;
448 449

    return;
450 451
}

452 453 454 455 456 457 458 459
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */