darshan-posix.c 12.2 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
7
#define _GNU_SOURCE

8
#include "darshan-runtime-config.h"
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
24
#include <libgen.h>
Philip Carns's avatar
Philip Carns committed
25
#include <aio.h>
26
#include <pthread.h>
27

28
#include "uthash.h"
29
#include "darshan.h"
30
#include "darshan-posix-log-format.h"
31

32
#ifndef HAVE_OFF64_T
33
34
typedef int64_t off64_t;
#endif
35
36
37
#ifndef HAVE_AIOCB64
#define aiocb64 aiocb
#endif
38

39
40
/* TODO these go where ? */

41
42
43
44
45
#define DARSHAN_FORWARD_DECL(name,ret,args) \
  extern ret __real_ ## name args;

#define DARSHAN_DECL(__name) __wrap_ ## __name

46
47
#define MAP_OR_FAIL(func)

48
struct posix_runtime_file
49
{
50
    struct darshan_posix_file* file_record;
51
    UT_hash_handle hlink;
52
};
53

54
struct posix_runtime_file_ref
55
{
56
57
58
59
60
61
62
    struct posix_runtime_file* file;
    int fd;
    UT_hash_handle hlink;
};

struct posix_runtime
{
63
64
    struct posix_runtime_file* file_runtime_array;
    struct darshan_posix_file* file_record_array;
65
    int file_array_size;
66
    int file_array_ndx;
67
68
    struct posix_runtime_file* file_hash;
    struct posix_runtime_file_ref* fd_hash;
69
70
};

71
static struct posix_runtime *posix_runtime = NULL;
72
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
Shane Snyder's avatar
Shane Snyder committed
73
static int instrumentation_disabled = 0;
74
static int my_rank = -1;
75

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* these are paths that we will not trace */
static char* exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

91
DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
Shane Snyder's avatar
Shane Snyder committed
92
DARSHAN_FORWARD_DECL(open64, int, (const char *path, int flags, ...));
93
DARSHAN_FORWARD_DECL(close, int, (int fd));
94

95
96
static void posix_runtime_initialize(void);

97
98
static struct posix_runtime_file* posix_file_by_name(const char *name);
static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
Shane Snyder's avatar
Shane Snyder committed
99
static struct posix_runtime_file* posix_file_by_fd(int fd);
100
101
static void posix_file_close_fd(int fd);

Shane Snyder's avatar
Shane Snyder committed
102
static void posix_disable_instrumentation(void);
103
104
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size);
static void posix_shutdown(void);
105

106
107
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
108

109
#define POSIX_SET(__file, __counter, __value) do {\
110
    (__file)->file_record->counters[__counter] = __value; \
111
112
113
} while(0)

#define POSIX_F_SET(__file, __counter, __value) do {\
114
    (__file)->file_record->fcounters[__counter] = __value; \
115
116
117
} while(0)

#define POSIX_INC(__file, __counter, __value) do {\
118
    (__file)->file_record->counters[__counter] += __value; \
119
120
121
} while(0)

#define POSIX_F_INC(__file, __counter, __value) do {\
122
    (__file)->file_record->fcounters[__counter] += __value; \
123
124
125
126
127
128
129
130
131
132
133
134
} while(0)

#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
    if(__tm1 > __last) \
        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
    else \
        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
    if(__tm2 > __last) \
        __last = __tm2; \
} while(0)

#define POSIX_VALUE(__file, __counter) \
135
    ((__file)->file_record->counters[__counter])
136
137

#define POSIX_F_VALUE(__file, __counter) \
138
    ((__file)->file_record->fcounters[__counter])
139
140

#define POSIX_MAX(__file, __counter, __value) do {\
141
    if((__file)->file_record->counters[__counter] < __value) \
142
    { \
143
        (__file)->file_record->counters[__counter] = __value; \
144
145
146
    } \
} while(0)

147
#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
148
    struct posix_runtime_file* file; \
149
150
151
152
153
154
155
156
157
    char* exclude; \
    int tmp_index = 0; \
    if(__ret < 0) break; \
    while((exclude = exclusions[tmp_index])) { \
        if(!(strncmp(exclude, __path, strlen(exclude)))) \
            break; \
        tmp_index++; \
    } \
    if(exclude) break; \
158
    file = posix_file_by_name_setfd(__path, __ret); \
159
    if(!file) break; \
160
    file->file_record->rank = my_rank; \
161
    if(__mode) \
162
        POSIX_SET(file, CP_MODE, __mode); \
163
    if(__stream_flag)\
164
        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
165
    else \
166
167
168
169
        POSIX_INC(file, CP_POSIX_OPENS, 1); \
    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
} while(0)
170

171
int DARSHAN_DECL(open)(const char *path, int flags, ...)
172
173
174
175
176
{
    int mode = 0;
    int ret;
    double tm1, tm2;

177
178
    MAP_OR_FAIL(open);

179
    if(flags & O_CREAT) 
180
181
182
183
184
185
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

186
        tm1 = darshan_core_wtime();
187
        ret = __real_open(path, flags, mode);
188
        tm2 = darshan_core_wtime();
189
190
191
    }
    else
    {
192
        tm1 = darshan_core_wtime();
193
        ret = __real_open(path, flags);
194
        tm2 = darshan_core_wtime();
195
196
    }

197
198
199
    POSIX_LOCK();
    posix_runtime_initialize();

200
    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
201

202
    POSIX_UNLOCK();
203
204
205
206

    return(ret);
}

Shane Snyder's avatar
Shane Snyder committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
int DARSHAN_DECL(open64)(const char *path, int flags, ...)
{
    int mode = 0;
    int ret;
    double tm1, tm2;

    MAP_OR_FAIL(open);

    if(flags & O_CREAT)
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

        tm1 = darshan_core_wtime();
        ret = __real_open64(path, flags, mode);
        tm2 = darshan_core_wtime();
    }
    else
    {
        tm1 = darshan_core_wtime();
        ret = __real_open64(path, flags);
        tm2 = darshan_core_wtime();
    }

    POSIX_LOCK();
    posix_runtime_initialize();

    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
    POSIX_UNLOCK();

    return(ret);
}

242
243
int DARSHAN_DECL(close)(int fd)
{
Shane Snyder's avatar
Shane Snyder committed
244
    struct posix_runtime_file* file;
245
246
247
248
249
250
251
252
253
254
255
    double tm1, tm2;
    int ret;

    MAP_OR_FAIL(close);

    tm1 = darshan_core_wtime();
    ret = __real_close(fd);
    tm2 = darshan_core_wtime();

    POSIX_LOCK();
    posix_runtime_initialize();
256

Shane Snyder's avatar
Shane Snyder committed
257
258
259
260
261
262
    file = posix_file_by_fd(fd);
    if(file)
    {
        POSIX_F_SET(file, CP_F_CLOSE_TIMESTAMP, darshan_core_wtime());
        posix_file_close_fd(fd);
    }
263
264
265
266
267
268
    POSIX_UNLOCK();    

    return(ret);
}

/* ***************************************************** */
269

270
static void posix_runtime_initialize()
271
{
272
273
274
275
276
277
    char *alignstr;
    int tmpval;
    int ret;
    int mem_limit;
    struct darshan_module_funcs posix_mod_fns =
    {
Shane Snyder's avatar
Shane Snyder committed
278
        .disable_instrumentation = &posix_disable_instrumentation,
279
        .get_output_data = &posix_get_output_data,
280
        .shutdown = &posix_shutdown
281
    };
282

Shane Snyder's avatar
Shane Snyder committed
283
284
    /* don't do anything if already initialized or instrumenation is disabled */
    if(posix_runtime || instrumentation_disabled)
285
        return;
286

287
288
    /* register the posix module with darshan core */
    darshan_core_register_module(
289
        DARSHAN_POSIX_MOD,
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
        &posix_mod_fns,
        &mem_limit);

    /* return if no memory assigned by darshan core */
    if(mem_limit == 0)
        return;

    posix_runtime = malloc(sizeof(*posix_runtime));
    if(!posix_runtime)
        return;
    memset(posix_runtime, 0, sizeof(*posix_runtime));

    /* set maximum number of file records according to max memory limit */
    /* NOTE: maximum number of records is based on the size of a posix file record */
    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
305
    posix_runtime->file_array_ndx = 0;
306
307

    /* allocate array of runtime file records */
308
309
310
311
312
    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
                                               sizeof(struct posix_runtime_file));
    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
                                              sizeof(struct darshan_posix_file));
    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
313
314
315
316
    {
        posix_runtime->file_array_size = 0;
        return;
    }
317
318
319
320
    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
           sizeof(struct posix_runtime_file));
    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
           sizeof(struct darshan_posix_file));
321

Shane Snyder's avatar
Shane Snyder committed
322
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank); /* TODO: can we move this out of here? */
323

324
325
    return;
}
326

327
static struct posix_runtime_file* posix_file_by_name(const char *name)
328
{
329
    struct posix_runtime_file *file = NULL;
330
    char *newname = NULL;
331
    darshan_record_id file_id;
332

Shane Snyder's avatar
Shane Snyder committed
333
    if(!posix_runtime || instrumentation_disabled)
334
        return(NULL);
335

336
    newname = darshan_clean_file_path(name);
337
    if(!newname)
338
        newname = (char*)name;
339

340
    /* get a unique id for this file from darshan core */
341
    darshan_core_lookup_record_id(
342
343
344
        (void*)newname,
        strlen(newname),
        1,
345
        &file_id);
346

347
    /* search the hash table for this file record, and return if found */
348
    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
349
    if(file)
350
    {
351
        if(newname != name)
352
            free(newname);
353
        return(file);
354
355
    }

356
357
358
359
360
361
    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
    {
        /* no existing record, assign a new file record from the global array */
        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
        file->file_record->f_id = file_id;
362

363
        /* add new record to file hash table */
364
        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
365

366
367
        posix_runtime->file_array_ndx++;
    }
368
369

    if(newname != name)
370
        free(newname);
371
372
373
374
375
376
377
378
    return(file);
}

static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
{
    struct posix_runtime_file* file;
    struct posix_runtime_file_ref* ref;

Shane Snyder's avatar
Shane Snyder committed
379
    if(!posix_runtime || instrumentation_disabled)
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
        return(NULL);

    /* find file record by name first */
    file = posix_file_by_name(name);

    if(!file)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
    {
        /* we have a reference.  Make sure it points to the correct file
         * and return it
         */
        ref->file = file;
        return(file);
    }

    /* if we hit this point, then we don't have a reference for this fd
     * in the table yet.  Add it.
     */
    ref = malloc(sizeof(*ref));
    if(!ref)
        return(NULL);
    memset(ref, 0, sizeof(*ref));
Shane Snyder's avatar
Shane Snyder committed
406

407
408
409
410
411
412
413
    ref->file = file;
    ref->fd = fd;    
    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);

    return(file);
}

Shane Snyder's avatar
Shane Snyder committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
static struct posix_runtime_file* posix_file_by_fd(int fd)
{
    struct posix_runtime_file_ref* ref;

    if(!posix_runtime || instrumentation_disabled)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
        return(ref->file);

    return(NULL);
}

429
430
static void posix_file_close_fd(int fd)
{
431
    struct posix_runtime_file_ref* ref;
432

Shane Snyder's avatar
Shane Snyder committed
433
    if(!posix_runtime || instrumentation_disabled)
434
435
436
437
        return;

    /* search hash table for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
438
    if(ref)
439
440
441
442
443
444
445
    {
        /* we have a reference, delete it */
        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
        free(ref);
    }

    return;
446
447
}

448
449
/* ***************************************************** */

Shane Snyder's avatar
Shane Snyder committed
450
static void posix_disable_instrumentation()
451
{
Shane Snyder's avatar
Shane Snyder committed
452
453
454
    POSIX_LOCK();
    instrumentation_disabled = 1;
    POSIX_UNLOCK();
455

Shane Snyder's avatar
Shane Snyder committed
456
457
458
459
460
461
    return;
}

static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size)
{
    /* TODO: shared file reduction */
462
463
464

    *buffer = (void *)(posix_runtime->file_record_array);
    *size = posix_runtime->file_array_ndx * sizeof(struct darshan_posix_file);
465

466
    return;
467
468
}

469
static void posix_shutdown()
470
{
471
472
473
474
475
476
    /* TODO destroy hash tables ?? */

    free(posix_runtime->file_runtime_array);
    free(posix_runtime->file_record_array);
    free(posix_runtime);
    posix_runtime = NULL;
477
478

    return;
479
480
}

481
482
483
484
485
486
487
488
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */