darshan-posix.c 11.6 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
7
#define _GNU_SOURCE

8
#include "darshan-runtime-config.h"
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
24
#include <libgen.h>
Philip Carns's avatar
Philip Carns committed
25
#include <aio.h>
26
#include <pthread.h>
27

28
#include "uthash.h"
29
#include "darshan.h"
30
#include "darshan-posix-log-format.h"
31

32
#ifndef HAVE_OFF64_T
33
34
typedef int64_t off64_t;
#endif
35
36
37
#ifndef HAVE_AIOCB64
#define aiocb64 aiocb
#endif
38

39
40
/* TODO these go where ? */

41
42
43
44
45
#define DARSHAN_FORWARD_DECL(name,ret,args) \
  extern ret __real_ ## name args;

#define DARSHAN_DECL(__name) __wrap_ ## __name

46
47
#define MAP_OR_FAIL(func)

48
struct posix_runtime_file
49
{
50
    struct darshan_posix_file* file_record;
51
    UT_hash_handle hlink;
52
};
53

54
struct posix_runtime_file_ref
55
{
56
57
58
59
60
61
62
    struct posix_runtime_file* file;
    int fd;
    UT_hash_handle hlink;
};

struct posix_runtime
{
63
64
    struct posix_runtime_file* file_runtime_array;
    struct darshan_posix_file* file_record_array;
65
    int file_array_size;
66
    int file_array_ndx;
67
68
    struct posix_runtime_file* file_hash;
    struct posix_runtime_file_ref* fd_hash;
69
70
};

71
static struct posix_runtime *posix_runtime = NULL;
72
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
73
static int my_rank = -1;
74

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
/* these are paths that we will not trace */
static char* exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

90
DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
91
DARSHAN_FORWARD_DECL(close, int, (int fd));
92

93
94
static void posix_runtime_initialize(void);

95
96
97
98
static struct posix_runtime_file* posix_file_by_name(const char *name);
static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
static void posix_file_close_fd(int fd);

99
100
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size);
static void posix_shutdown(void);
101

102
103
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
104

105
#define POSIX_SET(__file, __counter, __value) do {\
106
    (__file)->file_record->counters[__counter] = __value; \
107
108
109
} while(0)

#define POSIX_F_SET(__file, __counter, __value) do {\
110
    (__file)->file_record->fcounters[__counter] = __value; \
111
112
113
} while(0)

#define POSIX_INC(__file, __counter, __value) do {\
114
    (__file)->file_record->counters[__counter] += __value; \
115
116
117
} while(0)

#define POSIX_F_INC(__file, __counter, __value) do {\
118
    (__file)->file_record->fcounters[__counter] += __value; \
119
120
121
122
123
124
125
126
127
128
129
130
} while(0)

#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
    if(__tm1 > __last) \
        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
    else \
        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
    if(__tm2 > __last) \
        __last = __tm2; \
} while(0)

#define POSIX_VALUE(__file, __counter) \
131
    ((__file)->file_record->counters[__counter])
132
133

#define POSIX_F_VALUE(__file, __counter) \
134
    ((__file)->file_record->fcounters[__counter])
135
136

#define POSIX_MAX(__file, __counter, __value) do {\
137
    if((__file)->file_record->counters[__counter] < __value) \
138
    { \
139
        (__file)->file_record->counters[__counter] = __value; \
140
141
142
    } \
} while(0)

143
#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
144
    struct posix_runtime_file* file; \
145
146
147
148
149
150
151
152
153
    char* exclude; \
    int tmp_index = 0; \
    if(__ret < 0) break; \
    while((exclude = exclusions[tmp_index])) { \
        if(!(strncmp(exclude, __path, strlen(exclude)))) \
            break; \
        tmp_index++; \
    } \
    if(exclude) break; \
154
    file = posix_file_by_name_setfd(__path, __ret); \
155
    if(!file) break; \
156
    file->file_record->rank = my_rank; \
157
    if(__mode) \
158
        POSIX_SET(file, CP_MODE, __mode); \
159
    if(__stream_flag)\
160
        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
161
    else \
162
163
164
165
        POSIX_INC(file, CP_POSIX_OPENS, 1); \
    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
} while(0)
166

167
int DARSHAN_DECL(open)(const char *path, int flags, ...)
168
169
170
171
172
{
    int mode = 0;
    int ret;
    double tm1, tm2;

173
174
    MAP_OR_FAIL(open);

175
    if(flags & O_CREAT) 
176
177
178
179
180
181
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

182
        tm1 = darshan_core_wtime();
183
        ret = __real_open(path, flags, mode);
184
        tm2 = darshan_core_wtime();
185
186
187
    }
    else
    {
188
        tm1 = darshan_core_wtime();
189
        ret = __real_open(path, flags);
190
        tm2 = darshan_core_wtime();
191
192
    }

193
194
195
    POSIX_LOCK();
    posix_runtime_initialize();

196
    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
197

198
    POSIX_UNLOCK();
199
200
201
202

    return(ret);
}

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
int DARSHAN_DECL(close)(int fd)
{
    struct darshan_file_runtime* file;
    int tmp_fd = fd;
    double tm1, tm2;
    int ret;

    MAP_OR_FAIL(close);

    tm1 = darshan_core_wtime();
    ret = __real_close(fd);
    tm2 = darshan_core_wtime();

    POSIX_LOCK();
    posix_runtime_initialize();
218

219
    posix_file_close_fd(tmp_fd);
220
221
222
223
224
225
226

    POSIX_UNLOCK();    

    return(ret);
}

/* ***************************************************** */
227

228
static void posix_runtime_initialize()
229
{
230
231
232
233
234
235
236
    char *alignstr;
    int tmpval;
    int ret;
    int mem_limit;
    struct darshan_module_funcs posix_mod_fns =
    {
        .get_output_data = &posix_get_output_data,
237
        .shutdown = &posix_shutdown
238
    };
239

240
    if(posix_runtime)
241
        return;
242

243
244
    /* register the posix module with darshan core */
    darshan_core_register_module(
245
        DARSHAN_POSIX_MOD,
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
        &posix_mod_fns,
        &mem_limit);

    /* return if no memory assigned by darshan core */
    if(mem_limit == 0)
        return;

    posix_runtime = malloc(sizeof(*posix_runtime));
    if(!posix_runtime)
        return;
    memset(posix_runtime, 0, sizeof(*posix_runtime));

    /* set maximum number of file records according to max memory limit */
    /* NOTE: maximum number of records is based on the size of a posix file record */
    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
261
    posix_runtime->file_array_ndx = 0;
262
263

    /* allocate array of runtime file records */
264
265
266
267
268
    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
                                               sizeof(struct posix_runtime_file));
    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
                                              sizeof(struct darshan_posix_file));
    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
269
270
271
272
    {
        posix_runtime->file_array_size = 0;
        return;
    }
273
274
275
276
    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
           sizeof(struct posix_runtime_file));
    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
           sizeof(struct darshan_posix_file));
277

278
279
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);

280
#if 0
281
282
283
284
    /* set the memory alignment according to config or environment variables */
    #if (__CP_MEM_ALIGNMENT < 1)
        #error Darshan must be configured with a positive value for --with-mem-align
    #endif
285
    alignstr = getenv(CP_MEM_ALIGNMENT_OVERRIDE);
286
    if(alignstr)
287
288
289
290
291
292
293
294
    {
        ret = sscanf(alignstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            darshan_mem_alignment = tmpval;
        }
    }
295
    else
296
297
298
    {
        darshan_mem_alignment = __CP_MEM_ALIGNMENT;
    }
299

300
    /* avoid floating point errors on faulty input */
301
    if(darshan_mem_alignment < 1)
302
303
304
    {
        darshan_mem_alignment = 1;
    }
305
#endif
306

307
308
    return;
}
309

310
static struct posix_runtime_file* posix_file_by_name(const char *name)
311
{
312
    struct posix_runtime_file *file = NULL;
313
    char *newname = NULL;
314
    darshan_record_id file_id;
315

316
    if(!posix_runtime)
317
        return(NULL);
318

319
    newname = darshan_clean_file_path(name);
320
    if(!newname)
321
        newname = (char*)name;
322

323
    /* get a unique id for this file from darshan core */
324
    darshan_core_lookup_record_id(
325
326
327
        (void*)newname,
        strlen(newname),
        1,
328
        &file_id);
329

330
    /* search the hash table for this file record, and return if found */
331
    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
332
    if(file)
333
    {
334
        if(newname != name)
335
            free(newname);
336
        return(file);
337
338
    }

339
340
341
342
343
344
    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
    {
        /* no existing record, assign a new file record from the global array */
        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
        file->file_record->f_id = file_id;
345

346
        /* add new record to file hash table */
347
        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
348

349
350
        posix_runtime->file_array_ndx++;
    }
351
352

    if(newname != name)
353
        free(newname);
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
    return(file);
}

static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
{
    struct posix_runtime_file* file;
    struct posix_runtime_file_ref* ref;

    if(!posix_runtime)
        return(NULL);

    /* find file record by name first */
    file = posix_file_by_name(name);

    if(!file)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
    {
        /* we have a reference.  Make sure it points to the correct file
         * and return it
         */
        ref->file = file;
        return(file);
    }

    /* if we hit this point, then we don't have a reference for this fd
     * in the table yet.  Add it.
     */
    ref = malloc(sizeof(*ref));
    if(!ref)
        return(NULL);
    memset(ref, 0, sizeof(*ref));
    ref->file = file;
    ref->fd = fd;    

    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);

    return(file);
}

static void posix_file_close_fd(int fd)
{
399
    struct posix_runtime_file_ref* ref;
400
401
402
403
404
405

    if(!posix_runtime)
        return;

    /* search hash table for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
406
    if(ref)
407
408
409
410
411
412
413
    {
        /* we have a reference, delete it */
        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
        free(ref);
    }

    return;
414
415
}

416
417
/* ***************************************************** */

418
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size)
419
{
420
    int comm_cmp;
421
    
422
    DARSHAN_MPI_CALL(PMPI_Comm_compare)(MPI_COMM_WORLD, comm, &comm_cmp);
423
424
425
426
427
428
429
430
431
432
433
434
435

    /* only do shared file reductions if this communicator includes _everyone_ */
    if((comm_cmp == MPI_IDENT) || (comm_cmp == MPI_CONGRUENT))
    {
        /* don't reduce shared files if that feature is disabled, either */
        if(!getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
        {
            /* TODO reduction code */
        }
    }

    *buffer = (void *)(posix_runtime->file_record_array);
    *size = posix_runtime->file_array_ndx * sizeof(struct darshan_posix_file);
436

437
    return;
438
439
}

440
static void posix_shutdown()
441
{
442
443
444
445
446
447
    /* TODO destroy hash tables ?? */

    free(posix_runtime->file_runtime_array);
    free(posix_runtime->file_record_array);
    free(posix_runtime);
    posix_runtime = NULL;
448
449

    return;
450
451
}

452
453
454
455
456
457
458
459
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */