darshan-posix.c 11.6 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
7
#define _GNU_SOURCE

8
#include "darshan-runtime-config.h"
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <search.h>
#include <assert.h>
24
#include <libgen.h>
Philip Carns's avatar
Philip Carns committed
25
#include <aio.h>
26
#include <pthread.h>
27

28
#include "uthash.h"
29
#include "darshan.h"
30

31
#ifndef HAVE_OFF64_T
32
33
typedef int64_t off64_t;
#endif
34
35
36
#ifndef HAVE_AIOCB64
#define aiocb64 aiocb
#endif
37

38
39
/* TODO these go where ? */

40
41
42
43
44
#define DARSHAN_FORWARD_DECL(name,ret,args) \
  extern ret __real_ ## name args;

#define DARSHAN_DECL(__name) __wrap_ ## __name

45
46
#define MAP_OR_FAIL(func)

47
48
#define POSIX_MOD_NAME "POSIX"

49
struct posix_runtime_file
50
{
51
    struct darshan_posix_file* file_record;
52
    UT_hash_handle hlink;
53
};
54

55
struct posix_runtime_file_ref
56
{
57
58
59
60
61
62
63
    struct posix_runtime_file* file;
    int fd;
    UT_hash_handle hlink;
};

struct posix_runtime
{
64
65
    struct posix_runtime_file* file_runtime_array;
    struct darshan_posix_file* file_record_array;
66
    int file_array_size;
67
    int file_array_ndx;
68
69
    struct posix_runtime_file* file_hash;
    struct posix_runtime_file_ref* fd_hash;
70
71
};

72
static struct posix_runtime *posix_runtime = NULL;
73
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
74
static int my_rank = -1;
75

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* these are paths that we will not trace */
static char* exclusions[] = {
"/etc/",
"/dev/",
"/usr/",
"/bin/",
"/boot/",
"/lib/",
"/opt/",
"/sbin/",
"/sys/",
"/proc/",
NULL
};

91
DARSHAN_FORWARD_DECL(open, int, (const char *path, int flags, ...));
92
DARSHAN_FORWARD_DECL(close, int, (int fd));
93

94
95
static void posix_runtime_initialize(void);

96
97
98
99
static struct posix_runtime_file* posix_file_by_name(const char *name);
static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
static void posix_file_close_fd(int fd);

100
101
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size);
static void posix_shutdown(void);
102

103
104
#define POSIX_LOCK() pthread_mutex_lock(&posix_runtime_mutex)
#define POSIX_UNLOCK() pthread_mutex_unlock(&posix_runtime_mutex)
105

106
#define POSIX_SET(__file, __counter, __value) do {\
107
    (__file)->file_record->counters[__counter] = __value; \
108
109
110
} while(0)

#define POSIX_F_SET(__file, __counter, __value) do {\
111
    (__file)->file_record->fcounters[__counter] = __value; \
112
113
114
} while(0)

#define POSIX_INC(__file, __counter, __value) do {\
115
    (__file)->file_record->counters[__counter] += __value; \
116
117
118
} while(0)

#define POSIX_F_INC(__file, __counter, __value) do {\
119
    (__file)->file_record->fcounters[__counter] += __value; \
120
121
122
123
124
125
126
127
128
129
130
131
} while(0)

#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
    if(__tm1 > __last) \
        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
    else \
        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
    if(__tm2 > __last) \
        __last = __tm2; \
} while(0)

#define POSIX_VALUE(__file, __counter) \
132
    ((__file)->file_record->counters[__counter])
133
134

#define POSIX_F_VALUE(__file, __counter) \
135
    ((__file)->file_record->fcounters[__counter])
136
137

#define POSIX_MAX(__file, __counter, __value) do {\
138
    if((__file)->file_record->counters[__counter] < __value) \
139
    { \
140
        (__file)->file_record->counters[__counter] = __value; \
141
142
143
    } \
} while(0)

144
#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
145
    struct posix_runtime_file* file; \
146
147
148
149
150
151
152
153
154
    char* exclude; \
    int tmp_index = 0; \
    if(__ret < 0) break; \
    while((exclude = exclusions[tmp_index])) { \
        if(!(strncmp(exclude, __path, strlen(exclude)))) \
            break; \
        tmp_index++; \
    } \
    if(exclude) break; \
155
    file = posix_file_by_name_setfd(__path, __ret); \
156
    if(!file) break; \
157
    file->file_record->rank = my_rank; \
158
    if(__mode) \
159
        POSIX_SET(file, CP_MODE, __mode); \
160
    if(__stream_flag)\
161
        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
162
    else \
163
164
165
166
        POSIX_INC(file, CP_POSIX_OPENS, 1); \
    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
} while(0)
167

168
int DARSHAN_DECL(open)(const char *path, int flags, ...)
169
170
171
172
173
{
    int mode = 0;
    int ret;
    double tm1, tm2;

174
175
    MAP_OR_FAIL(open);

176
    if(flags & O_CREAT) 
177
178
179
180
181
182
    {
        va_list arg;
        va_start(arg, flags);
        mode = va_arg(arg, int);
        va_end(arg);

183
        tm1 = darshan_core_wtime();
184
        ret = __real_open(path, flags, mode);
185
        tm2 = darshan_core_wtime();
186
187
188
    }
    else
    {
189
        tm1 = darshan_core_wtime();
190
        ret = __real_open(path, flags);
191
        tm2 = darshan_core_wtime();
192
193
    }

194
195
196
    POSIX_LOCK();
    posix_runtime_initialize();

197
    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
198

199
    POSIX_UNLOCK();
200
201
202
203

    return(ret);
}

204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
int DARSHAN_DECL(close)(int fd)
{
    struct darshan_file_runtime* file;
    int tmp_fd = fd;
    double tm1, tm2;
    int ret;

    MAP_OR_FAIL(close);

    tm1 = darshan_core_wtime();
    ret = __real_close(fd);
    tm2 = darshan_core_wtime();

    POSIX_LOCK();
    posix_runtime_initialize();
219

220
    posix_file_close_fd(tmp_fd);
221
222
223
224
225
226
227

    POSIX_UNLOCK();    

    return(ret);
}

/* ***************************************************** */
228

229
static void posix_runtime_initialize()
230
{
231
232
233
234
235
236
237
    char *alignstr;
    int tmpval;
    int ret;
    int mem_limit;
    struct darshan_module_funcs posix_mod_fns =
    {
        .get_output_data = &posix_get_output_data,
238
        .shutdown = &posix_shutdown
239
    };
240

241
    if(posix_runtime)
242
        return;
243

244
245
    /* register the posix module with darshan core */
    darshan_core_register_module(
246
        DARSHAN_POSIX_MOD,
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
        POSIX_MOD_NAME,
        &posix_mod_fns,
        &mem_limit);

    /* return if no memory assigned by darshan core */
    if(mem_limit == 0)
        return;

    posix_runtime = malloc(sizeof(*posix_runtime));
    if(!posix_runtime)
        return;
    memset(posix_runtime, 0, sizeof(*posix_runtime));

    /* set maximum number of file records according to max memory limit */
    /* NOTE: maximum number of records is based on the size of a posix file record */
    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
263
    posix_runtime->file_array_ndx = 0;
264
265

    /* allocate array of runtime file records */
266
267
268
269
270
    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
                                               sizeof(struct posix_runtime_file));
    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
                                              sizeof(struct darshan_posix_file));
    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
271
272
273
274
    {
        posix_runtime->file_array_size = 0;
        return;
    }
275
276
277
278
    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
           sizeof(struct posix_runtime_file));
    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
           sizeof(struct darshan_posix_file));
279

280
281
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);

282
#if 0
283
284
285
286
    /* set the memory alignment according to config or environment variables */
    #if (__CP_MEM_ALIGNMENT < 1)
        #error Darshan must be configured with a positive value for --with-mem-align
    #endif
287
    alignstr = getenv(CP_MEM_ALIGNMENT_OVERRIDE);
288
    if(alignstr)
289
290
291
292
293
294
295
296
    {
        ret = sscanf(alignstr, "%d", &tmpval);
        /* silently ignore if the env variable is set poorly */
        if(ret == 1 && tmpval > 0)
        {
            darshan_mem_alignment = tmpval;
        }
    }
297
    else
298
299
300
    {
        darshan_mem_alignment = __CP_MEM_ALIGNMENT;
    }
301

302
    /* avoid floating point errors on faulty input */
303
    if(darshan_mem_alignment < 1)
304
305
306
    {
        darshan_mem_alignment = 1;
    }
307
#endif
308

309
310
    return;
}
311

312
static struct posix_runtime_file* posix_file_by_name(const char *name)
313
{
314
    struct posix_runtime_file *file = NULL;
315
    char *newname = NULL;
316
    darshan_record_id file_id;
317

318
    if(!posix_runtime)
319
        return(NULL);
320

321
    newname = darshan_clean_file_path(name);
322
    if(!newname)
323
        newname = (char*)name;
324

325
    /* get a unique id for this file from darshan core */
326
    darshan_core_lookup_record_id(
327
328
329
        (void*)newname,
        strlen(newname),
        1,
330
        &file_id);
331

332
    /* search the hash table for this file record, and return if found */
333
    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
334
    if(file)
335
    {
336
        if(newname != name)
337
            free(newname);
338
        return(file);
339
340
    }

341
342
343
344
345
346
    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
    {
        /* no existing record, assign a new file record from the global array */
        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
        file->file_record->f_id = file_id;
347

348
        /* add new record to file hash table */
349
        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
350

351
352
        posix_runtime->file_array_ndx++;
    }
353
354

    if(newname != name)
355
        free(newname);
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
    return(file);
}

static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
{
    struct posix_runtime_file* file;
    struct posix_runtime_file_ref* ref;

    if(!posix_runtime)
        return(NULL);

    /* find file record by name first */
    file = posix_file_by_name(name);

    if(!file)
        return(NULL);

    /* search hash table for existing file ref for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
    if(ref)
    {
        /* we have a reference.  Make sure it points to the correct file
         * and return it
         */
        ref->file = file;
        return(file);
    }

    /* if we hit this point, then we don't have a reference for this fd
     * in the table yet.  Add it.
     */
    ref = malloc(sizeof(*ref));
    if(!ref)
        return(NULL);
    memset(ref, 0, sizeof(*ref));
    ref->file = file;
    ref->fd = fd;    

    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);

    return(file);
}

static void posix_file_close_fd(int fd)
{
401
    struct posix_runtime_file_ref* ref;
402
403
404
405
406
407

    if(!posix_runtime)
        return;

    /* search hash table for this fd */
    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
408
    if(ref)
409
410
411
412
413
414
415
    {
        /* we have a reference, delete it */
        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
        free(ref);
    }

    return;
416
417
}

418
419
/* ***************************************************** */

420
static void posix_get_output_data(MPI_Comm comm, void **buffer, int *size)
421
{
422
    int comm_cmp;
423
    
424
425
426
427
428
429
430
431
432
433
434
435
436
437
    MPI_Comm_compare(MPI_COMM_WORLD, comm, &comm_cmp);

    /* only do shared file reductions if this communicator includes _everyone_ */
    if((comm_cmp == MPI_IDENT) || (comm_cmp == MPI_CONGRUENT))
    {
        /* don't reduce shared files if that feature is disabled, either */
        if(!getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
        {
            /* TODO reduction code */
        }
    }

    *buffer = (void *)(posix_runtime->file_record_array);
    *size = posix_runtime->file_array_ndx * sizeof(struct darshan_posix_file);
438

439
    return;
440
441
}

442
static void posix_shutdown()
443
{
444
445
446
447
448
449
    /* TODO destroy hash tables ?? */

    free(posix_runtime->file_runtime_array);
    free(posix_runtime->file_record_array);
    free(posix_runtime);
    posix_runtime = NULL;
450
451

    return;
452
453
}

454
455
456
457
458
459
460
461
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */