darshan-core.c 14.2 KB
Newer Older
1
2
3
4
5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6
#define _GNU_SOURCE
7

8
9
10
11
12
13
#include "darshan-runtime-config.h"

#include <stdio.h>
#ifdef HAVE_MNTENT_H
#include <mntent.h>
#endif
14
15
16
17
18
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <pthread.h>
19
20
21
22
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/vfs.h>
#include <mpi.h>
23

24
25
#include "darshan-core.h"

26
27
/* TODO is __progname_full needed here */
extern char* __progname;
28

29
/* internal variables */
30
static struct darshan_core_job_runtime *darshan_core_job = NULL;
31
static pthread_mutex_t darshan_mutex = PTHREAD_MUTEX_INITIALIZER;
32
33
34
35
36
static int my_rank = -1;

static void darshan_core_initialize(int *argc, char ***argv);
static void darshan_core_shutdown(void);
static void darshan_core_cleanup(struct darshan_core_job_runtime* job);
37
static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* start_tm);
38

39
40
41
#define DARSHAN_LOCK() pthread_mutex_lock(&darshan_mutex)
#define DARSHAN_UNLOCK() pthread_mutex_unlock(&darshan_mutex)

42
/* intercept MPI initialize and finalize to manage darshan core runtime */
43
44
45
46
47
48
49
50
51
52
int MPI_Init(int *argc, char ***argv)
{
    int ret;

    ret = DARSHAN_MPI_CALL(PMPI_Init)(argc, argv);
    if(ret != MPI_SUCCESS)
    {
        return(ret);
    }

53
    darshan_core_initialize(argc, argv);
54
55
56
57

    return(ret);
}

58
int MPI_Init_thread(int *argc, char ***argv, int required, int *provided)
59
60
61
62
{
    int ret;

    ret = DARSHAN_MPI_CALL(PMPI_Init_thread)(argc, argv, required, provided);
63
    if(ret != MPI_SUCCESS)
64
65
66
67
    {
        return(ret);
    }

68
    darshan_core_initialize(argc, argv);
69
70
71
72
73
74
75
76

    return(ret);
}

int MPI_Finalize(void)
{
    int ret;

77
    darshan_core_shutdown();
78
79
80
81
82

    ret = DARSHAN_MPI_CALL(PMPI_Finalize)();
    return(ret);
}

83
84
/* *********************************** */

85
static void darshan_core_initialize(int *argc, char ***argv)
86
87
88
89
90
91
92
93
94
95
{
    int i;
    int nprocs;
    int internal_timing_flag = 0;
    double init_start, init_time, init_max;
    char* truncate_string = "<TRUNCATED>";
    int truncate_offset;
    int chars_left = 0;

    DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
96
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
97
98
99
100

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

101
    if(internal_timing_flag)
102
103
104
        init_start = DARSHAN_MPI_CALL(PMPI_Wtime)();

    /* setup darshan runtime if darshan is enabled and hasn't been initialized already */
105
    if(!getenv("DARSHAN_DISABLE") && !darshan_core_job)
106
    {
107
108
109
        /* allocate structure to track darshan_core_job information */
        darshan_core_job = malloc(sizeof(*darshan_core_job));
        if(darshan_core_job)
110
        {
111
            memset(darshan_core_job, 0, sizeof(*darshan_core_job));
112

113
            if(getenv("DARSHAN_DISABLE_TIMING"))
114
            {
115
                darshan_core_job->flags |= CP_FLAG_NOTIMING;
116
117
            }

118
119
120
121
122
123
            strcpy(darshan_core_job->log_job.version_string, CP_VERSION);
            darshan_core_job->log_job.magic_nr = CP_MAGIC_NR;
            darshan_core_job->log_job.uid = getuid();
            darshan_core_job->log_job.start_time = time(NULL);
            darshan_core_job->log_job.nprocs = nprocs;
            darshan_core_job->wtime_offset = DARSHAN_MPI_CALL(PMPI_Wtime)();
124
125
126
127

            /* record exe and arguments */
            for(i=0; i<(*argc); i++)
            {
128
129
                chars_left = CP_EXE_LEN-strlen(darshan_core_job->exe);
                strncat(darshan_core_job->exe, *(argv[i]), chars_left);
130
131
                if(i < ((*argc)-1))
                {
132
133
                    chars_left = CP_EXE_LEN-strlen(darshan_core_job->exe);
                    strncat(darshan_core_job->exe, " ", chars_left);
134
135
136
137
138
139
140
141
                }
            }

            /* if we don't see any arguments, then use glibc symbol to get
             * program name at least (this happens in fortran)
             */
            if(argc == 0)
            {
142
                chars_left = CP_EXE_LEN-strlen(darshan_core_job->exe);
143
                strncat(darshan_core_job->exe, __progname, chars_left);
144
145
                chars_left = CP_EXE_LEN-strlen(darshan_core_job->exe);
                strncat(darshan_core_job->exe, " <unknown args>", chars_left);
146
147
148
149
150
151
            }

            if(chars_left == 0)
            {
                /* we ran out of room; mark that string was truncated */
                truncate_offset = CP_EXE_LEN - strlen(truncate_string);
152
                sprintf(&darshan_core_job->exe[truncate_offset], "%s",
153
154
155
156
157
158
159
160
161
162
                    truncate_string);
            }
        }
    }

    if(internal_timing_flag)
    {
        init_time = DARSHAN_MPI_CALL(PMPI_Wtime)() - init_start;
        DARSHAN_MPI_CALL(PMPI_Reduce)(&init_time, &init_max, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
163
        if(my_rank == 0)
164
165
166
167
168
169
170
171
172
        {
            printf("#darshan:<op>\t<nprocs>\t<time>\n");
            printf("darshan:init\t%d\t%f\n", nprocs, init_max);
        }
    }

    return;
}

173
static void darshan_core_shutdown()
174
{
175
176
    char *logfile_name;
    struct darshan_core_job_runtime* final_job;
177
    struct darshan_core_module *mod, *tmp;
178
    int internal_timing_flag = 0;
179
    char* envjobid;
180
181
182
183
    char* jobid_str;
    int jobid;
    struct tm* start_tm;
    time_t start_time_tmp;
184
    int ret;
185
186
    int64_t first_start_time;
    int64_t last_end_time;
187
188
189
    int local_mod_use[DARSHAN_MAX_MODS] = {0};
    int global_mod_use_count[DARSHAN_MAX_MODS] = {0};
    int i;
190
191
192
193

    if(getenv("DARSHAN_INTERNAL_TIMING"))
        internal_timing_flag = 1;

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    DARSHAN_LOCK();
    if(!darshan_core_job)
    {
        DARSHAN_UNLOCK();
        return;
    }
    /* disable further tracing while hanging onto the data so that we can
     * write it out
     */
    final_job = darshan_core_job;
    darshan_core_job = NULL;
    DARSHAN_UNLOCK();

    logfile_name = malloc(PATH_MAX);
    if(!logfile_name)
    {
        darshan_core_cleanup(final_job);
        return;
    }

214
    /* set jobid and logfile name on rank 0 */
215
    if(my_rank == 0)
216
217
218
    {
        /* Use CP_JOBID_OVERRIDE for the env var or CP_JOBID */
        envjobid = getenv(CP_JOBID_OVERRIDE);
219
        if(!envjobid)
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
        {
            envjobid = CP_JOBID;
        }

        jobid_str = getenv(envjobid);
        if(jobid_str)
        {
            /* in cobalt we can find it in env var */
            ret = sscanf(jobid_str, "%d", &jobid);
        }
        if(!jobid_str || ret != 1)
        {
            /* use pid as fall back */
            jobid = getpid();
        }

236
237
        /* add to darshan core job */
        final_job->log_job.jobid = (int64_t)jobid;
238

239
240
241
        /* use human readable start time format in log filename */
        start_time_tmp = final_job->log_job.start_time;
        start_tm = localtime(&start_time_tmp);
242

243
244
        /* construct log file name */
        darshan_get_logfile_name(logfile_name, jobid, start_tm);
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
    }

    /* broadcast log file name */
    DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
        MPI_COMM_WORLD);

    if(strlen(logfile_name) == 0)
    {
        /* failed to generate log file name */
        darshan_core_cleanup(final_job);
        return;
    }

    final_job->log_job.end_time = time(NULL);

260
261
262
263
264
265
266
267
268
269
    /* reduce to report first start time and last end time across all ranks
     * at rank 0
     */
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_job->log_job.start_time, &first_start_time, 1, MPI_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
    DARSHAN_MPI_CALL(PMPI_Reduce)(&final_job->log_job.end_time, &last_end_time, 1, MPI_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
    if(my_rank == 0)
    {
        final_job->log_job.start_time = first_start_time;
        final_job->log_job.end_time = last_end_time;
    }
270

271
272
273
274
275
276
277
278
279
280
281
    /* set which local modules were actually used */
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
    {
        if(final_job->mod_array[i])
            local_mod_use[i] = 1;
    }

    /* reduce the number of times a module was opened globally and bcast to everyone */   
    DARSHAN_MPI_CALL(PMPI_Allreduce)(local_mod_use, global_mod_use_count, DARSHAN_MAX_MODS, MPI_INT, MPI_SUM, MPI_COMM_WORLD);

    for(i = 0; i < DARSHAN_MAX_MODS; i++)
282
    {
283

284
    }
285
286
287
288

    free(logfile_name);
    darshan_core_cleanup(final_job);

289
    if(internal_timing_flag)
290
    {
291
292
293
294
295
        /* TODO: what do we want to time in new darshan version? */
    }
    
    return;
}
296

297
298
static void darshan_core_cleanup(struct darshan_core_job_runtime* job)
{
299
    int i;
300

301
    for(i = 0; i < DARSHAN_MAX_MODS; i++)
302
    {
303

304
    }
305

306
    free(job);
307
308
309
310

    return;
}

311
static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* start_tm)
312
{
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
    char* logpath;
    char* logname_string;
    char* logpath_override = NULL;
#ifdef __CP_LOG_ENV
    char env_check[256];
    char* env_tok;
#endif
    uint64_t hlevel;
    char hname[HOST_NAME_MAX];
    uint64_t logmod;
    char cuser[L_cuserid] = {0};
    int ret;

    /* Use CP_LOG_PATH_OVERRIDE for the value or __CP_LOG_PATH */
    logpath = getenv(CP_LOG_PATH_OVERRIDE);
    if(!logpath)
    {
#ifdef __CP_LOG_PATH
        logpath = __CP_LOG_PATH;
#endif
    }

    /* get the username for this job.  In order we will try each of the
     * following until one of them succeeds:
     *
     * - cuserid()
     * - getenv("LOGNAME")
     * - snprintf(..., geteuid());
     *
     * Note that we do not use getpwuid() because it generally will not
     * work in statically compiled binaries.
     */

#ifndef DARSHAN_DISABLE_CUSERID
    cuserid(cuser);
#endif

    /* if cuserid() didn't work, then check the environment */
    if(strcmp(cuser, "") == 0)
    {
        logname_string = getenv("LOGNAME");
        if(logname_string)
        {
            strncpy(cuser, logname_string, (L_cuserid-1));
        }

    }

    /* if cuserid() and environment both fail, then fall back to uid */
    if(strcmp(cuser, "") == 0)
    {
        uid_t uid = geteuid();
        snprintf(cuser, sizeof(cuser), "%u", uid);
    }

    /* generate a random number to help differentiate the log */
    hlevel=DARSHAN_MPI_CALL(PMPI_Wtime)() * 1000000;
    (void)gethostname(hname, sizeof(hname));
    logmod = darshan_hash((void*)hname,strlen(hname),hlevel);

    /* see if darshan was configured using the --with-logpath-by-env
     * argument, which allows the user to specify an absolute path to
     * place logs via an env variable.
     */
#ifdef __CP_LOG_ENV
    /* just silently skip if the environment variable list is too big */
    if(strlen(__CP_LOG_ENV) < 256)
    {
        /* copy env variable list to a temporary buffer */
        strcpy(env_check, __CP_LOG_ENV);
        /* tokenize the comma-separated list */
        env_tok = strtok(env_check, ",");
        if(env_tok)
        {
            do
            {
                /* check each env variable in order */
                logpath_override = getenv(env_tok);
                if(logpath_override)
                {
                    /* stop as soon as we find a match */
                    break;
                }
            }while((env_tok = strtok(NULL, ",")));
        }
    }
#endif

    if(logpath_override)
    {
        ret = snprintf(logfile_name, PATH_MAX,
            "%s/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
            logpath_override,
            cuser, __progname, jobid,
            (start_tm->tm_mon+1),
            start_tm->tm_mday,
            (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
            logmod);
        if(ret == (PATH_MAX-1))
        {
            /* file name was too big; squish it down */
            snprintf(logfile_name, PATH_MAX,
                "%s/id%d.darshan_partial",
                logpath_override, jobid);
        }
    }
    else if(logpath)
    {
        ret = snprintf(logfile_name, PATH_MAX,
            "%s/%d/%d/%d/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
            logpath, (start_tm->tm_year+1900),
            (start_tm->tm_mon+1), start_tm->tm_mday,
            cuser, __progname, jobid,
            (start_tm->tm_mon+1),
            start_tm->tm_mday,
            (start_tm->tm_hour*60*60 + start_tm->tm_min*60 + start_tm->tm_sec),
            logmod);
        if(ret == (PATH_MAX-1))
        {
            /* file name was too big; squish it down */
            snprintf(logfile_name, PATH_MAX,
                "%s/id%d.darshan_partial",
                logpath, jobid);
        }
    }
    else
    {
        logfile_name[0] = '\0';
    }

    return;
444
445
}

446
/* ********************************************************* */
447
448

void darshan_core_register_module(
449
    darshan_module_id id,
450
451
452
453
    char *name,
    struct darshan_module_funcs *funcs,
    int *runtime_mem_limit)
{
454
    struct darshan_core_module* mod;
455

456
457
    DARSHAN_LOCK();

458
    *runtime_mem_limit = 0;
459
    if(!darshan_core_job || (id >= DARSHAN_MAX_MODS))
460
461
    {
        DARSHAN_UNLOCK();
462
        return;
463
    }
464
465

    /* see if this module is already registered */
466
    if(darshan_core_job->mod_array[id])
467
    {
468
        /* if module is already registered just return */
469
470
471
        /* NOTE: we do not recalculate memory limit here, just set to 0 */
        DARSHAN_UNLOCK();
        return;
472
473
    }

474
    /* this module has not been registered yet, allocate and initialize it */
475
476
    mod = malloc(sizeof(*mod));
    if(!mod)
477
478
479
480
    {
        DARSHAN_UNLOCK();
        return;
    }
481
    memset(mod, 0, sizeof(*mod));
482

483
    mod->id = id;
484
485
    strncpy(mod->name, name, DARSHAN_MOD_NAME_LEN);
    mod->mod_funcs = *funcs;
486
487
488

    /* register module with darshan */
    darshan_core_job->mod_array[id] = mod;
489

490
491
    /* TODO: something smarter than just 2 MiB per module */
    *runtime_mem_limit = 2 * 1024 * 1024;
492

493
494
    DARSHAN_UNLOCK();

495
496
497
498
499
500
501
502
503
    return;
}

void darshan_core_lookup_id(
    void *name,
    int len,
    int printable_flag,
    darshan_file_id *id)
{
504
    darshan_file_id tmp_id;
505

506
    if(!darshan_core_job)
507
508
509
510
511
512
513
514
515
516
        return;

    /* TODO: what do you do with printable flag? */

    /* hash the input name to get a unique id for this record */
    tmp_id = darshan_hash(name, len, 0);
    
    /* TODO: how to store the filename to hash mapping? */

    *id = tmp_id;
517
518
519
    return;
}

520
521
double darshan_core_wtime()
{
522
    /* TODO since NOTIMING is the only flag (currently), maybe we just drop 'flags' */
523
    if(!darshan_core_job || darshan_core_job->flags & CP_FLAG_NOTIMING)
524
525
526
527
528
529
    {
        return(0);
    }

    return DARSHAN_MPI_CALL(PMPI_Wtime)();
}
530
531
532
533
534
535
536
537
538

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */