darshan-pnetcdf.c 13.6 KB
Newer Older
Shane Snyder's avatar
Shane Snyder committed
1 2 3 4 5 6
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

7 8 9
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

Shane Snyder's avatar
Shane Snyder committed
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <search.h>
#include <assert.h>
#include <pthread.h>

#include "darshan.h"
#include "darshan-dynamic.h"

DARSHAN_FORWARD_DECL(ncmpi_create, int, (MPI_Comm comm, const char *path, int cmode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_open, int, (MPI_Comm comm, const char *path, int omode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_close, int, (int ncid));

31 32
/* structure that can track i/o stats for a given PNETCDF file record at runtime */
struct pnetcdf_file_record_ref
Shane Snyder's avatar
Shane Snyder committed
33
{
34
    struct darshan_pnetcdf_file* file_rec;
Shane Snyder's avatar
Shane Snyder committed
35 36
};

37
/* struct to encapsulate runtime state for the PNETCDF module */
Shane Snyder's avatar
Shane Snyder committed
38 39
struct pnetcdf_runtime
{
40 41 42
    void *rec_id_hash;
    void *ncid_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
43 44
};

45 46 47 48 49 50 51 52 53 54 55 56 57
static void pnetcdf_runtime_initialize(
    void);
static struct pnetcdf_file_record_ref *pnetcdf_track_new_file_record(
    darshan_record_id rec_id, const char *path);
static void pnetcdf_record_reduction_op(
    void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void pnetcdf_cleanup_runtime(
    void);

static void pnetcdf_shutdown(
    MPI_Comm mod_comm, darshan_record_id *shared_recs,
    int shared_rec_count, void **pnetcdf_buf, int *pnetcdf_buf_sz);

Shane Snyder's avatar
Shane Snyder committed
58 59 60 61 62 63 64
static struct pnetcdf_runtime *pnetcdf_runtime = NULL;
static pthread_mutex_t pnetcdf_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;

#define PNETCDF_LOCK() pthread_mutex_lock(&pnetcdf_runtime_mutex)
#define PNETCDF_UNLOCK() pthread_mutex_unlock(&pnetcdf_runtime_mutex)

65 66
#define PNETCDF_PRE_RECORD() do { \
    PNETCDF_LOCK(); \
67
    if(!darshan_core_disabled_instrumentation()) { \
68 69
        if(!pnetcdf_runtime) pnetcdf_runtime_initialize(); \
        if(pnetcdf_runtime) break; \
70
    } \
71 72
    PNETCDF_UNLOCK(); \
    return(ret); \
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
} while(0)

#define PNETCDF_POST_RECORD() do { \
    PNETCDF_UNLOCK(); \
} while(0)

#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1) do { \
    darshan_record_id rec_id; \
    struct pnetcdf_file_record_ref *rec_ref; \
    char *newpath; \
    int comm_size; \
    newpath = darshan_clean_file_path(__path); \
    if(!newpath) newpath = (char *)__path; \
    if(darshan_core_excluded_path(newpath)) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
    rec_id = darshan_core_gen_record_id(newpath); \
    rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
    if(!rec_ref) rec_ref = pnetcdf_track_new_file_record(rec_id, newpath); \
    if(!rec_ref) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
    DARSHAN_MPI_CALL(PMPI_Comm_size)(__comm, &comm_size); \
    if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] == 0) \
        rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_TIMESTAMP] = __tm1; \
    if(comm_size == 1) rec_ref->file_rec->counters[PNETCDF_INDEP_OPENS] += 1; \
    else rec_ref->file_rec->counters[PNETCDF_COLL_OPENS] += 1; \
    darshan_add_record_ref(&(pnetcdf_runtime->ncid_hash), __ncidp, sizeof(int), rec_ref); \
    if(newpath != __path) free(newpath); \
} while(0)

Shane Snyder's avatar
Shane Snyder committed
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
/*********************************************************
 *      Wrappers for PNETCDF functions of interest       * 
 *********************************************************/

int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
    int cmode, MPI_Info info, int *ncidp)
{
    int ret;
    char* tmp;
    double tm1;

    MAP_OR_FAIL(ncmpi_create);

    tm1 = darshan_core_wtime();
    ret = __real_ncmpi_create(comm, path, cmode, info, ncidp);
    if(ret == 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(path, ':');
        if (tmp > path + 1) {
            path = tmp + 1;
        }

133 134 135
        PNETCDF_PRE_RECORD();
        PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
        PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
    }

    return(ret);
}

int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
    int omode, MPI_Info info, int *ncidp)
{
    int ret;
    char* tmp;
    double tm1;

    MAP_OR_FAIL(ncmpi_open);

    tm1 = darshan_core_wtime();
    ret = __real_ncmpi_open(comm, path, omode, info, ncidp);
    if(ret == 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(path, ':');
        if (tmp > path + 1) {
            path = tmp + 1;
        }

164 165 166
        PNETCDF_PRE_RECORD();
        PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1);
        PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
167 168 169 170 171 172 173
    }

    return(ret);
}

int DARSHAN_DECL(ncmpi_close)(int ncid)
{
174
    struct pnetcdf_file_record_ref *rec_ref;
Shane Snyder's avatar
Shane Snyder committed
175 176 177 178 179 180
    int ret;

    MAP_OR_FAIL(ncmpi_close);

    ret = __real_ncmpi_close(ncid);

181 182 183 184
    PNETCDF_PRE_RECORD();
    rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->ncid_hash,
        &ncid, sizeof(int));
    if(rec_ref)
Shane Snyder's avatar
Shane Snyder committed
185
    {
186
        rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_TIMESTAMP] =
Shane Snyder's avatar
Shane Snyder committed
187
            darshan_core_wtime();
188 189
        darshan_delete_record_ref(&(pnetcdf_runtime->ncid_hash),
            &ncid, sizeof(int));
Shane Snyder's avatar
Shane Snyder committed
190
    }
191
    PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
192 193 194 195 196 197 198 199 200 201 202

    return(ret);
}

/************************************************************
 * Internal functions for manipulating PNETCDF module state *
 ************************************************************/

/* initialize internal PNETCDF module data strucutres and register with darshan-core */
static void pnetcdf_runtime_initialize()
{
203
    int pnetcdf_buf_size;
Shane Snyder's avatar
Shane Snyder committed
204

205 206 207
    /* try and store the default number of records for this module */
    pnetcdf_buf_size = DARSHAN_DEF_MOD_REC_COUNT * sizeof(struct darshan_pnetcdf_file);

Shane Snyder's avatar
Shane Snyder committed
208 209 210
    /* register pnetcdf module with darshan-core */
    darshan_core_register_module(
        DARSHAN_PNETCDF_MOD,
211
        &pnetcdf_shutdown,
212
        &pnetcdf_buf_size,
Shane Snyder's avatar
Shane Snyder committed
213 214 215
        &my_rank,
        NULL);

216 217
    /* return if darshan-core does not provide enough module memory */
    if(pnetcdf_buf_size < sizeof(struct darshan_pnetcdf_file))
218 219
    {
        darshan_core_unregister_module(DARSHAN_PNETCDF_MOD);
Shane Snyder's avatar
Shane Snyder committed
220
        return;
221
    }
Shane Snyder's avatar
Shane Snyder committed
222 223 224

    pnetcdf_runtime = malloc(sizeof(*pnetcdf_runtime));
    if(!pnetcdf_runtime)
225 226
    {
        darshan_core_unregister_module(DARSHAN_PNETCDF_MOD);
Shane Snyder's avatar
Shane Snyder committed
227
        return;
228
    }
Shane Snyder's avatar
Shane Snyder committed
229 230 231 232 233
    memset(pnetcdf_runtime, 0, sizeof(*pnetcdf_runtime));

    return;
}

234 235
static struct pnetcdf_file_record_ref *pnetcdf_track_new_file_record(
    darshan_record_id rec_id, const char *path)
Shane Snyder's avatar
Shane Snyder committed
236
{
237 238
    struct darshan_pnetcdf_file *file_rec = NULL;
    struct pnetcdf_file_record_ref *rec_ref = NULL;
239
    int ret;
Shane Snyder's avatar
Shane Snyder committed
240

241 242
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
Shane Snyder's avatar
Shane Snyder committed
243
        return(NULL);
244
    memset(rec_ref, 0, sizeof(*rec_ref));
Shane Snyder's avatar
Shane Snyder committed
245

246 247 248 249
    /* add a reference to this file record based on record id */
    ret = darshan_add_record_ref(&(pnetcdf_runtime->rec_id_hash), &rec_id,
        sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
Shane Snyder's avatar
Shane Snyder committed
250
    {
251
        free(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
252 253 254
        return(NULL);
    }

255 256
    /* register the actual file record with darshan-core so it is persisted
     * in the log file
Shane Snyder's avatar
Shane Snyder committed
257
     */
258 259 260 261 262 263
    file_rec = darshan_core_register_record(
        rec_id,
        path,
        DARSHAN_PNETCDF_MOD,
        sizeof(struct darshan_pnetcdf_file),
        NULL);
Shane Snyder's avatar
Shane Snyder committed
264

265
    if(!file_rec)
Shane Snyder's avatar
Shane Snyder committed
266
    {
267 268 269 270
        darshan_delete_record_ref(&(pnetcdf_runtime->rec_id_hash),
            &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return(NULL);
Shane Snyder's avatar
Shane Snyder committed
271 272
    }

273 274 275 276 277 278 279
    /* registering this file record was successful, so initialize some fields */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = my_rank;
    rec_ref->file_rec = file_rec;
    pnetcdf_runtime->file_rec_count++;

    return(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
}

static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
    int *len, MPI_Datatype *datatype)
{
    struct darshan_pnetcdf_file tmp_file;
    struct darshan_pnetcdf_file *infile = infile_v;
    struct darshan_pnetcdf_file *inoutfile = inoutfile_v;
    int i, j;

    assert(pnetcdf_runtime);

    for(i=0; i<*len; i++)
    {
        memset(&tmp_file, 0, sizeof(struct darshan_pnetcdf_file));
295 296
        tmp_file.base_rec.id = infile->base_rec.id;
        tmp_file.base_rec.rank = -1;
Shane Snyder's avatar
Shane Snyder committed
297 298 299 300 301 302 303 304 305 306

        /* sum */
        for(j=PNETCDF_INDEP_OPENS; j<=PNETCDF_COLL_OPENS; j++)
        {
            tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
        }

        /* min non-zero (if available) value */
        for(j=PNETCDF_F_OPEN_TIMESTAMP; j<=PNETCDF_F_OPEN_TIMESTAMP; j++)
        {
307 308
            if((infile->fcounters[j] < inoutfile->fcounters[j] &&
               infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0) 
Shane Snyder's avatar
Shane Snyder committed
309
                tmp_file.fcounters[j] = infile->fcounters[j];
310 311
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
Shane Snyder's avatar
Shane Snyder committed
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
        }

        /* max */
        for(j=PNETCDF_F_CLOSE_TIMESTAMP; j<=PNETCDF_F_CLOSE_TIMESTAMP; j++)
        {
            if(infile->fcounters[j] > inoutfile->fcounters[j])
                tmp_file.fcounters[j] = infile->fcounters[j];
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
        }

        /* update pointers */
        *inoutfile = tmp_file;
        inoutfile++;
        infile++;
    }

    return;
}

332
static void pnetcdf_cleanup_runtime()
Shane Snyder's avatar
Shane Snyder committed
333
{
334 335
    darshan_clear_record_refs(&(pnetcdf_runtime->ncid_hash), 0);
    darshan_clear_record_refs(&(pnetcdf_runtime->rec_id_hash), 1);
Shane Snyder's avatar
Shane Snyder committed
336

337 338
    free(pnetcdf_runtime);
    pnetcdf_runtime = NULL;
Shane Snyder's avatar
Shane Snyder committed
339 340 341 342

    return;
}

343 344 345 346 347
/***************************************************************************
 * Functions exported by PNETCDF module for coordinating with darshan-core *
 ***************************************************************************/

static void pnetcdf_shutdown(
Shane Snyder's avatar
Shane Snyder committed
348 349 350 351 352 353
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
    void **pnetcdf_buf,
    int *pnetcdf_buf_sz)
{
354 355 356 357
    struct pnetcdf_file_record_ref *rec_ref;
    struct darshan_pnetcdf_file *pnetcdf_rec_buf =
        *(struct darshan_pnetcdf_file **)pnetcdf_buf;
    int pnetcdf_rec_count;
Shane Snyder's avatar
Shane Snyder committed
358 359 360 361
    struct darshan_pnetcdf_file *red_send_buf = NULL;
    struct darshan_pnetcdf_file *red_recv_buf = NULL;
    MPI_Datatype red_type;
    MPI_Op red_op;
362
    int i;
Shane Snyder's avatar
Shane Snyder committed
363

364
    PNETCDF_LOCK();
Shane Snyder's avatar
Shane Snyder committed
365
    assert(pnetcdf_runtime);
366

367
    pnetcdf_rec_count = pnetcdf_runtime->file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
368 369

    /* if there are globally shared files, do a shared file reduction */
370 371 372 373
    /* NOTE: the shared file reduction is also skipped if the 
     * DARSHAN_DISABLE_SHARED_REDUCTION environment variable is set.
     */
    if(shared_rec_count && !getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
Shane Snyder's avatar
Shane Snyder committed
374 375 376 377
    {
        /* necessary initialization of shared records */
        for(i = 0; i < shared_rec_count; i++)
        {
378 379 380
            rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->rec_id_hash,
                &shared_recs[i], sizeof(darshan_record_id));
            assert(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
381

382
            rec_ref->file_rec->base_rec.rank = -1;
Shane Snyder's avatar
Shane Snyder committed
383 384
        }

385 386 387

        /* sort the array of records so we get all of the shared records
         * (marked by rank -1) in a contiguous portion at end of the array
Shane Snyder's avatar
Shane Snyder committed
388
         */
389 390
        darshan_record_sort(pnetcdf_rec_buf, pnetcdf_rec_count,
            sizeof(struct darshan_pnetcdf_file));
Shane Snyder's avatar
Shane Snyder committed
391 392

        /* make *send_buf point to the shared files at the end of sorted array */
393
        red_send_buf = &(pnetcdf_rec_buf[pnetcdf_rec_count-shared_rec_count]);
Shane Snyder's avatar
Shane Snyder committed
394 395 396 397 398 399

        /* allocate memory for the reduction output on rank 0 */
        if(my_rank == 0)
        {
            red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_pnetcdf_file));
            if(!red_recv_buf)
400
            {
401
                PNETCDF_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
402
                return;
403
            }
Shane Snyder's avatar
Shane Snyder committed
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
        }

        /* construct a datatype for a PNETCDF file record.  This is serving no purpose
         * except to make sure we can do a reduction on proper boundaries
         */
        DARSHAN_MPI_CALL(PMPI_Type_contiguous)(sizeof(struct darshan_pnetcdf_file),
            MPI_BYTE, &red_type);
        DARSHAN_MPI_CALL(PMPI_Type_commit)(&red_type);

        /* register a PNETCDF file record reduction operator */
        DARSHAN_MPI_CALL(PMPI_Op_create)(pnetcdf_record_reduction_op, 1, &red_op);

        /* reduce shared PNETCDF file records */
        DARSHAN_MPI_CALL(PMPI_Reduce)(red_send_buf, red_recv_buf,
            shared_rec_count, red_type, red_op, 0, mod_comm);

        /* clean up reduction state */
        if(my_rank == 0)
        {
423 424
            int tmp_ndx = pnetcdf_rec_count - shared_rec_count;
            memcpy(&(pnetcdf_rec_buf[tmp_ndx]), red_recv_buf,
Shane Snyder's avatar
Shane Snyder committed
425 426 427 428 429
                shared_rec_count * sizeof(struct darshan_pnetcdf_file));
            free(red_recv_buf);
        }
        else
        {
430
            pnetcdf_rec_count -= shared_rec_count;
Shane Snyder's avatar
Shane Snyder committed
431 432 433 434 435 436
        }

        DARSHAN_MPI_CALL(PMPI_Type_free)(&red_type);
        DARSHAN_MPI_CALL(PMPI_Op_free)(&red_op);
    }

437 438
    /* update output buffer size to account for shared file reduction */
    *pnetcdf_buf_sz = pnetcdf_rec_count * sizeof(struct darshan_pnetcdf_file);
Shane Snyder's avatar
Shane Snyder committed
439

440 441
    /* shutdown internal structures used for instrumenting */
    pnetcdf_cleanup_runtime();
Shane Snyder's avatar
Shane Snyder committed
442

443
    PNETCDF_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
444 445 446 447 448 449 450 451 452 453 454
    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */