darshan-pnetcdf.c 14 KB
Newer Older
Shane Snyder's avatar
Shane Snyder committed
1 2 3 4 5 6
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

7 8 9
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

Shane Snyder's avatar
Shane Snyder committed
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <search.h>
#include <assert.h>
#include <pthread.h>

#include "darshan.h"
#include "darshan-dynamic.h"

DARSHAN_FORWARD_DECL(ncmpi_create, int, (MPI_Comm comm, const char *path, int cmode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_open, int, (MPI_Comm comm, const char *path, int omode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_close, int, (int ncid));

31 32
/* structure that can track i/o stats for a given PNETCDF file record at runtime */
struct pnetcdf_file_record_ref
Shane Snyder's avatar
Shane Snyder committed
33
{
34
    struct darshan_pnetcdf_file* file_rec;
Shane Snyder's avatar
Shane Snyder committed
35 36
};

37
/* struct to encapsulate runtime state for the PNETCDF module */
Shane Snyder's avatar
Shane Snyder committed
38 39
struct pnetcdf_runtime
{
40 41 42
    void *rec_id_hash;
    void *ncid_hash;
    int file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
43 44
};

45 46 47 48 49 50
static void pnetcdf_runtime_initialize(
    void);
static struct pnetcdf_file_record_ref *pnetcdf_track_new_file_record(
    darshan_record_id rec_id, const char *path);
static void pnetcdf_cleanup_runtime(
    void);
51 52 53 54 55 56 57
#ifdef HAVE_MPI
static void pnetcdf_record_reduction_op(
    void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void pnetcdf_mpi_redux(
    void *pnetcdf_buf, MPI_Comm mod_comm,
    darshan_record_id *shared_recs, int shared_rec_count);
#endif
58
static void pnetcdf_shutdown(
59
    void **pnetcdf_buf, int *pnetcdf_buf_sz);
60

Shane Snyder's avatar
Shane Snyder committed
61 62 63 64 65 66 67
static struct pnetcdf_runtime *pnetcdf_runtime = NULL;
static pthread_mutex_t pnetcdf_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;

#define PNETCDF_LOCK() pthread_mutex_lock(&pnetcdf_runtime_mutex)
#define PNETCDF_UNLOCK() pthread_mutex_unlock(&pnetcdf_runtime_mutex)

68 69
#define PNETCDF_PRE_RECORD() do { \
    PNETCDF_LOCK(); \
70
    if(!darshan_core_disabled_instrumentation()) { \
71 72
        if(!pnetcdf_runtime) pnetcdf_runtime_initialize(); \
        if(pnetcdf_runtime) break; \
73
    } \
74 75
    PNETCDF_UNLOCK(); \
    return(ret); \
76 77 78 79 80 81
} while(0)

#define PNETCDF_POST_RECORD() do { \
    PNETCDF_UNLOCK(); \
} while(0)

82
#define PNETCDF_RECORD_OPEN(__ncidp, __path, __comm, __tm1, __tm2) do { \
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
    darshan_record_id rec_id; \
    struct pnetcdf_file_record_ref *rec_ref; \
    char *newpath; \
    int comm_size; \
    newpath = darshan_clean_file_path(__path); \
    if(!newpath) newpath = (char *)__path; \
    if(darshan_core_excluded_path(newpath)) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
    rec_id = darshan_core_gen_record_id(newpath); \
    rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
    if(!rec_ref) rec_ref = pnetcdf_track_new_file_record(rec_id, newpath); \
    if(!rec_ref) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
100
    PMPI_Comm_size(__comm, &comm_size); \
101 102 103 104
    if(rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] == 0 || \
     rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] > __tm1) \
        rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_START_TIMESTAMP] = __tm1; \
    rec_ref->file_rec->fcounters[PNETCDF_F_OPEN_END_TIMESTAMP] = __tm2; \
105 106 107 108 109 110
    if(comm_size == 1) rec_ref->file_rec->counters[PNETCDF_INDEP_OPENS] += 1; \
    else rec_ref->file_rec->counters[PNETCDF_COLL_OPENS] += 1; \
    darshan_add_record_ref(&(pnetcdf_runtime->ncid_hash), __ncidp, sizeof(int), rec_ref); \
    if(newpath != __path) free(newpath); \
} while(0)

Shane Snyder's avatar
Shane Snyder committed
111 112 113 114 115 116 117 118 119
/*********************************************************
 *      Wrappers for PNETCDF functions of interest       * 
 *********************************************************/

int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
    int cmode, MPI_Info info, int *ncidp)
{
    int ret;
    char* tmp;
120
    double tm1, tm2;
Shane Snyder's avatar
Shane Snyder committed
121 122 123 124 125

    MAP_OR_FAIL(ncmpi_create);

    tm1 = darshan_core_wtime();
    ret = __real_ncmpi_create(comm, path, cmode, info, ncidp);
126
    tm2 = darshan_core_wtime();
Shane Snyder's avatar
Shane Snyder committed
127 128 129 130 131 132 133 134 135 136 137 138
    if(ret == 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(path, ':');
        if (tmp > path + 1) {
            path = tmp + 1;
        }

139
        PNETCDF_PRE_RECORD();
140
        PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
141
        PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
142 143 144 145 146 147 148 149 150 151
    }

    return(ret);
}

int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
    int omode, MPI_Info info, int *ncidp)
{
    int ret;
    char* tmp;
152
    double tm1, tm2;
Shane Snyder's avatar
Shane Snyder committed
153 154 155 156 157

    MAP_OR_FAIL(ncmpi_open);

    tm1 = darshan_core_wtime();
    ret = __real_ncmpi_open(comm, path, omode, info, ncidp);
158
    tm2 = darshan_core_wtime();
Shane Snyder's avatar
Shane Snyder committed
159 160 161 162 163 164 165 166 167 168 169 170
    if(ret == 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(path, ':');
        if (tmp > path + 1) {
            path = tmp + 1;
        }

171
        PNETCDF_PRE_RECORD();
172
        PNETCDF_RECORD_OPEN(ncidp, path, comm, tm1, tm2);
173
        PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
174 175 176 177 178 179 180
    }

    return(ret);
}

int DARSHAN_DECL(ncmpi_close)(int ncid)
{
181
    struct pnetcdf_file_record_ref *rec_ref;
Shane Snyder's avatar
Shane Snyder committed
182
    int ret;
183
    double tm1, tm2;
Shane Snyder's avatar
Shane Snyder committed
184 185 186

    MAP_OR_FAIL(ncmpi_close);

187
    tm1 = darshan_core_wtime();
Shane Snyder's avatar
Shane Snyder committed
188
    ret = __real_ncmpi_close(ncid);
189
    tm2 = darshan_core_wtime();
Shane Snyder's avatar
Shane Snyder committed
190

191 192 193 194
    PNETCDF_PRE_RECORD();
    rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->ncid_hash,
        &ncid, sizeof(int));
    if(rec_ref)
Shane Snyder's avatar
Shane Snyder committed
195
    {
196 197 198 199
        if(rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] == 0 ||
         rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] > tm1)
           rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_START_TIMESTAMP] = tm1;
        rec_ref->file_rec->fcounters[PNETCDF_F_CLOSE_END_TIMESTAMP] = tm2;
200 201
        darshan_delete_record_ref(&(pnetcdf_runtime->ncid_hash),
            &ncid, sizeof(int));
Shane Snyder's avatar
Shane Snyder committed
202
    }
203
    PNETCDF_POST_RECORD();
Shane Snyder's avatar
Shane Snyder committed
204 205 206 207 208 209 210 211 212 213 214

    return(ret);
}

/************************************************************
 * Internal functions for manipulating PNETCDF module state *
 ************************************************************/

/* initialize internal PNETCDF module data strucutres and register with darshan-core */
static void pnetcdf_runtime_initialize()
{
215
    int pnetcdf_buf_size;
216 217 218 219 220 221
    darshan_module_funcs mod_funcs = {
#ifdef HAVE_MPI
    .mod_redux_func = &pnetcdf_mpi_redux,
#endif
    .mod_shutdown_func = &pnetcdf_shutdown
    };
Shane Snyder's avatar
Shane Snyder committed
222

223 224 225
    /* try and store the default number of records for this module */
    pnetcdf_buf_size = DARSHAN_DEF_MOD_REC_COUNT * sizeof(struct darshan_pnetcdf_file);

Shane Snyder's avatar
Shane Snyder committed
226 227 228
    /* register pnetcdf module with darshan-core */
    darshan_core_register_module(
        DARSHAN_PNETCDF_MOD,
229
        mod_funcs,
230
        &pnetcdf_buf_size,
Shane Snyder's avatar
Shane Snyder committed
231 232 233
        &my_rank,
        NULL);

234 235
    /* return if darshan-core does not provide enough module memory */
    if(pnetcdf_buf_size < sizeof(struct darshan_pnetcdf_file))
236 237
    {
        darshan_core_unregister_module(DARSHAN_PNETCDF_MOD);
Shane Snyder's avatar
Shane Snyder committed
238
        return;
239
    }
Shane Snyder's avatar
Shane Snyder committed
240 241 242

    pnetcdf_runtime = malloc(sizeof(*pnetcdf_runtime));
    if(!pnetcdf_runtime)
243 244
    {
        darshan_core_unregister_module(DARSHAN_PNETCDF_MOD);
Shane Snyder's avatar
Shane Snyder committed
245
        return;
246
    }
Shane Snyder's avatar
Shane Snyder committed
247 248 249 250 251
    memset(pnetcdf_runtime, 0, sizeof(*pnetcdf_runtime));

    return;
}

252 253
static struct pnetcdf_file_record_ref *pnetcdf_track_new_file_record(
    darshan_record_id rec_id, const char *path)
Shane Snyder's avatar
Shane Snyder committed
254
{
255 256
    struct darshan_pnetcdf_file *file_rec = NULL;
    struct pnetcdf_file_record_ref *rec_ref = NULL;
257
    int ret;
Shane Snyder's avatar
Shane Snyder committed
258

259 260
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
Shane Snyder's avatar
Shane Snyder committed
261
        return(NULL);
262
    memset(rec_ref, 0, sizeof(*rec_ref));
Shane Snyder's avatar
Shane Snyder committed
263

264 265 266 267
    /* add a reference to this file record based on record id */
    ret = darshan_add_record_ref(&(pnetcdf_runtime->rec_id_hash), &rec_id,
        sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
Shane Snyder's avatar
Shane Snyder committed
268
    {
269
        free(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
270 271 272
        return(NULL);
    }

273 274
    /* register the actual file record with darshan-core so it is persisted
     * in the log file
Shane Snyder's avatar
Shane Snyder committed
275
     */
276 277 278 279 280 281
    file_rec = darshan_core_register_record(
        rec_id,
        path,
        DARSHAN_PNETCDF_MOD,
        sizeof(struct darshan_pnetcdf_file),
        NULL);
Shane Snyder's avatar
Shane Snyder committed
282

283
    if(!file_rec)
Shane Snyder's avatar
Shane Snyder committed
284
    {
285 286 287 288
        darshan_delete_record_ref(&(pnetcdf_runtime->rec_id_hash),
            &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return(NULL);
Shane Snyder's avatar
Shane Snyder committed
289 290
    }

291 292 293 294 295 296 297
    /* registering this file record was successful, so initialize some fields */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = my_rank;
    rec_ref->file_rec = file_rec;
    pnetcdf_runtime->file_rec_count++;

    return(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
298 299
}

300 301 302 303 304 305 306 307 308 309 310 311
static void pnetcdf_cleanup_runtime()
{
    darshan_clear_record_refs(&(pnetcdf_runtime->ncid_hash), 0);
    darshan_clear_record_refs(&(pnetcdf_runtime->rec_id_hash), 1);

    free(pnetcdf_runtime);
    pnetcdf_runtime = NULL;

    return;
}

#ifdef HAVE_MPI
Shane Snyder's avatar
Shane Snyder committed
312 313 314 315 316 317 318 319 320 321 322 323 324
static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
    int *len, MPI_Datatype *datatype)
{
    struct darshan_pnetcdf_file tmp_file;
    struct darshan_pnetcdf_file *infile = infile_v;
    struct darshan_pnetcdf_file *inoutfile = inoutfile_v;
    int i, j;

    assert(pnetcdf_runtime);

    for(i=0; i<*len; i++)
    {
        memset(&tmp_file, 0, sizeof(struct darshan_pnetcdf_file));
325 326
        tmp_file.base_rec.id = infile->base_rec.id;
        tmp_file.base_rec.rank = -1;
Shane Snyder's avatar
Shane Snyder committed
327 328 329 330 331 332 333 334

        /* sum */
        for(j=PNETCDF_INDEP_OPENS; j<=PNETCDF_COLL_OPENS; j++)
        {
            tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
        }

        /* min non-zero (if available) value */
335
        for(j=PNETCDF_F_OPEN_START_TIMESTAMP; j<=PNETCDF_F_CLOSE_START_TIMESTAMP; j++)
Shane Snyder's avatar
Shane Snyder committed
336
        {
337 338
            if((infile->fcounters[j] < inoutfile->fcounters[j] &&
               infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0) 
Shane Snyder's avatar
Shane Snyder committed
339
                tmp_file.fcounters[j] = infile->fcounters[j];
340 341
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
Shane Snyder's avatar
Shane Snyder committed
342 343 344
        }

        /* max */
345
        for(j=PNETCDF_F_OPEN_END_TIMESTAMP; j<=PNETCDF_F_CLOSE_END_TIMESTAMP; j++)
Shane Snyder's avatar
Shane Snyder committed
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
        {
            if(infile->fcounters[j] > inoutfile->fcounters[j])
                tmp_file.fcounters[j] = infile->fcounters[j];
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
        }

        /* update pointers */
        *inoutfile = tmp_file;
        inoutfile++;
        infile++;
    }

    return;
}
361
#endif
Shane Snyder's avatar
Shane Snyder committed
362

363 364 365 366
/***************************************************************************
 * Functions exported by PNETCDF module for coordinating with darshan-core *
 ***************************************************************************/

367 368 369
#ifdef HAVE_MPI
static void pnetcdf_mpi_redux(
    void *pnetcdf_buf,
Shane Snyder's avatar
Shane Snyder committed
370 371
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
372
    int shared_rec_count)
Shane Snyder's avatar
Shane Snyder committed
373
{
374
    int pnetcdf_rec_count;
375 376
    struct pnetcdf_file_record_ref *rec_ref;
    struct darshan_pnetcdf_file *pnetcdf_rec_buf = (struct darshan_pnetcdf_file *)pnetcdf_buf;
Shane Snyder's avatar
Shane Snyder committed
377 378 379 380
    struct darshan_pnetcdf_file *red_send_buf = NULL;
    struct darshan_pnetcdf_file *red_recv_buf = NULL;
    MPI_Datatype red_type;
    MPI_Op red_op;
381
    int i;
Shane Snyder's avatar
Shane Snyder committed
382

383
    PNETCDF_LOCK();
Shane Snyder's avatar
Shane Snyder committed
384
    assert(pnetcdf_runtime);
385

386
    pnetcdf_rec_count = pnetcdf_runtime->file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
387

388 389
    /* necessary initialization of shared records */
    for(i = 0; i < shared_rec_count; i++)
Shane Snyder's avatar
Shane Snyder committed
390
    {
391 392 393
        rec_ref = darshan_lookup_record_ref(pnetcdf_runtime->rec_id_hash,
            &shared_recs[i], sizeof(darshan_record_id));
        assert(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
394

395 396
        rec_ref->file_rec->base_rec.rank = -1;
    }
397

398 399 400 401 402
    /* sort the array of records so we get all of the shared records
     * (marked by rank -1) in a contiguous portion at end of the array
     */
    darshan_record_sort(pnetcdf_rec_buf, pnetcdf_rec_count,
        sizeof(struct darshan_pnetcdf_file));
Shane Snyder's avatar
Shane Snyder committed
403

404 405
    /* make *send_buf point to the shared files at the end of sorted array */
    red_send_buf = &(pnetcdf_rec_buf[pnetcdf_rec_count-shared_rec_count]);
Shane Snyder's avatar
Shane Snyder committed
406

407 408 409 410 411
    /* allocate memory for the reduction output on rank 0 */
    if(my_rank == 0)
    {
        red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_pnetcdf_file));
        if(!red_recv_buf)
Shane Snyder's avatar
Shane Snyder committed
412
        {
413 414
            PNETCDF_UNLOCK();
            return;
Shane Snyder's avatar
Shane Snyder committed
415
        }
416
    }
Shane Snyder's avatar
Shane Snyder committed
417

418 419 420 421 422 423
    /* construct a datatype for a PNETCDF file record.  This is serving no purpose
     * except to make sure we can do a reduction on proper boundaries
     */
    PMPI_Type_contiguous(sizeof(struct darshan_pnetcdf_file),
        MPI_BYTE, &red_type);
    PMPI_Type_commit(&red_type);
Shane Snyder's avatar
Shane Snyder committed
424

425 426
    /* register a PNETCDF file record reduction operator */
    PMPI_Op_create(pnetcdf_record_reduction_op, 1, &red_op);
Shane Snyder's avatar
Shane Snyder committed
427

428 429 430
    /* reduce shared PNETCDF file records */
    PMPI_Reduce(red_send_buf, red_recv_buf,
        shared_rec_count, red_type, red_op, 0, mod_comm);
Shane Snyder's avatar
Shane Snyder committed
431

432 433 434 435 436 437 438 439 440 441
    /* clean up reduction state */
    if(my_rank == 0)
    {
        int tmp_ndx = pnetcdf_rec_count - shared_rec_count;
        memcpy(&(pnetcdf_rec_buf[tmp_ndx]), red_recv_buf,
            shared_rec_count * sizeof(struct darshan_pnetcdf_file));
        free(red_recv_buf);
    }
    else
    {
442
        pnetcdf_runtime->file_rec_count -= shared_rec_count;
Shane Snyder's avatar
Shane Snyder committed
443 444
    }

445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
    PMPI_Type_free(&red_type);
    PMPI_Op_free(&red_op);

    PNETCDF_UNLOCK();
    return;
}
#endif

static void pnetcdf_shutdown(
    void **pnetcdf_buf,
    int *pnetcdf_buf_sz)
{
    int pnetcdf_rec_count;

    PNETCDF_LOCK();
    assert(pnetcdf_runtime);

    pnetcdf_rec_count = pnetcdf_runtime->file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
463

464 465
    /* shutdown internal structures used for instrumenting */
    pnetcdf_cleanup_runtime();
Shane Snyder's avatar
Shane Snyder committed
466

467 468 469
    /* update output buffer size to account for shared file reduction */
    *pnetcdf_buf_sz = pnetcdf_rec_count * sizeof(struct darshan_pnetcdf_file);

470
    PNETCDF_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
471 472 473 474 475 476 477 478 479 480 481
    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */