darshan-hdf5.c 14.8 KB
Newer Older
1 2 3 4 5 6
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

7 8 9
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <search.h>
#include <assert.h>
#include <pthread.h>

#include "darshan.h"
#include "darshan-dynamic.h"

/* hope this doesn't change any time soon */
28 29 30 31 32 33 34
typedef int herr_t;     //hf5-1.10.0p1: H5public.h:126

#ifdef __DARSHAN_ENABLE_HDF5110
  typedef int64_t hid_t;  //hf5-1.10.0p1: H5Ipublic.h:56
#else
  typedef int hid_t;
#endif
35 36 37 38

DARSHAN_FORWARD_DECL(H5Fcreate, hid_t, (const char *filename, unsigned flags, hid_t create_plist, hid_t access_plist));
DARSHAN_FORWARD_DECL(H5Fopen, hid_t, (const char *filename, unsigned flags, hid_t access_plist));
DARSHAN_FORWARD_DECL(H5Fclose, herr_t, (hid_t file_id));
39 40 41 42 43

/* prototype for HDF symbols that we will call directly from within other
 * wrappers if HDF is linked in
 */
extern herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, unsigned *relnum);
44

45 46
/* structure that can track i/o stats for a given HDF5 file record at runtime */
struct hdf5_file_record_ref
Shane Snyder's avatar
Shane Snyder committed
47
{
48
    struct darshan_hdf5_file* file_rec;
Shane Snyder's avatar
Shane Snyder committed
49 50
};

51
/* struct to encapsulate runtime state for the HDF5 module */
52 53
struct hdf5_runtime
{
54 55 56
    void *rec_id_hash;
    void *hid_hash;
    int file_rec_count;
57 58
};

59 60 61 62 63 64
static void hdf5_runtime_initialize(
    void);
static struct hdf5_file_record_ref *hdf5_track_new_file_record(
    darshan_record_id rec_id, const char *path);
static void hdf5_cleanup_runtime(
    void);
65 66 67 68 69 70 71
#ifdef HAVE_MPI
static void hdf5_record_reduction_op(
    void* infile_v, void* inoutfile_v, int *len, MPI_Datatype *datatype);
static void hdf5_mpi_redux(
    void *hdf5_buf, MPI_Comm mod_comm,
    darshan_record_id *shared_recs, int shared_rec_count);
#endif
72
static void hdf5_shutdown(
73
    void **hdf5_buf, int *hdf5_buf_sz);
74

75 76 77 78 79 80 81
static struct hdf5_runtime *hdf5_runtime = NULL;
static pthread_mutex_t hdf5_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int my_rank = -1;

#define HDF5_LOCK() pthread_mutex_lock(&hdf5_runtime_mutex)
#define HDF5_UNLOCK() pthread_mutex_unlock(&hdf5_runtime_mutex)

82 83
#define HDF5_PRE_RECORD() do { \
    HDF5_LOCK(); \
84
    if(!darshan_core_disabled_instrumentation()) { \
85 86
        if(!hdf5_runtime) hdf5_runtime_initialize(); \
        if(hdf5_runtime) break; \
87
    } \
88 89
    HDF5_UNLOCK(); \
    return(ret); \
90 91 92 93 94 95
} while(0)

#define HDF5_POST_RECORD() do { \
    HDF5_UNLOCK(); \
} while(0)

96
#define HDF5_RECORD_OPEN(__ret, __path, __tm1, __tm2) do { \
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
    darshan_record_id rec_id; \
    struct hdf5_file_record_ref *rec_ref; \
    char *newpath; \
    newpath = darshan_clean_file_path(__path); \
    if(!newpath) newpath = (char *)__path; \
    if(darshan_core_excluded_path(newpath)) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
    rec_id = darshan_core_gen_record_id(newpath); \
    rec_ref = darshan_lookup_record_ref(hdf5_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
    if(!rec_ref) rec_ref = hdf5_track_new_file_record(rec_id, newpath); \
    if(!rec_ref) { \
        if(newpath != __path) free(newpath); \
        break; \
    } \
113 114 115 116
    if(rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] == 0 || \
     rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] > __tm1) \
        rec_ref->file_rec->fcounters[HDF5_F_OPEN_START_TIMESTAMP] = __tm1; \
    rec_ref->file_rec->fcounters[HDF5_F_OPEN_END_TIMESTAMP] = __tm2; \
117 118 119 120 121
    rec_ref->file_rec->counters[HDF5_OPENS] += 1; \
    darshan_add_record_ref(&(hdf5_runtime->hid_hash), &__ret, sizeof(hid_t), rec_ref); \
    if(newpath != __path) free(newpath); \
} while(0)

122 123 124 125 126 127 128
/*********************************************************
 *        Wrappers for HDF5 functions of interest        * 
 *********************************************************/

hid_t DARSHAN_DECL(H5Fcreate)(const char *filename, unsigned flags,
    hid_t create_plist, hid_t access_plist)
{
129
    hid_t ret;
130
    char* tmp;
131
    double tm1, tm2;
132 133
    unsigned majnum, minnum, relnum;

134
    H5get_libversion(&majnum, &minnum, &relnum);
135
#ifdef __DARSHAN_ENABLE_HDF5110
Philip Carns's avatar
bug fix  
Philip Carns committed
136
    if(majnum < 1 || (majnum == 1 && minnum < 10))
137
#else
Philip Carns's avatar
bug fix  
Philip Carns committed
138
    if(majnum > 1 || (majnum == 1 && minnum >= 10))
139 140
#endif
    {
Philip Carns's avatar
bug fix  
Philip Carns committed
141 142
        if(my_rank < 0)
            MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
143 144
        if(my_rank == 0)
        {
145
            darshan_core_fprintf(stderr, "Darshan HDF5 module error: runtime library version does not match Darshan module.\n");
146 147 148
        }
        return(-1);
    }
149 150 151

    MAP_OR_FAIL(H5Fcreate);

Shane Snyder's avatar
Shane Snyder committed
152
    tm1 = darshan_core_wtime();
153
    ret = __real_H5Fcreate(filename, flags, create_plist, access_plist);
154
    tm2 = darshan_core_wtime();
155 156 157 158 159 160 161 162 163 164 165 166
    if(ret >= 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(filename, ':');
        if (tmp > filename + 1) {
            filename = tmp + 1;
        }

167
        HDF5_PRE_RECORD();
168
        HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
169
        HDF5_POST_RECORD();
170 171 172 173 174 175 176 177
    }

    return(ret);
}

hid_t DARSHAN_DECL(H5Fopen)(const char *filename, unsigned flags,
    hid_t access_plist)
{
178
    hid_t ret;
179
    char* tmp;
180
    double tm1, tm2;
181 182
    unsigned majnum, minnum, relnum;

183
    H5get_libversion(&majnum, &minnum, &relnum);
184
#ifdef __DARSHAN_ENABLE_HDF5110
Philip Carns's avatar
bug fix  
Philip Carns committed
185
    if(majnum < 1 || (majnum == 1 && minnum < 10))
186
#else
Philip Carns's avatar
bug fix  
Philip Carns committed
187
    if(majnum > 1 || (majnum == 1 && minnum >= 10))
188 189
#endif
    {
Philip Carns's avatar
bug fix  
Philip Carns committed
190 191
        if(my_rank < 0)
            MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
192 193
        if(my_rank == 0)
        {
194
            darshan_core_fprintf(stderr, "Darshan HDF5 module error: runtime library version does not match Darshan module.\n");
195 196 197
        }
        return(-1);
    }
198 199 200

    MAP_OR_FAIL(H5Fopen);

Shane Snyder's avatar
Shane Snyder committed
201
    tm1 = darshan_core_wtime();
202
    ret = __real_H5Fopen(filename, flags, access_plist);
203
    tm2 = darshan_core_wtime();
204 205 206 207 208 209 210 211 212 213 214 215
    if(ret >= 0)
    {
        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(filename, ':');
        if (tmp > filename + 1) {
            filename = tmp + 1;
        }

216
        HDF5_PRE_RECORD();
217
        HDF5_RECORD_OPEN(ret, filename, tm1, tm2);
218
        HDF5_POST_RECORD();
219 220 221 222 223 224 225 226
    }

    return(ret);

}

herr_t DARSHAN_DECL(H5Fclose)(hid_t file_id)
{
227
    struct hdf5_file_record_ref *rec_ref;
228
    double tm1, tm2;
229
    herr_t ret;
230 231 232

    MAP_OR_FAIL(H5Fclose);

233
    tm1 = darshan_core_wtime();
234
    ret = __real_H5Fclose(file_id);
235
    tm2 = darshan_core_wtime();
236

237 238 239 240
    HDF5_PRE_RECORD();
    rec_ref = darshan_lookup_record_ref(hdf5_runtime->hid_hash,
        &file_id, sizeof(hid_t));
    if(rec_ref)
241
    {
242 243 244 245
        if(rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] == 0 ||
         rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] > tm1)
           rec_ref->file_rec->fcounters[HDF5_F_CLOSE_START_TIMESTAMP] = tm1;
        rec_ref->file_rec->fcounters[HDF5_F_CLOSE_END_TIMESTAMP] = tm2;
246 247
        darshan_delete_record_ref(&(hdf5_runtime->hid_hash),
            &file_id, sizeof(hid_t));
248
    }
249
    HDF5_POST_RECORD();
250 251 252 253 254 255 256 257 258

    return(ret);

}

/*********************************************************
 * Internal functions for manipulating HDF5 module state *
 *********************************************************/

Shane Snyder's avatar
Shane Snyder committed
259 260 261
/* initialize internal HDF5 module data strucutres and register with darshan-core */
static void hdf5_runtime_initialize()
{
262
    int hdf5_buf_size;
263 264 265 266 267 268
    darshan_module_funcs mod_funcs = {
#ifdef HAVE_MPI
    .mod_redux_func = &hdf5_mpi_redux,
#endif
    .mod_shutdown_func = &hdf5_shutdown
    };
Shane Snyder's avatar
Shane Snyder committed
269

270 271 272
    /* try and store the default number of records for this module */
    hdf5_buf_size = DARSHAN_DEF_MOD_REC_COUNT * sizeof(struct darshan_hdf5_file);

Shane Snyder's avatar
Shane Snyder committed
273 274 275
    /* register hdf5 module with darshan-core */
    darshan_core_register_module(
        DARSHAN_HDF5_MOD,
276
        mod_funcs,
277
        &hdf5_buf_size,
Shane Snyder's avatar
Shane Snyder committed
278 279 280
        &my_rank,
        NULL);

281 282
    /* return if darshan-core does not provide enough module memory */
    if(hdf5_buf_size < sizeof(struct darshan_hdf5_file))
283 284
    {
        darshan_core_unregister_module(DARSHAN_HDF5_MOD);
Shane Snyder's avatar
Shane Snyder committed
285
        return;
286
    }
Shane Snyder's avatar
Shane Snyder committed
287 288 289

    hdf5_runtime = malloc(sizeof(*hdf5_runtime));
    if(!hdf5_runtime)
290 291
    {
        darshan_core_unregister_module(DARSHAN_HDF5_MOD);
Shane Snyder's avatar
Shane Snyder committed
292
        return;
293
    }
Shane Snyder's avatar
Shane Snyder committed
294 295 296 297 298
    memset(hdf5_runtime, 0, sizeof(*hdf5_runtime));

    return;
}

299 300
static struct hdf5_file_record_ref *hdf5_track_new_file_record(
    darshan_record_id rec_id, const char *path)
Shane Snyder's avatar
Shane Snyder committed
301
{
302 303
    struct darshan_hdf5_file *file_rec = NULL;
    struct hdf5_file_record_ref *rec_ref = NULL;
304
    int ret;
Shane Snyder's avatar
Shane Snyder committed
305

306 307
    rec_ref = malloc(sizeof(*rec_ref));
    if(!rec_ref)
Shane Snyder's avatar
Shane Snyder committed
308
        return(NULL);
309
    memset(rec_ref, 0, sizeof(*rec_ref));
Shane Snyder's avatar
Shane Snyder committed
310

311 312 313 314
    /* add a reference to this file record based on record id */
    ret = darshan_add_record_ref(&(hdf5_runtime->rec_id_hash), &rec_id,
        sizeof(darshan_record_id), rec_ref);
    if(ret == 0)
Shane Snyder's avatar
Shane Snyder committed
315
    {
316
        free(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
317 318 319
        return(NULL);
    }

320 321
    /* register the actual file record with darshan-core so it is persisted
     * in the log file
Shane Snyder's avatar
Shane Snyder committed
322
     */
323 324 325 326 327 328
    file_rec = darshan_core_register_record(
        rec_id,
        path,
        DARSHAN_HDF5_MOD,
        sizeof(struct darshan_hdf5_file),
        NULL);
Shane Snyder's avatar
Shane Snyder committed
329

330
    if(!file_rec)
Shane Snyder's avatar
Shane Snyder committed
331
    {
332 333 334 335
        darshan_delete_record_ref(&(hdf5_runtime->rec_id_hash),
            &rec_id, sizeof(darshan_record_id));
        free(rec_ref);
        return(NULL);
Shane Snyder's avatar
Shane Snyder committed
336 337
    }

338 339 340 341 342 343 344
    /* registering this file record was successful, so initialize some fields */
    file_rec->base_rec.id = rec_id;
    file_rec->base_rec.rank = my_rank;
    rec_ref->file_rec = file_rec;
    hdf5_runtime->file_rec_count++;

    return(rec_ref);
Shane Snyder's avatar
Shane Snyder committed
345 346
}

347 348 349 350 351 352 353 354 355 356 357 358
static void hdf5_cleanup_runtime()
{
    darshan_clear_record_refs(&(hdf5_runtime->hid_hash), 0);
    darshan_clear_record_refs(&(hdf5_runtime->rec_id_hash), 1);

    free(hdf5_runtime);
    hdf5_runtime = NULL;

    return;
}

#ifdef HAVE_MPI
Shane Snyder's avatar
Shane Snyder committed
359 360 361 362 363 364 365 366 367 368 369 370 371
static void hdf5_record_reduction_op(void* infile_v, void* inoutfile_v,
    int *len, MPI_Datatype *datatype)
{
    struct darshan_hdf5_file tmp_file;
    struct darshan_hdf5_file *infile = infile_v;
    struct darshan_hdf5_file *inoutfile = inoutfile_v;
    int i, j;

    assert(hdf5_runtime);

    for(i=0; i<*len; i++)
    {
        memset(&tmp_file, 0, sizeof(struct darshan_hdf5_file));
372 373
        tmp_file.base_rec.id = infile->base_rec.id;
        tmp_file.base_rec.rank = -1;
Shane Snyder's avatar
Shane Snyder committed
374 375 376 377 378 379 380 381

        /* sum */
        for(j=HDF5_OPENS; j<=HDF5_OPENS; j++)
        {
            tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
        }

        /* min non-zero (if available) value */
382
        for(j=HDF5_F_OPEN_START_TIMESTAMP; j<=HDF5_F_CLOSE_START_TIMESTAMP; j++)
Shane Snyder's avatar
Shane Snyder committed
383
        {
384 385
            if((infile->fcounters[j] < inoutfile->fcounters[j] &&
               infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0) 
Shane Snyder's avatar
Shane Snyder committed
386
                tmp_file.fcounters[j] = infile->fcounters[j];
387 388
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
Shane Snyder's avatar
Shane Snyder committed
389 390 391
        }

        /* max */
392
        for(j=HDF5_F_OPEN_END_TIMESTAMP; j<=HDF5_F_CLOSE_END_TIMESTAMP; j++)
Shane Snyder's avatar
Shane Snyder committed
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
        {
            if(infile->fcounters[j] > inoutfile->fcounters[j])
                tmp_file.fcounters[j] = infile->fcounters[j];
            else
                tmp_file.fcounters[j] = inoutfile->fcounters[j];
        }

        /* update pointers */
        *inoutfile = tmp_file;
        inoutfile++;
        infile++;
    }

    return;
}
408
#endif
Shane Snyder's avatar
Shane Snyder committed
409

410 411 412 413
/************************************************************************
 * Functions exported by HDF5 module for coordinating with darshan-core *
 ************************************************************************/

414 415 416
#ifdef HAVE_MPI
static void hdf5_mpi_redux(
    void *hdf5_buf,
Shane Snyder's avatar
Shane Snyder committed
417 418
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
419
    int shared_rec_count)
Shane Snyder's avatar
Shane Snyder committed
420
{
421
    int hdf5_rec_count;
422 423
    struct hdf5_file_record_ref *rec_ref;
    struct darshan_hdf5_file *hdf5_rec_buf = *(struct darshan_hdf5_file **)hdf5_buf;
Shane Snyder's avatar
Shane Snyder committed
424 425 426 427
    struct darshan_hdf5_file *red_send_buf = NULL;
    struct darshan_hdf5_file *red_recv_buf = NULL;
    MPI_Datatype red_type;
    MPI_Op red_op;
428
    int i;
Shane Snyder's avatar
Shane Snyder committed
429

430
    HDF5_LOCK();
Shane Snyder's avatar
Shane Snyder committed
431
    assert(hdf5_runtime);
432

433 434 435 436 437 438 439 440 441
    /* necessary initialization of shared records */
    for(i = 0; i < shared_rec_count; i++)
    {
        rec_ref = darshan_lookup_record_ref(hdf5_runtime->rec_id_hash,
            &shared_recs[i], sizeof(darshan_record_id));
        assert(rec_ref);

        rec_ref->file_rec->base_rec.rank = -1;
    }
Shane Snyder's avatar
Shane Snyder committed
442

443 444
    /* sort the array of records so we get all of the shared records
     * (marked by rank -1) in a contiguous portion at end of the array
445
     */
446 447 448 449 450 451 452 453
    darshan_record_sort(hdf5_rec_buf, hdf5_rec_count,
        sizeof(struct darshan_hdf5_file));

    /* make *send_buf point to the shared files at the end of sorted array */
    red_send_buf = &(hdf5_rec_buf[hdf5_rec_count-shared_rec_count]);

    /* allocate memory for the reduction output on rank 0 */
    if(my_rank == 0)
Shane Snyder's avatar
Shane Snyder committed
454
    {
455 456
        red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_hdf5_file));
        if(!red_recv_buf)
Shane Snyder's avatar
Shane Snyder committed
457
        {
458 459
            HDF5_UNLOCK();
            return;
Shane Snyder's avatar
Shane Snyder committed
460
        }
461
    }
Shane Snyder's avatar
Shane Snyder committed
462

463 464 465 466 467 468
    /* construct a datatype for a HDF5 file record.  This is serving no purpose
     * except to make sure we can do a reduction on proper boundaries
     */
    PMPI_Type_contiguous(sizeof(struct darshan_hdf5_file),
        MPI_BYTE, &red_type);
    PMPI_Type_commit(&red_type);
Shane Snyder's avatar
Shane Snyder committed
469

470 471
    /* register a HDF5 file record reduction operator */
    PMPI_Op_create(hdf5_record_reduction_op, 1, &red_op);
Shane Snyder's avatar
Shane Snyder committed
472

473 474 475
    /* reduce shared HDF5 file records */
    PMPI_Reduce(red_send_buf, red_recv_buf,
        shared_rec_count, red_type, red_op, 0, mod_comm);
Shane Snyder's avatar
Shane Snyder committed
476

477 478 479 480 481 482 483 484 485 486
    /* clean up reduction state */
    if(my_rank == 0)
    {
        int tmp_ndx = hdf5_rec_count - shared_rec_count;
        memcpy(&(hdf5_rec_buf[tmp_ndx]), red_recv_buf,
            shared_rec_count * sizeof(struct darshan_hdf5_file));
        free(red_recv_buf);
    }
    else
    {
487
        hdf5_runtime->file_rec_count -= shared_rec_count;
488
    }
Shane Snyder's avatar
Shane Snyder committed
489

490 491
    PMPI_Type_free(&red_type);
    PMPI_Op_free(&red_op);
Shane Snyder's avatar
Shane Snyder committed
492

493 494 495 496
    HDF5_UNLOCK();
    return;
}
#endif
Shane Snyder's avatar
Shane Snyder committed
497

498 499 500 501 502
static void hdf5_shutdown(
    void **hdf5_buf,
    int *hdf5_buf_sz)
{
    int hdf5_rec_count;
Shane Snyder's avatar
Shane Snyder committed
503

504 505
    HDF5_LOCK();
    assert(hdf5_runtime);
Shane Snyder's avatar
Shane Snyder committed
506

507
    hdf5_rec_count = hdf5_runtime->file_rec_count;
Shane Snyder's avatar
Shane Snyder committed
508

509 510
    /* shutdown internal structures used for instrumenting */
    hdf5_cleanup_runtime();
Shane Snyder's avatar
Shane Snyder committed
511

512 513 514
    /* update output buffer size to account for shared file reduction */
    *hdf5_buf_sz = hdf5_rec_count * sizeof(struct darshan_hdf5_file);

515
    HDF5_UNLOCK();
Shane Snyder's avatar
Shane Snyder committed
516 517
    return;
}
518 519 520 521 522 523 524 525 526

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */