GitLab maintenance scheduled for Tomorrow, 2020-08-11, from 17:00 to 18:00 CT - Services will be unavailable during this time.

darshan-hdf5-log-format.h 8.11 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#ifndef __DARSHAN_HDF5_LOG_FORMAT_H
#define __DARSHAN_HDF5_LOG_FORMAT_H

10 11 12
/* current HDF5 log format versions */
#define DARSHAN_H5F_VER 3
#define DARSHAN_H5D_VER 1
13

14 15 16 17 18 19 20 21 22
#define H5D_MAX_NDIMS 5

#define H5F_COUNTERS \
    /* count of HDF5 file opens/creates */\
    X(H5F_OPENS) \
    /* count of HDF5 file flushes */\
    X(H5F_FLUSHES) \
    /* flag indicating whether MPI-IO is used for accessing this file */\
    X(H5F_USE_MPIIO) \
23
    /* end of counters */\
24 25 26 27 28 29 30 31 32 33 34 35 36
    X(H5F_NUM_INDICES)

#define H5F_F_COUNTERS \
    /* timestamp of first file open */\
    X(H5F_F_OPEN_START_TIMESTAMP) \
    /* timestamp of first file close */\
    X(H5F_F_CLOSE_START_TIMESTAMP) \
    /* timestamp of last file open */\
    X(H5F_F_OPEN_END_TIMESTAMP) \
    /* timestamp of last file close */\
    X(H5F_F_CLOSE_END_TIMESTAMP) \
    /* cumulative H5F meta time */\
    X(H5F_F_META_TIME) \
37
    /* end of counters*/\
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
    X(H5F_F_NUM_INDICES)

#define H5D_COUNTERS \
    /* count of HDF5 dataset opens/creates */\
    X(H5D_OPENS) \
    /* count of HDF5 datset reads */\
    X(H5D_READS) \
    /* count of HDF5 datset writes */\
    X(H5D_WRITES) \
    /* count of HDF5 dataset flushes */\
    X(H5D_FLUSHES) \
    /* total bytes read */\
    X(H5D_BYTES_READ) \
    /* total bytes written */\
    X(H5D_BYTES_WRITTEN) \
    /* number of times switching between H5D read and write */\
    X(H5D_RW_SWITCHES) \
    /* number of read/write ops with regular hyperslab selections */\
    X(H5D_REGULAR_HYPERSLAB_SELECTS) \
    /* number of read/write ops with irregular hyperslab selections */\
    X(H5D_IRREGULAR_HYPERSLAB_SELECTS) \
    /* number of read/write ops with point selections */\
    X(H5D_POINT_SELECTS) \
    /* sizes of the maximum read/write operations */\
    X(H5D_MAX_READ_TIME_SIZE) \
    X(H5D_MAX_WRITE_TIME_SIZE) \
    /* buckets for H5D read size ranges */\
    X(H5D_SIZE_READ_AGG_0_100) \
    X(H5D_SIZE_READ_AGG_100_1K) \
    X(H5D_SIZE_READ_AGG_1K_10K) \
    X(H5D_SIZE_READ_AGG_10K_100K) \
    X(H5D_SIZE_READ_AGG_100K_1M) \
    X(H5D_SIZE_READ_AGG_1M_4M) \
    X(H5D_SIZE_READ_AGG_4M_10M) \
    X(H5D_SIZE_READ_AGG_10M_100M) \
    X(H5D_SIZE_READ_AGG_100M_1G) \
    X(H5D_SIZE_READ_AGG_1G_PLUS) \
    /* buckets for H5D write size ranges */\
    X(H5D_SIZE_WRITE_AGG_0_100) \
    X(H5D_SIZE_WRITE_AGG_100_1K) \
    X(H5D_SIZE_WRITE_AGG_1K_10K) \
    X(H5D_SIZE_WRITE_AGG_10K_100K) \
    X(H5D_SIZE_WRITE_AGG_100K_1M) \
    X(H5D_SIZE_WRITE_AGG_1M_4M) \
    X(H5D_SIZE_WRITE_AGG_4M_10M) \
    X(H5D_SIZE_WRITE_AGG_10M_100M) \
    X(H5D_SIZE_WRITE_AGG_100M_1G) \
    X(H5D_SIZE_WRITE_AGG_1G_PLUS) \
    /* the four most frequently appearing H5D accesses, identified by
     * access size and access lengths (count * block) and strides
     * in last 5 hyperslab dimensions */\
    X(H5D_ACCESS1_ACCESS) \
    X(H5D_ACCESS1_LENGTH_D1) \
    X(H5D_ACCESS1_LENGTH_D2) \
    X(H5D_ACCESS1_LENGTH_D3) \
    X(H5D_ACCESS1_LENGTH_D4) \
    X(H5D_ACCESS1_LENGTH_D5) \
    X(H5D_ACCESS1_STRIDE_D1) \
    X(H5D_ACCESS1_STRIDE_D2) \
    X(H5D_ACCESS1_STRIDE_D3) \
    X(H5D_ACCESS1_STRIDE_D4) \
    X(H5D_ACCESS1_STRIDE_D5) \
    X(H5D_ACCESS2_ACCESS) \
    X(H5D_ACCESS2_LENGTH_D1) \
    X(H5D_ACCESS2_LENGTH_D2) \
    X(H5D_ACCESS2_LENGTH_D3) \
    X(H5D_ACCESS2_LENGTH_D4) \
    X(H5D_ACCESS2_LENGTH_D5) \
    X(H5D_ACCESS2_STRIDE_D1) \
    X(H5D_ACCESS2_STRIDE_D2) \
    X(H5D_ACCESS2_STRIDE_D3) \
    X(H5D_ACCESS2_STRIDE_D4) \
    X(H5D_ACCESS2_STRIDE_D5) \
    X(H5D_ACCESS3_ACCESS) \
    X(H5D_ACCESS3_LENGTH_D1) \
    X(H5D_ACCESS3_LENGTH_D2) \
    X(H5D_ACCESS3_LENGTH_D3) \
    X(H5D_ACCESS3_LENGTH_D4) \
    X(H5D_ACCESS3_LENGTH_D5) \
    X(H5D_ACCESS3_STRIDE_D1) \
    X(H5D_ACCESS3_STRIDE_D2) \
    X(H5D_ACCESS3_STRIDE_D3) \
    X(H5D_ACCESS3_STRIDE_D4) \
    X(H5D_ACCESS3_STRIDE_D5) \
    X(H5D_ACCESS4_ACCESS) \
    X(H5D_ACCESS4_LENGTH_D1) \
    X(H5D_ACCESS4_LENGTH_D2) \
    X(H5D_ACCESS4_LENGTH_D3) \
    X(H5D_ACCESS4_LENGTH_D4) \
    X(H5D_ACCESS4_LENGTH_D5) \
    X(H5D_ACCESS4_STRIDE_D1) \
    X(H5D_ACCESS4_STRIDE_D2) \
    X(H5D_ACCESS4_STRIDE_D3) \
    X(H5D_ACCESS4_STRIDE_D4) \
    X(H5D_ACCESS4_STRIDE_D5) \
    /* count of each of the most frequent H5D access sizes */\
    X(H5D_ACCESS1_COUNT) \
    X(H5D_ACCESS2_COUNT) \
    X(H5D_ACCESS3_COUNT) \
    X(H5D_ACCESS4_COUNT) \
    /* number of dimensions in dataset's dataspace */\
    X(H5D_DATASPACE_NDIMS) \
    /* number of points in dataset's dataspace */\
    X(H5D_DATASPACE_NPOINTS) \
    /* size of dataset elements in bytes */\
    X(H5D_DATATYPE_SIZE) \
    /* chunk sizes in the last 5 dimensions of the dataset */\
    /* NOTE: D1 is the last dimension (i.e., row) , D2 is 2nd to last (i.e., column), and so on */\
    X(H5D_CHUNK_SIZE_D1) \
    X(H5D_CHUNK_SIZE_D2) \
    X(H5D_CHUNK_SIZE_D3) \
    X(H5D_CHUNK_SIZE_D4) \
    X(H5D_CHUNK_SIZE_D5) \
    /* flag indicating use of MPI-IO collectives */\
    X(H5D_USE_MPIIO_COLLECTIVE) \
    /* flag indicating whether deprecated create/open calls were used */\
    X(H5D_USE_DEPRECATED) \
    /* rank and number of bytes moved for fastest/slowest ranks */\
    X(H5D_FASTEST_RANK) \
    X(H5D_FASTEST_RANK_BYTES) \
    X(H5D_SLOWEST_RANK) \
    X(H5D_SLOWEST_RANK_BYTES) \
    /* end of counters */\
    X(H5D_NUM_INDICES)

#define H5D_F_COUNTERS \
    /* timestamp of first dataset open */\
    X(H5D_F_OPEN_START_TIMESTAMP) \
    /* timestamp of first dataset read */\
    X(H5D_F_READ_START_TIMESTAMP) \
    /* timestamp of first dataset write */\
    X(H5D_F_WRITE_START_TIMESTAMP) \
    /* timestamp of first dataset close */\
    X(H5D_F_CLOSE_START_TIMESTAMP) \
    /* timestamp of last dataset open */\
    X(H5D_F_OPEN_END_TIMESTAMP) \
    /* timestamp of last dataset read */\
    X(H5D_F_READ_END_TIMESTAMP) \
    /* timestamp of last dataset write */\
    X(H5D_F_WRITE_END_TIMESTAMP) \
    /* timestamp of last dataset close */\
    X(H5D_F_CLOSE_END_TIMESTAMP) \
    /* cumulative H5D read time */\
    X(H5D_F_READ_TIME) \
    /* cumulative H5D write time */\
    X(H5D_F_WRITE_TIME) \
    /* cumulative H5D meta time */\
    X(H5D_F_META_TIME) \
    /* maximum H5D read duration */\
    X(H5D_F_MAX_READ_TIME) \
    /* maximum H5D write duration */\
    X(H5D_F_MAX_WRITE_TIME) \
    /* total i/o and meta time for fastest/slowest ranks */\
    X(H5D_F_FASTEST_RANK_TIME) \
    X(H5D_F_SLOWEST_RANK_TIME) \
    /* variance of total i/o time and bytes moved across all ranks */\
    /* NOTE: for shared records only */\
    X(H5D_F_VARIANCE_RANK_TIME) \
    X(H5D_F_VARIANCE_RANK_BYTES) \
    /* end of counters*/\
    X(H5D_F_NUM_INDICES)
199 200 201

#define X(a) a,
/* integer statistics for HDF5 file records */
202 203 204 205 206 207
enum darshan_h5f_indices
{
    H5F_COUNTERS
};
/* integer statistics for HDF5 dataset records */
enum darshan_h5d_indices
208
{
209
    H5D_COUNTERS
210 211 212
};

/* floating point statistics for HDF5 file records */
213
enum darshan_h5f_f_indices
214
{
215 216 217 218 219 220
    H5F_F_COUNTERS
};
/* floating point statistics for HDF5 dataset records */
enum darshan_h5d_f_indices
{
    H5D_F_COUNTERS
221 222 223
};
#undef X

224
/* record structure for HDF5 files. a record is created and stored for
225 226
 * every HDF5 file opened by the original application. For the HDF5 module,
 * the record includes:
227
 *      - a darshan_base_record structure, which contains the record id & rank
228
 *      - integer file I/O statistics (open, read/write counts, etc)
229
 *      - floating point file I/O statistics (timestamps, cumulative timers, etc.)
230 231 232
 */
struct darshan_hdf5_file
{
233
    struct darshan_base_record base_rec;
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
    int64_t counters[H5F_NUM_INDICES];
    double fcounters[H5F_F_NUM_INDICES];
};

/* record structure for HDF5 datasets. a record is created and stored for
 * every HDF5 dataset opened by the original application. For the HDF5 module,
 * the record includes:
 *      - a darshan_base_record structure, which contains the record id & rank
 *      - integer dataset I/O statistics (open, read/write counts, etc)
 *      - floating point dataset I/O statistics (timestamps, cumulative timers, etc.)
 */
struct darshan_hdf5_dataset
{
    struct darshan_base_record base_rec;
    int64_t counters[H5D_NUM_INDICES];
    double fcounters[H5D_F_NUM_INDICES];
250 251 252
};

#endif /* __DARSHAN_HDF5_LOG_FORMAT_H */