darshan-apmpi-log-format.h 6.31 KB
Newer Older
Sudheer Chunduri's avatar
Sudheer Chunduri committed
1 2 3 4 5 6
/*
 * Copyright (C) 2017 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

Sudheer Chunduri's avatar
Sudheer Chunduri committed
7 8
#ifndef __APMPI_LOG_FORMAT_H
#define __APMPI_LOG_FORMAT_H
9 10 11
#ifndef MPI_MAX_PROCESSOR_NAME
#define MPI_MAX_PROCESSOR_NAME 128
#endif
Sudheer Chunduri's avatar
Sudheer Chunduri committed
12
/* current AutoPerf MPI log format version */
13
#define APMPI_VER 1
Sudheer Chunduri's avatar
Sudheer Chunduri committed
14

Sudheer Chunduri's avatar
Sudheer Chunduri committed
15
#define APMPI_MAGIC ('A'*0x100000000+\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
16
                            'P'*0x1000000+\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
17 18 19
                            'M'*0x10000+\
                            'P'*0x100+\
                            'I'*0x1)
20

Sudheer Chunduri's avatar
Sudheer Chunduri committed
21 22
#define APMPI_MPI_BLOCKING_P2P \
        X(MPI_SEND) \
23
        X(MPI_SSEND) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
24 25
        X(MPI_RSEND) \
        X(MPI_BSEND) \
26
        /*X(MPI_SEND_INIT) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
27 28 29 30
        X(MPI_SSEND_INIT) \
        X(MPI_RSEND_INIT) \
        X(MPI_BSEND_INIT) \ */ \
        X(MPI_RECV)  /*
31
        X(MPI_RECV_INIT) */\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
32
        X(MPI_SENDRECV) \
33 34
        X(MPI_SENDRECV_REPLACE) 

Sudheer Chunduri's avatar
Sudheer Chunduri committed
35 36 37 38 39
#define APMPI_MPI_NONBLOCKING_P2P \
        X(MPI_ISEND) \
        X(MPI_ISSEND) \
        X(MPI_IRSEND) \
        X(MPI_IBSEND) \
40 41 42
        X(MPI_IRECV) /*\
	X(MPI_ISENDRECV) \
	X(MPI_ISENDRECV_REPLACE) */
43 44
     /*   C(MPI_NONBLOCKING_P2P_CNT) */

Sudheer Chunduri's avatar
Sudheer Chunduri committed
45
#define AMPI_MPI_P2P_MISC \
46 47 48 49 50 51 52 53 54 55 56 57 58
        V(MPI_PROBE) \
        V(MPI_IPROBE) \
	V(MPI_TEST) \
	V(MPI_TESTANY) \
	V(MPI_TESTALL) \
	V(MPI_TESTSOME) \
	V(MPI_WAIT) \
	V(MPI_WAITANY) \
	V(MPI_WAITALL) \
	V(MPI_WAITSOME) /*
	V(MPI_START) \
	V(MPI_STARTALL)		*/

59 60
#define APMPI_MPI_COLL_SYNC \
        V(MPI_BARRIER) 
61 62
#define APMPI_MPI_ICOLL_SYNC \
        V(MPI_IBARRIER) 
63

Sudheer Chunduri's avatar
Sudheer Chunduri committed
64
#define APMPI_MPI_BLOCKING_COLL \
65
        X(MPI_BCAST) \
66
        X(MPI_GATHER)   \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
67 68 69 70
        X(MPI_GATHERV)  \
        X(MPI_SCATTER)  \
        X(MPI_SCATTERV) \
        X(MPI_SCAN)     \
71
        X(MPI_EXSCAN)     \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
72 73 74
        X(MPI_ALLGATHER)        \
        X(MPI_ALLGATHERV)       \
        X(MPI_REDUCE)   \
75
        X(MPI_ALLREDUCE)        \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
76
        X(MPI_REDUCE_SCATTER)   \
77
        X(MPI_ALLTOALL)         \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
78
        X(MPI_ALLTOALLV)        \
79
        X(MPI_ALLTOALLW)   /*    \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
80
        Y(MPI_BLOCKING_COLL_CNT) */
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98

#define APMPI_MPI_NONBLOCKING_COLL \
        X(MPI_IBCAST) \
        X(MPI_IGATHER)   \
        X(MPI_IGATHERV)  \
        X(MPI_ISCATTER)  \
        X(MPI_ISCATTERV) \
        X(MPI_ISCAN)     \
        X(MPI_IEXSCAN)     \
        X(MPI_IALLGATHER)        \
        X(MPI_IALLGATHERV)       \
        X(MPI_IREDUCE)   \
        X(MPI_IALLREDUCE)        \
        X(MPI_IREDUCE_SCATTER)   \
        X(MPI_IALLTOALL)         \
        X(MPI_IALLTOALLV)        \
        X(MPI_IALLTOALLW)   

99 100
#define APMPI_MPI_ONESIDED \
	X(MPI_PUT) \
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
	X(MPI_GET) \
	X(MPI_ACCUMULATE) \
	X(MPI_GET_ACCUMULATE) \
	V(MPI_FETCH_AND_OP) \
	V(MPI_COMPARE_AND_SWAP) /*\
	X(MPI_WIN_CREATE) \
	X(MPI_WIN_ALLOCATE) \
	X(MPI_WIN_CREATE_DYNAMIC) \
	X(MPI_WIN_ALLOCATE_SHARED) \
	X(MPI_WIN_ATTACH) \
	X(MPI_WIN_DETACH) */ \
	V(MPI_WIN_FENCE) \
	V(MPI_WIN_START) \
	V(MPI_WIN_COMPLETE) \
	V(MPI_WIN_POST) \
	V(MPI_WIN_WAIT) \
	V(MPI_WIN_TEST) \
	V(MPI_WIN_LOCK) \
	V(MPI_WIN_UNLOCK) \
	V(MPI_WIN_UNLOCK_ALL) \
	V(MPI_WIN_FLUSH) \
	V(MPI_WIN_FLUSH_ALL) \
	V(MPI_WIN_FLUSH_LOCAL) \
	V(MPI_WIN_FLUSH_LOCAL_ALL) \
	V(MPI_WIN_SYNC)
126

Sudheer Chunduri's avatar
Sudheer Chunduri committed
127 128
#define I(a) \
         Y(a ## _CALL_COUNT) \
129 130 131
      /* Y(MPIOP_BUF_SOURCE) \  0-CPU, 1-GPU (if we can determine if it is GPU buffer then we can repeat all the counters in this record 
				for GPU buffer based calls? 
				For an MPIOP, some of its call can use CPU buffers and some can be using GPU buffers ... */ \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
132 133 134 135 136 137 138 139
         Y(a ## _TOTAL_BYTES) \
         Y(a ## _MSG_SIZE_AGG_0_256) \
         Y(a ## _MSG_SIZE_AGG_256_1K) \
         Y(a ## _MSG_SIZE_AGG_1K_8K) \
         Y(a ## _MSG_SIZE_AGG_8K_256K) \
         Y(a ## _MSG_SIZE_AGG_256K_1M) \
         Y(a ## _MSG_SIZE_AGG_1M_PLUS) \

140 141 142
#define J(a) \
         Y(a ## _CALL_COUNT) \

143
#define APMPI_MPIOP_COUNTERS \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
144
        APMPI_MPI_BLOCKING_P2P \
145 146
        APMPI_MPI_NONBLOCKING_P2P \
	AMPI_MPI_P2P_MISC \
147
	APMPI_MPI_COLL_SYNC \
148
	APMPI_MPI_ICOLL_SYNC \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
149
        APMPI_MPI_BLOCKING_COLL \
150 151
        APMPI_MPI_NONBLOCKING_COLL \
	APMPI_MPI_ONESIDED \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
152
        Z(APMPI_NUM_INDICES)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
153

Sudheer Chunduri's avatar
Sudheer Chunduri committed
154 155 156
#define Y(a) a,
#define Z(a) a
#define X I
157
#define V J
158 159
/* integer counters for the "APMPI" module */
enum apmpi_mpiop_indices
Sudheer Chunduri's avatar
Sudheer Chunduri committed
160
{
161
    APMPI_MPIOP_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
162
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
163
#undef X
164
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
165

166
	/* per MPI op total times across the calls */
167
#define F_TIME(a) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
168
        Y(a ## _TOTAL_TIME) \
169 170
        Y(a ## _MIN_TIME) \
        Y(a ## _MAX_TIME) 
Sudheer Chunduri's avatar
Sudheer Chunduri committed
171

172
#define APMPI_F_MPIOP_TOTALTIME_COUNTERS \
173 174 175
        APMPI_MPI_BLOCKING_P2P  \
        APMPI_MPI_NONBLOCKING_P2P  \
	AMPI_MPI_P2P_MISC \
176
	APMPI_MPI_COLL_SYNC \
177
	APMPI_MPI_ICOLL_SYNC \
178
        APMPI_MPI_BLOCKING_COLL \
179 180
        APMPI_MPI_NONBLOCKING_COLL \
	APMPI_MPI_ONESIDED \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
181
        Z(APMPI_F_MPIOP_TOTALTIME_NUM_INDICES) 
Sudheer Chunduri's avatar
Sudheer Chunduri committed
182

183
/* float counters for the "APMPI" module */
184 185
#define X F_TIME
#define V F_TIME
186
enum apmpi_f_mpiop_totaltime_indices
Sudheer Chunduri's avatar
Sudheer Chunduri committed
187
{
188
    APMPI_F_MPIOP_TOTALTIME_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
189
};
190
#undef X
191
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
192

193 194 195
#define F_SYNC(a) \
        Y(a ## _TOTAL_SYNC_TIME) 
#define APMPI_F_MPIOP_SYNCTIME_COUNTERS \
196
	APMPI_MPI_COLL_SYNC \
197
	APMPI_MPI_BLOCKING_COLL \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
198
        Z(APMPI_F_MPIOP_SYNCTIME_NUM_INDICES) 
199 200
/* float counters for the "APMPI" module */
#define X F_SYNC
201
#define V F_SYNC
Sudheer Chunduri's avatar
Sudheer Chunduri committed
202
enum apmpi_f_mpiop_synctime_indices
203 204 205
{
    APMPI_F_MPIOP_SYNCTIME_COUNTERS
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
206
#undef X
207
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
208

209
        /* aggregate (across all the ranks) per MPI op times  */ 
210
#define APMPI_F_MPI_GLOBAL_COUNTERS \
211
	Y(RANK_TOTAL_MPITIME) \
212
	Y(RANK_TOTAL_MPISYNCTIME) \
213
	Z(APMPI_F_MPI_GLOBAL_NUM_INDICES)
214 215
enum apmpi_f_mpi_global_indices
{
216
    APMPI_F_MPI_GLOBAL_COUNTERS
217 218 219
};
#undef Z
#undef Y
Sudheer Chunduri's avatar
Sudheer Chunduri committed
220 221
/* the darshan_apmpi_record structure encompasses the data/counters
 * which would actually be logged to file by Darshan for the AP MPI
Sudheer Chunduri's avatar
Sudheer Chunduri committed
222 223 224 225 226 227
 * module. This example implementation logs the following data for each
 * record:
 *      - a darshan_base_record structure, which contains the record id & rank
 *      - integer I/O counters 
 *      - floating point I/O counters 
 */
Sudheer Chunduri's avatar
Sudheer Chunduri committed
228
struct darshan_apmpi_perf_record
Sudheer Chunduri's avatar
Sudheer Chunduri committed
229 230
{
    struct darshan_base_record base_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
231
    uint64_t counters[APMPI_NUM_INDICES];
Sudheer Chunduri's avatar
Sudheer Chunduri committed
232 233
    double fcounters[APMPI_F_MPIOP_TOTALTIME_NUM_INDICES];
    double fsynccounters[APMPI_F_MPIOP_SYNCTIME_NUM_INDICES];
234
    double fglobalcounters[APMPI_F_MPI_GLOBAL_NUM_INDICES];
235
    char node_name[MPI_MAX_PROCESSOR_NAME];
Sudheer Chunduri's avatar
Sudheer Chunduri committed
236
};
237 238 239 240
struct darshan_apmpi_header_record
{
    struct darshan_base_record base_rec;
    int64_t magic;
241
    uint32_t sync_flag;
242
    double apmpi_f_variance_total_mpitime;
243
    double apmpi_f_variance_total_mpisynctime;
244
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
245

Sudheer Chunduri's avatar
Sudheer Chunduri committed
246
#endif /* __APMPI_LOG_FORMAT_H */