darshan-apmpi-log-format.h 6.31 KB
Newer Older
Sudheer Chunduri's avatar
Sudheer Chunduri committed
1
2
3
4
5
6
/*
 * Copyright (C) 2017 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

Sudheer Chunduri's avatar
Sudheer Chunduri committed
7
8
#ifndef __APMPI_LOG_FORMAT_H
#define __APMPI_LOG_FORMAT_H
9
10
11
#ifndef MPI_MAX_PROCESSOR_NAME
#define MPI_MAX_PROCESSOR_NAME 128
#endif
Sudheer Chunduri's avatar
Sudheer Chunduri committed
12
/* current AutoPerf MPI log format version */
13
#define APMPI_VER 1
Sudheer Chunduri's avatar
Sudheer Chunduri committed
14

Sudheer Chunduri's avatar
Sudheer Chunduri committed
15
#define APMPI_MAGIC ('A'*0x100000000+\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
16
                            'P'*0x1000000+\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
17
18
19
                            'M'*0x10000+\
                            'P'*0x100+\
                            'I'*0x1)
20

Sudheer Chunduri's avatar
Sudheer Chunduri committed
21
22
#define APMPI_MPI_BLOCKING_P2P \
        X(MPI_SEND) \
23
        X(MPI_SSEND) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
24
25
        X(MPI_RSEND) \
        X(MPI_BSEND) \
26
        /*X(MPI_SEND_INIT) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
27
28
29
30
        X(MPI_SSEND_INIT) \
        X(MPI_RSEND_INIT) \
        X(MPI_BSEND_INIT) \ */ \
        X(MPI_RECV)  /*
31
        X(MPI_RECV_INIT) */\
Sudheer Chunduri's avatar
Sudheer Chunduri committed
32
        X(MPI_SENDRECV) \
33
34
        X(MPI_SENDRECV_REPLACE) 

Sudheer Chunduri's avatar
Sudheer Chunduri committed
35
36
37
38
39
#define APMPI_MPI_NONBLOCKING_P2P \
        X(MPI_ISEND) \
        X(MPI_ISSEND) \
        X(MPI_IRSEND) \
        X(MPI_IBSEND) \
40
41
42
        X(MPI_IRECV) /*\
	X(MPI_ISENDRECV) \
	X(MPI_ISENDRECV_REPLACE) */
43
44
     /*   C(MPI_NONBLOCKING_P2P_CNT) */

Sudheer Chunduri's avatar
Sudheer Chunduri committed
45
#define AMPI_MPI_P2P_MISC \
46
47
48
49
50
51
52
53
54
55
56
57
58
        V(MPI_PROBE) \
        V(MPI_IPROBE) \
	V(MPI_TEST) \
	V(MPI_TESTANY) \
	V(MPI_TESTALL) \
	V(MPI_TESTSOME) \
	V(MPI_WAIT) \
	V(MPI_WAITANY) \
	V(MPI_WAITALL) \
	V(MPI_WAITSOME) /*
	V(MPI_START) \
	V(MPI_STARTALL)		*/

59
60
#define APMPI_MPI_COLL_SYNC \
        V(MPI_BARRIER) 
61
62
#define APMPI_MPI_ICOLL_SYNC \
        V(MPI_IBARRIER) 
63

Sudheer Chunduri's avatar
Sudheer Chunduri committed
64
#define APMPI_MPI_BLOCKING_COLL \
65
        X(MPI_BCAST) \
66
        X(MPI_GATHER)   \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
67
68
69
70
        X(MPI_GATHERV)  \
        X(MPI_SCATTER)  \
        X(MPI_SCATTERV) \
        X(MPI_SCAN)     \
71
        X(MPI_EXSCAN)     \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
72
73
74
        X(MPI_ALLGATHER)        \
        X(MPI_ALLGATHERV)       \
        X(MPI_REDUCE)   \
75
        X(MPI_ALLREDUCE)        \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
76
        X(MPI_REDUCE_SCATTER)   \
77
        X(MPI_ALLTOALL)         \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
78
        X(MPI_ALLTOALLV)        \
79
        X(MPI_ALLTOALLW)   /*    \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
80
        Y(MPI_BLOCKING_COLL_CNT) */
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

#define APMPI_MPI_NONBLOCKING_COLL \
        X(MPI_IBCAST) \
        X(MPI_IGATHER)   \
        X(MPI_IGATHERV)  \
        X(MPI_ISCATTER)  \
        X(MPI_ISCATTERV) \
        X(MPI_ISCAN)     \
        X(MPI_IEXSCAN)     \
        X(MPI_IALLGATHER)        \
        X(MPI_IALLGATHERV)       \
        X(MPI_IREDUCE)   \
        X(MPI_IALLREDUCE)        \
        X(MPI_IREDUCE_SCATTER)   \
        X(MPI_IALLTOALL)         \
        X(MPI_IALLTOALLV)        \
        X(MPI_IALLTOALLW)   

99
100
#define APMPI_MPI_ONESIDED \
	X(MPI_PUT) \
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
	X(MPI_GET) \
	X(MPI_ACCUMULATE) \
	X(MPI_GET_ACCUMULATE) \
	V(MPI_FETCH_AND_OP) \
	V(MPI_COMPARE_AND_SWAP) /*\
	X(MPI_WIN_CREATE) \
	X(MPI_WIN_ALLOCATE) \
	X(MPI_WIN_CREATE_DYNAMIC) \
	X(MPI_WIN_ALLOCATE_SHARED) \
	X(MPI_WIN_ATTACH) \
	X(MPI_WIN_DETACH) */ \
	V(MPI_WIN_FENCE) \
	V(MPI_WIN_START) \
	V(MPI_WIN_COMPLETE) \
	V(MPI_WIN_POST) \
	V(MPI_WIN_WAIT) \
	V(MPI_WIN_TEST) \
	V(MPI_WIN_LOCK) \
	V(MPI_WIN_UNLOCK) \
	V(MPI_WIN_UNLOCK_ALL) \
	V(MPI_WIN_FLUSH) \
	V(MPI_WIN_FLUSH_ALL) \
	V(MPI_WIN_FLUSH_LOCAL) \
	V(MPI_WIN_FLUSH_LOCAL_ALL) \
	V(MPI_WIN_SYNC)
126

Sudheer Chunduri's avatar
Sudheer Chunduri committed
127
128
#define I(a) \
         Y(a ## _CALL_COUNT) \
129
130
131
      /* Y(MPIOP_BUF_SOURCE) \  0-CPU, 1-GPU (if we can determine if it is GPU buffer then we can repeat all the counters in this record 
				for GPU buffer based calls? 
				For an MPIOP, some of its call can use CPU buffers and some can be using GPU buffers ... */ \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
132
133
134
135
136
137
138
139
         Y(a ## _TOTAL_BYTES) \
         Y(a ## _MSG_SIZE_AGG_0_256) \
         Y(a ## _MSG_SIZE_AGG_256_1K) \
         Y(a ## _MSG_SIZE_AGG_1K_8K) \
         Y(a ## _MSG_SIZE_AGG_8K_256K) \
         Y(a ## _MSG_SIZE_AGG_256K_1M) \
         Y(a ## _MSG_SIZE_AGG_1M_PLUS) \

140
141
142
#define J(a) \
         Y(a ## _CALL_COUNT) \

143
#define APMPI_MPIOP_COUNTERS \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
144
        APMPI_MPI_BLOCKING_P2P \
145
146
        APMPI_MPI_NONBLOCKING_P2P \
	AMPI_MPI_P2P_MISC \
147
	APMPI_MPI_COLL_SYNC \
148
	APMPI_MPI_ICOLL_SYNC \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
149
        APMPI_MPI_BLOCKING_COLL \
150
151
        APMPI_MPI_NONBLOCKING_COLL \
	APMPI_MPI_ONESIDED \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
152
        Z(APMPI_NUM_INDICES)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
153

Sudheer Chunduri's avatar
Sudheer Chunduri committed
154
155
156
#define Y(a) a,
#define Z(a) a
#define X I
157
#define V J
158
159
/* integer counters for the "APMPI" module */
enum apmpi_mpiop_indices
Sudheer Chunduri's avatar
Sudheer Chunduri committed
160
{
161
    APMPI_MPIOP_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
162
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
163
#undef X
164
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
165

166
	/* per MPI op total times across the calls */
167
#define F_TIME(a) \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
168
        Y(a ## _TOTAL_TIME) \
169
170
        Y(a ## _MIN_TIME) \
        Y(a ## _MAX_TIME) 
Sudheer Chunduri's avatar
Sudheer Chunduri committed
171

172
#define APMPI_F_MPIOP_TOTALTIME_COUNTERS \
173
174
175
        APMPI_MPI_BLOCKING_P2P  \
        APMPI_MPI_NONBLOCKING_P2P  \
	AMPI_MPI_P2P_MISC \
176
	APMPI_MPI_COLL_SYNC \
177
	APMPI_MPI_ICOLL_SYNC \
178
        APMPI_MPI_BLOCKING_COLL \
179
180
        APMPI_MPI_NONBLOCKING_COLL \
	APMPI_MPI_ONESIDED \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
181
        Z(APMPI_F_MPIOP_TOTALTIME_NUM_INDICES) 
Sudheer Chunduri's avatar
Sudheer Chunduri committed
182

183
/* float counters for the "APMPI" module */
184
185
#define X F_TIME
#define V F_TIME
186
enum apmpi_f_mpiop_totaltime_indices
Sudheer Chunduri's avatar
Sudheer Chunduri committed
187
{
188
    APMPI_F_MPIOP_TOTALTIME_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
189
};
190
#undef X
191
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
192

193
194
195
#define F_SYNC(a) \
        Y(a ## _TOTAL_SYNC_TIME) 
#define APMPI_F_MPIOP_SYNCTIME_COUNTERS \
196
	APMPI_MPI_COLL_SYNC \
197
	APMPI_MPI_BLOCKING_COLL \
Sudheer Chunduri's avatar
Sudheer Chunduri committed
198
        Z(APMPI_F_MPIOP_SYNCTIME_NUM_INDICES) 
199
200
/* float counters for the "APMPI" module */
#define X F_SYNC
201
#define V F_SYNC
Sudheer Chunduri's avatar
Sudheer Chunduri committed
202
enum apmpi_f_mpiop_synctime_indices
203
204
205
{
    APMPI_F_MPIOP_SYNCTIME_COUNTERS
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
206
#undef X
207
#undef V
Sudheer Chunduri's avatar
Sudheer Chunduri committed
208

209
        /* aggregate (across all the ranks) per MPI op times  */ 
210
#define APMPI_F_MPI_GLOBAL_COUNTERS \
211
212
	Y(MPI_TOTAL_COMM_TIME) \
	Y(MPI_TOTAL_COMM_SYNC_TIME) \
213
	Z(APMPI_F_MPI_GLOBAL_NUM_INDICES)
214
215
enum apmpi_f_mpi_global_indices
{
216
    APMPI_F_MPI_GLOBAL_COUNTERS
217
218
219
};
#undef Z
#undef Y
Sudheer Chunduri's avatar
Sudheer Chunduri committed
220
221
/* the darshan_apmpi_record structure encompasses the data/counters
 * which would actually be logged to file by Darshan for the AP MPI
Sudheer Chunduri's avatar
Sudheer Chunduri committed
222
223
224
225
226
227
 * module. This example implementation logs the following data for each
 * record:
 *      - a darshan_base_record structure, which contains the record id & rank
 *      - integer I/O counters 
 *      - floating point I/O counters 
 */
Sudheer Chunduri's avatar
Sudheer Chunduri committed
228
struct darshan_apmpi_perf_record
Sudheer Chunduri's avatar
Sudheer Chunduri committed
229
230
{
    struct darshan_base_record base_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
231
    uint64_t counters[APMPI_NUM_INDICES];
Sudheer Chunduri's avatar
Sudheer Chunduri committed
232
233
    double fcounters[APMPI_F_MPIOP_TOTALTIME_NUM_INDICES];
    double fsynccounters[APMPI_F_MPIOP_SYNCTIME_NUM_INDICES];
234
    double fglobalcounters[APMPI_F_MPI_GLOBAL_NUM_INDICES];
235
    char node_name[MPI_MAX_PROCESSOR_NAME];
Sudheer Chunduri's avatar
Sudheer Chunduri committed
236
};
237
238
239
240
struct darshan_apmpi_header_record
{
    struct darshan_base_record base_rec;
    int64_t magic;
241
    uint32_t sync_flag;
242
    double apmpi_f_variance_total_mpitime;
243
    double apmpi_f_variance_total_mpisynctime;
244
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
245

Sudheer Chunduri's avatar
Sudheer Chunduri committed
246
#endif /* __APMPI_LOG_FORMAT_H */