mpidpkt.h 42.9 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef HAVE_MPIDPKT_H
#define HAVE_MPIDPKT_H
9

10 11
#include "oputil.h"

12
#ifdef HAVE_STDINT_H
13
#include <stdint.h>
14 15
#endif
#ifdef HAVE_INTTYPES_H
16
#include <inttypes.h>
17 18
#endif

19 20 21
/* Enable the use of data within the message packet for small messages */
#define USE_EAGER_SHORT
#define MPIDI_EAGER_SHORT_INTS 4
22
/* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */
23 24
#define MPIDI_EAGER_SHORT_SIZE 16

25
/* This is the number of ints that can be carried within an RMA packet */
26
#define MPIDI_RMA_IMMED_BYTES 8
27

28 29 30
/* Union over all types (integer, logical, and multi-language types) that are
   allowed in a CAS operation.  This is used to allocate enough space in the
   packet header for immediate data.  */
31
typedef union {
32
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_;
33 34 35 36 37 38 39 40
    MPIR_OP_TYPE_GROUP(C_INTEGER)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER)
    MPIR_OP_TYPE_GROUP(LOGICAL)
    MPIR_OP_TYPE_GROUP(BYTE)
    MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA)
    MPIR_OP_TYPE_GROUP(BYTE_EXTRA)
41
#undef MPIR_OP_TYPE_MACRO
42 43
} MPIDI_CH3_CAS_Immed_u;

44 45 46 47
/* Union over all types (all predefined types) that are allowed in a
   Fetch-and-op operation.  This can be too large for the packet header, so we
   limit the immediate space in the header to FOP_IMMED_INTS. */

48 49 50
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the fields below */
51
typedef union {
52
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_;
53 54
    MPIR_OP_TYPE_GROUP_ALL_BASIC
    MPIR_OP_TYPE_GROUP_ALL_EXTRA
55
#undef MPIR_OP_TYPE_MACRO
56
} MPIDI_CH3_FOP_Immed_u;
57
/* *INDENT-ON* */
58

59
/*
60
 * Predefined packet types.  This simplifies some of the code.
61
 */
62 63
/* FIXME: Having predefined names makes it harder to add new message types,
   such as different RMA types. */
64
typedef enum {
Xin Zhao's avatar
Xin Zhao committed
65
    MPIDI_CH3_PKT_EAGER_SEND = 0,
66
#if defined(USE_EAGER_SHORT)
67
    MPIDI_CH3_PKT_EAGERSHORT_SEND,
68
#endif /* defined(USE_EAGER_SHORT) */
69
    MPIDI_CH3_PKT_EAGER_SYNC_SEND,      /* FIXME: no sync eager */
70 71 72 73
    MPIDI_CH3_PKT_EAGER_SYNC_ACK,
    MPIDI_CH3_PKT_READY_SEND,
    MPIDI_CH3_PKT_RNDV_REQ_TO_SEND,
    MPIDI_CH3_PKT_RNDV_CLR_TO_SEND,
74
    MPIDI_CH3_PKT_RNDV_SEND,    /* FIXME: should be stream put */
75 76
    MPIDI_CH3_PKT_CANCEL_SEND_REQ,
    MPIDI_CH3_PKT_CANCEL_SEND_RESP,
Xin Zhao's avatar
Xin Zhao committed
77 78
    /* RMA Packets begin here */
    MPIDI_CH3_PKT_PUT,
79
    MPIDI_CH3_PKT_PUT_IMMED,
80 81
    MPIDI_CH3_PKT_GET,
    MPIDI_CH3_PKT_ACCUMULATE,
82
    MPIDI_CH3_PKT_ACCUMULATE_IMMED,
Xin Zhao's avatar
Xin Zhao committed
83
    MPIDI_CH3_PKT_GET_ACCUM,
84
    MPIDI_CH3_PKT_GET_ACCUM_IMMED,
Xin Zhao's avatar
Xin Zhao committed
85
    MPIDI_CH3_PKT_FOP,
86 87
    MPIDI_CH3_PKT_FOP_IMMED,
    MPIDI_CH3_PKT_CAS_IMMED,
Xin Zhao's avatar
Xin Zhao committed
88
    MPIDI_CH3_PKT_GET_RESP,
89
    MPIDI_CH3_PKT_GET_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
90
    MPIDI_CH3_PKT_GET_ACCUM_RESP,
91
    MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
92
    MPIDI_CH3_PKT_FOP_RESP,
93 94
    MPIDI_CH3_PKT_FOP_RESP_IMMED,
    MPIDI_CH3_PKT_CAS_RESP_IMMED,
95
    MPIDI_CH3_PKT_LOCK,
96 97
    MPIDI_CH3_PKT_LOCK_ACK,
    MPIDI_CH3_PKT_LOCK_OP_ACK,
98
    MPIDI_CH3_PKT_UNLOCK,
99
    MPIDI_CH3_PKT_FLUSH,
100
    MPIDI_CH3_PKT_FLUSH_ACK,
101
    MPIDI_CH3_PKT_DECR_AT_COUNTER,
102 103
    /* RMA Packets end here */
    MPIDI_CH3_PKT_FLOW_CNTL_UPDATE,     /* FIXME: Unused */
104
    MPIDI_CH3_PKT_CLOSE,
Wesley Bland's avatar
Wesley Bland committed
105
    MPIDI_CH3_PKT_REVOKE,
Pavan Balaji's avatar
Pavan Balaji committed
106
    MPIDI_CH3_PKT_END_CH3,
107
    /* The channel can define additional types by defining the value
108
     * MPIDI_CH3_PKT_ENUM */
109
#if defined(MPIDI_CH3_PKT_ENUM)
Pavan Balaji's avatar
Pavan Balaji committed
110
    MPIDI_CH3_PKT_ENUM,
111
#endif
Pavan Balaji's avatar
Pavan Balaji committed
112
    MPIDI_CH3_PKT_END_ALL,
113
    MPIDI_CH3_PKT_INVALID = -1  /* forces a signed enum to quash warnings */
Pavan Balaji's avatar
Pavan Balaji committed
114
} MPIDI_CH3_Pkt_type_t;
115

116
/* These flags can be "OR'ed" together */
117 118
typedef enum {
    MPIDI_CH3_PKT_FLAG_NONE = 0,
119 120 121 122 123 124 125 126 127 128 129 130 131
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8,
    MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16,
    MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32,
    MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096,
132 133
    MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192,
    MPIDI_CH3_PKT_FLAG_RMA_STREAM = 16384
134
} MPIDI_CH3_Pkt_flags_t;
135

136
typedef struct MPIDI_CH3_Pkt_send {
137 138
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
139
    MPI_Request sender_req_id;  /* needed for ssend and send cancel */
140 141 142
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
143 144
#endif
} MPIDI_CH3_Pkt_send_t;
145

146
/* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends,
147 148 149 150
   use the same structure but have a different type value. */
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t;
151 152

#if defined(USE_EAGER_SHORT)
153
typedef struct MPIDI_CH3_Pkt_eagershort_send {
154 155 156 157 158 159
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
#endif
160 161
    int data[MPIDI_EAGER_SHORT_INTS];   /* FIXME: Experimental for now */
} MPIDI_CH3_Pkt_eagershort_send_t;
162
#endif /* defined(USE_EAGER_SHORT) */
163

164
typedef struct MPIDI_CH3_Pkt_eager_sync_ack {
165 166
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
167
} MPIDI_CH3_Pkt_eager_sync_ack_t;
168 169 170

typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t;

171
typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send {
172 173 174
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    MPI_Request receiver_req_id;
175
} MPIDI_CH3_Pkt_rndv_clr_to_send_t;
176

177
typedef struct MPIDI_CH3_Pkt_rndv_send {
178 179
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request receiver_req_id;
180
} MPIDI_CH3_Pkt_rndv_send_t;
181

182
typedef struct MPIDI_CH3_Pkt_cancel_send_req {
183 184 185
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
186
} MPIDI_CH3_Pkt_cancel_send_req_t;
187

188
typedef struct MPIDI_CH3_Pkt_cancel_send_resp {
189 190 191
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    int ack;
192
} MPIDI_CH3_Pkt_cancel_send_resp_t;
193

194 195 196
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the field below */
197 198 199
#if defined(MPIDI_CH3_PKT_DEFS)
MPIDI_CH3_PKT_DEFS
#endif
200
/* *INDENT-ON* */
201

202 203
#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_)    \
    {                                                                   \
204 205
        /* This macro returns target_datatype in RMA operation          \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
206
        err_ = MPI_SUCCESS;                                             \
Xin Zhao's avatar
Xin Zhao committed
207
        switch((pkt_).type) {                                           \
208
        case (MPIDI_CH3_PKT_PUT):                                       \
209
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
210
            datatype_ = (pkt_).put.datatype;                            \
211 212
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
Xin Zhao's avatar
Xin Zhao committed
213
            datatype_ = (pkt_).get.datatype;                            \
214 215
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
216
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
217
            datatype_ = (pkt_).accum.datatype;                          \
218
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
219
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
220
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
221
            datatype_ = (pkt_).get_accum.datatype;                      \
Xin Zhao's avatar
Xin Zhao committed
222
            break;                                                      \
223
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
224
            datatype_ = (pkt_).cas.datatype;                            \
225 226
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
227
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
228
            datatype_ = (pkt_).fop.datatype;                            \
229 230
            break;                                                      \
        default:                                                        \
Xin Zhao's avatar
Xin Zhao committed
231
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
232 233 234
        }                                                               \
    }

235 236
#define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_)          \
    {                                                                   \
237 238
        /* This macro returns target_count in RMA operation             \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
239 240 241
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
242
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
243 244 245 246 247 248
            count_ = (pkt_).put.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            count_ = (pkt_).get.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
249
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
250 251 252
            count_ = (pkt_).accum.count;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
253
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
254 255
            count_ = (pkt_).get_accum.count;                            \
            break;                                                      \
256
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
257
        case (MPIDI_CH3_PKT_FOP):                                       \
258
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
259 260 261 262 263 264 265 266 267
            count_ = 1;                                                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_)   \
    {                                                                   \
268 269 270
        /* This macro returns pointer to immed data in RMA operation    \
           packets (PUT, ACC, GACC, FOP, CAS) and RMA response          \
           packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */        \
271 272
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
273 274
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
            immed_data_ = (pkt_).put.info.data;                         \
275
            break;                                                      \
276 277
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
            immed_data_ = (pkt_).accum.info.data;                       \
278
            break;                                                      \
279 280
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            immed_data_ = (pkt_).get_accum.info.data;                   \
281
            break;                                                      \
282 283
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            immed_data_ = (pkt_).fop.info.data;                         \
284
            break;                                                      \
285
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
286 287 288 289
            /* Note that here we return pointer of origin data, not     \
               pointer of compare data. */                              \
            immed_data_ = &((pkt_).cas.origin_data);                    \
            break;                                                      \
290 291
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            immed_data_ = (pkt_).get_resp.info.data;                    \
292
            break;                                                      \
293 294
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            immed_data_ = (pkt_).get_accum_resp.info.data;              \
295
            break;                                                      \
296 297
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            immed_data_ = (pkt_).fop_resp.info.data;                    \
298
            break;                                                      \
299 300
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            immed_data_ = &((pkt_).cas_resp.info.data);                 \
301 302 303 304 305 306
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

Xin Zhao's avatar
Xin Zhao committed
307 308
#define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_)                 \
    {                                                                   \
309 310 311 312 313
        /* This macro returns flags in RMA operation packets (PUT, GET, \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
Xin Zhao's avatar
Xin Zhao committed
314 315 316
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
317
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
318 319 320 321 322 323
            flags_ = (pkt_).put.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            flags_ = (pkt_).get.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
324
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
325 326 327
            flags_ = (pkt_).accum.flags;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
328
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
329 330
            flags_ = (pkt_).get_accum.flags;                            \
            break;                                                      \
331
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
332 333 334
            flags_ = (pkt_).cas.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
335
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
336 337
            flags_ = (pkt_).fop.flags;                                  \
            break;                                                      \
338
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
339
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
340 341 342
            flags_ = (pkt_).get_resp.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
343
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
344 345 346
            flags_ = (pkt_).get_accum_resp.flags;                       \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
347
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
348 349
            flags_ = (pkt_).fop_resp.flags;                             \
            break;                                                      \
350
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
351 352
            flags_ = (pkt_).cas_resp.flags;                             \
            break;                                                      \
353 354 355
        case (MPIDI_CH3_PKT_LOCK):                                      \
            flags_ = (pkt_).lock.flags;                                 \
            break;                                                      \
356 357 358 359 360 361 362 363 364
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            flags_ = (pkt_).unlock.flags;                               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            flags_ = (pkt_).lock_ack.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            flags_ = (pkt_).lock_op_ack.flags;                          \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
365 366 367
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
    }

#define MPIDI_CH3_PKT_RMA_GET_OP(pkt_, op_, err_)                       \
    {                                                                   \
        /* This macro returns op in RMA operation packets (ACC, GACC,   \
           FOP) */                                                      \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
            op_ = (pkt_).accum.op;                                      \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            op_ = (pkt_).get_accum.op;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            op_ = (pkt_).fop.op;                                        \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
Xin Zhao's avatar
Xin Zhao committed
391 392
    }

393 394
#define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_)                       \
    {                                                                   \
395 396 397 398 399
        /* This macro erases flags in RMA operation packets (PUT, GET,  \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
400 401 402
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
403
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
404 405 406 407 408 409
            (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
410
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
411 412 413
            (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
414
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
415 416
            (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE;           \
            break;                                                      \
417
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
418 419 420
            (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
421
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
422 423
            (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
424
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
425
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
426 427 428
            (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
429
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
430 431 432
            (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;      \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
433
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
434 435
            (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
436
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
437 438
            (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
439 440 441
        case (MPIDI_CH3_PKT_LOCK):                                      \
            (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE;                \
            break;                                                      \
442 443 444 445 446 447 448 449 450
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;         \
            break;                                                      \
451 452 453 454 455 456 457
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_)   \
    {                                                                   \
458 459 460 461 462
        /* This macro returns source_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation       \
           response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP),  \
           RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control   \
           response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */      \
463 464 465
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
466
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
467 468 469
            win_hdl_ = (pkt_).put.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
470
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
471 472 473 474 475
            win_hdl_ = (pkt_).accum.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.source_win_handle;                   \
            break;                                                      \
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.source_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            win_hdl_ = (pkt_).lock_ack.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            win_hdl_ = (pkt_).lock_op_ack.source_win_handle;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH_ACK):                                 \
            win_hdl_ = (pkt_).flush_ack.source_win_handle;              \
            break;                                                      \
491 492 493 494 495
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

496
#define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_)   \
Xin Zhao's avatar
Xin Zhao committed
497
    {                                                                   \
498 499 500
        /* This macro returns target_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control      \
           packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */                \
Xin Zhao's avatar
Xin Zhao committed
501 502 503
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
504
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
505
            win_hdl_ = (pkt_).put.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
506 507
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
508
            win_hdl_ = (pkt_).get.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
509 510
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
511
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
512
            win_hdl_ = (pkt_).accum.target_win_handle;                  \
Xin Zhao's avatar
Xin Zhao committed
513 514
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
515
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
516
            win_hdl_ = (pkt_).get_accum.target_win_handle;              \
Xin Zhao's avatar
Xin Zhao committed
517
            break;                                                      \
518
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
519
            win_hdl_ = (pkt_).cas.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
520 521
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
522
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
523
            win_hdl_ = (pkt_).fop.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
524
            break;                                                      \
525 526 527
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.target_win_handle;                   \
            break;                                                      \
528 529 530 531 532 533 534 535 536
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.target_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.target_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_DECR_AT_COUNTER):                           \
            win_hdl_ = (pkt_).decr_at_cnt.target_win_handle;            \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
537 538 539 540 541 542 543
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \
    {                                                                   \
544 545
        /* This macro sets dataloop_size in RMA operation packets       \
           (PUT, GET, ACC, GACC) */                                     \
Xin Zhao's avatar
Xin Zhao committed
546 547 548
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
549
            (pkt_).put.info.dataloop_size = (dataloop_size_);           \
Xin Zhao's avatar
Xin Zhao committed
550 551
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
552
            (pkt_).get.info.dataloop_size = (dataloop_size_);           \
Xin Zhao's avatar
Xin Zhao committed
553 554
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
555
            (pkt_).accum.info.dataloop_size = (dataloop_size_);         \
Xin Zhao's avatar
Xin Zhao committed
556 557
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
558
            (pkt_).get_accum.info.dataloop_size = (dataloop_size_);     \
559 560 561 562 563 564
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
#define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_)  \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_GET):                                       \
            request_hdl_ = (pkt_).get.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            request_hdl_ = (pkt_).get_accum.request_handle;             \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
            request_hdl_ = (pkt_).cas.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            request_hdl_ = (pkt_).fop.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            request_hdl_ = (pkt_).get_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            request_hdl_ = (pkt_).get_accum_resp.request_handle;        \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            request_hdl_ = (pkt_).fop_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            request_hdl_ = (pkt_).cas_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            request_hdl_ = (pkt_).lock.request_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            request_hdl_ = (pkt_).lock_ack.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            request_hdl_ = (pkt_).lock_op_ack.request_handle;           \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

612
typedef struct MPIDI_CH3_Pkt_put {
613
    MPIDI_CH3_Pkt_type_t type;
614
    MPIDI_CH3_Pkt_flags_t flags;
615 616 617
    void *addr;
    int count;
    MPI_Datatype datatype;
618 619
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
620
    union {
621
        int dataloop_size;
622 623
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
624 625 626
} MPIDI_CH3_Pkt_put_t;

typedef struct MPIDI_CH3_Pkt_get {
627
    MPIDI_CH3_Pkt_type_t type;
628
    MPIDI_CH3_Pkt_flags_t flags;
629 630 631
    void *addr;
    int count;
    MPI_Datatype datatype;
632
    struct {
633
        int dataloop_size;      /* for derived datatypes */
634
    } info;
635
    MPI_Request request_handle;
636
    MPI_Win target_win_handle;
637 638
} MPIDI_CH3_Pkt_get_t;

639 640 641 642 643 644
typedef struct MPIDI_CH3_Pkt_get_resp {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
645
    /* Followings are to piggyback IMMED data */
646 647
    struct {
        /* note that we use struct here in order
648 649
         * to consistently access data
         * by "pkt->info.data". */
650 651
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
652 653
} MPIDI_CH3_Pkt_get_resp_t;

654
typedef struct MPIDI_CH3_Pkt_accum {
655
    MPIDI_CH3_Pkt_type_t type;
656
    MPIDI_CH3_Pkt_flags_t flags;
657 658 659 660
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
661 662
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
663
    union {
664
        int dataloop_size;
665 666
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
667 668
} MPIDI_CH3_Pkt_accum_t;

Xin Zhao's avatar
Xin Zhao committed
669 670 671
typedef struct MPIDI_CH3_Pkt_get_accum {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
672
    MPI_Request request_handle; /* For get_accumulate response */
Xin Zhao's avatar
Xin Zhao committed
673 674 675 676
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
677
    MPI_Win target_win_handle;
678
    union {
679
        int dataloop_size;
680 681
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
Xin Zhao's avatar
Xin Zhao committed
682 683
} MPIDI_CH3_Pkt_get_accum_t;

684
typedef struct MPIDI_CH3_Pkt_get_accum_resp {
685
    MPIDI_CH3_Pkt_type_t type;
686
    MPI_Request request_handle;
687 688 689
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
690
    /* Followings are to piggyback IMMED data */
691 692
    struct {
        /* note that we use struct here in order
693 694
         * to consistently access data
         * by "pkt->info.data". */
695 696
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
697
} MPIDI_CH3_Pkt_get_accum_resp_t;
698

699
typedef struct MPIDI_CH3_Pkt_cas {
700
    MPIDI_CH3_Pkt_type_t type;
701
    MPIDI_CH3_Pkt_flags_t flags;
702
    MPI_Datatype datatype;
703
    void *addr;
704
    MPI_Request request_handle;
705
    MPI_Win target_win_handle;
706 707
    MPIDI_CH3_CAS_Immed_u origin_data;
    MPIDI_CH3_CAS_Immed_u compare_data;
708
} MPIDI_CH3_Pkt_cas_t;
709

710
typedef struct MPIDI_CH3_Pkt_cas_resp {
711 712
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
713 714
    struct {
        /* note that we use struct here in order
715 716
         * to consistently access data
         * by "pkt->info.data". */
717 718
        MPIDI_CH3_CAS_Immed_u data;
    } info;
719
    /* followings are used to decrement ack_counter at orign */
720
    int target_rank;
721 722
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_cas_resp_t;
723

724
typedef struct MPIDI_CH3_Pkt_fop {
725
    MPIDI_CH3_Pkt_type_t type;
726
    MPIDI_CH3_Pkt_flags_t flags;
727 728 729
    MPI_Datatype datatype;
    void *addr;
    MPI_Op op;
730
    MPI_Request request_handle;
731
    MPI_Win target_win_handle;
732 733
    struct {
        /* note that we use struct here in order
734 735
         * to consistently access data
         * by "pkt->info.data". */
736 737
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
738
} MPIDI_CH3_Pkt_fop_t;
739

740
typedef struct MPIDI_CH3_Pkt_fop_resp {
741 742
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
743 744
    struct {
        /* note that we use struct here in order
745 746
         * to consistently access data
         * by "pkt->info.data". */
747 748
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
749
    /* followings are used to decrement ack_counter at orign */
750
    int target_rank;
751 752
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_fop_resp_t;
753

754
typedef struct MPIDI_CH3_Pkt_lock {
755
    MPIDI_CH3_Pkt_type_t type;
756
    MPIDI_CH3_Pkt_flags_t flags;
757
    MPI_Win target_win_handle;
758
    /* Note that either source_win_handle
759 760 761 762 763
     * or request_handle will be used. Here
     * we need both of them because PUT/GET
     * may be converted to LOCK packet,
     * PUT has source_win_handle area and
     * GET has request_handle area. */
764
    MPI_Win source_win_handle;
765
    MPI_Request request_handle;
766
} MPIDI_CH3_Pkt_lock_t;
767

Xin Zhao's avatar
Xin Zhao committed
768 769 770 771
typedef struct MPIDI_CH3_Pkt_unlock {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
772
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
773 774 775 776 777 778 779 780
} MPIDI_CH3_Pkt_unlock_t;

typedef struct MPIDI_CH3_Pkt_flush {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
} MPIDI_CH3_Pkt_flush_t;

781
typedef struct MPIDI_CH3_Pkt_lock_ack {
782 783
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
784
    /* note that either source_win_handle
785
     * or request_handle is used. */
786
    MPI_Win source_win_handle;
787
    MPI_Request request_handle;
788
    int target_rank;
789 790 791 792 793
} MPIDI_CH3_Pkt_lock_ack_t;

typedef struct MPIDI_CH3_Pkt_lock_op_ack {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
794
    /* note that either source_win_handle
795
     * or request_handle is used. */
796
    MPI_Win source_win_handle;
797
    MPI_Request request_handle;
798 799
    int target_rank;
} MPIDI_CH3_Pkt_lock_op_ack_t;
800

Xin Zhao's avatar
Xin Zhao committed
801 802
typedef struct MPIDI_CH3_Pkt_flush_ack {
    MPIDI_CH3_Pkt_type_t type;
803
    MPI_Win source_win_handle;
804
    int target_rank;
805
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
806
} MPIDI_CH3_Pkt_flush_ack_t;
807

808 809 810
typedef struct MPIDI_CH3_Pkt_decr_at_counter {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
811 812
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
813
} MPIDI_CH3_Pkt_decr_at_counter_t;
814

815
typedef struct MPIDI_CH3_Pkt_close {
816 817
    MPIDI_CH3_Pkt_type_t type;
    int ack;
818
} MPIDI_CH3_Pkt_close_t;
819

820
typedef struct MPIDI_CH3_Pkt_revoke {
Wesley Bland's avatar
Wesley Bland committed
821 822
    MPIDI_CH3_Pkt_type_t type;
    MPIR_Context_id_t revoked_comm;
823
} MPIDI_CH3_Pkt_revoke_t;
Wesley Bland's avatar
Wesley Bland committed
824

825
typedef union MPIDI_CH3_Pkt {
826 827
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_eager_send_t eager_send;
828
#if defined(USE_EAGER_SHORT)
829
    MPIDI_CH3_Pkt_eagershort_send_t eagershort_send;
830
#endif                          /* defined(USE_EAGER_SHORT) */
831 832 833 834 835 836 837 838 839 840
    MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send;
    MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack;
    MPIDI_CH3_Pkt_eager_send_t ready_send;
    MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send;
    MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send;
    MPIDI_CH3_Pkt_rndv_send_t rndv_send;
    MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req;
    MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp;
    MPIDI_CH3_Pkt_put_t put;
    MPIDI_CH3_Pkt_get_t get;
841
    MPIDI_CH3_Pkt_get_resp_t get_resp;
842
    MPIDI_CH3_Pkt_accum_t accum;
Xin Zhao's avatar
Xin Zhao committed
843
    MPIDI_CH3_Pkt_get_accum_t get_accum;
844
    MPIDI_CH3_Pkt_lock_t lock;
845 846
    MPIDI_CH3_Pkt_lock_ack_t lock_ack;
    MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack;
847
    MPIDI_CH3_Pkt_unlock_t unlock;
848
    MPIDI_CH3_Pkt_flush_t flush;
849
    MPIDI_CH3_Pkt_flush_ack_t flush_ack;
850
    MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt;
851
    MPIDI_CH3_Pkt_close_t close;
852 853 854 855 856
    MPIDI_CH3_Pkt_cas_t cas;
    MPIDI_CH3_Pkt_cas_resp_t cas_resp;
    MPIDI_CH3_Pkt_fop_t fop;
    MPIDI_CH3_Pkt_fop_resp_t fop_resp;
    MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp;
Wesley Bland's avatar
Wesley Bland committed