mpidpkt.h 42.6 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef HAVE_MPIDPKT_H
#define HAVE_MPIDPKT_H
9

10 11
#include "oputil.h"

12
#ifdef HAVE_STDINT_H
13
#include <stdint.h>
14 15
#endif
#ifdef HAVE_INTTYPES_H
16
#include <inttypes.h>
17 18
#endif

19 20 21
/* Enable the use of data within the message packet for small messages */
#define USE_EAGER_SHORT
#define MPIDI_EAGER_SHORT_INTS 4
22
/* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */
23 24
#define MPIDI_EAGER_SHORT_SIZE 16

25
/* This is the number of ints that can be carried within an RMA packet */
26
#define MPIDI_RMA_IMMED_BYTES 8
27

28 29 30
/* Union over all types (integer, logical, and multi-language types) that are
   allowed in a CAS operation.  This is used to allocate enough space in the
   packet header for immediate data.  */
31
typedef union {
32
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_;
33 34 35 36 37 38 39 40
    MPIR_OP_TYPE_GROUP(C_INTEGER)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER)
    MPIR_OP_TYPE_GROUP(LOGICAL)
    MPIR_OP_TYPE_GROUP(BYTE)
    MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA)
    MPIR_OP_TYPE_GROUP(BYTE_EXTRA)
41
#undef MPIR_OP_TYPE_MACRO
42 43
} MPIDI_CH3_CAS_Immed_u;

44 45 46 47
/* Union over all types (all predefined types) that are allowed in a
   Fetch-and-op operation.  This can be too large for the packet header, so we
   limit the immediate space in the header to FOP_IMMED_INTS. */

48 49 50
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the fields below */
51
typedef union {
52
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_;
53 54
    MPIR_OP_TYPE_GROUP_ALL_BASIC
    MPIR_OP_TYPE_GROUP_ALL_EXTRA
55
#undef MPIR_OP_TYPE_MACRO
56
} MPIDI_CH3_FOP_Immed_u;
57
/* *INDENT-ON* */
58

59
/*
60
 * Predefined packet types.  This simplifies some of the code.
61
 */
62 63
/* FIXME: Having predefined names makes it harder to add new message types,
   such as different RMA types. */
64
typedef enum {
Xin Zhao's avatar
Xin Zhao committed
65
    MPIDI_CH3_PKT_EAGER_SEND = 0,
66
#if defined(USE_EAGER_SHORT)
67
    MPIDI_CH3_PKT_EAGERSHORT_SEND,
68
#endif /* defined(USE_EAGER_SHORT) */
69
    MPIDI_CH3_PKT_EAGER_SYNC_SEND,      /* FIXME: no sync eager */
70 71 72 73
    MPIDI_CH3_PKT_EAGER_SYNC_ACK,
    MPIDI_CH3_PKT_READY_SEND,
    MPIDI_CH3_PKT_RNDV_REQ_TO_SEND,
    MPIDI_CH3_PKT_RNDV_CLR_TO_SEND,
74
    MPIDI_CH3_PKT_RNDV_SEND,    /* FIXME: should be stream put */
75 76
    MPIDI_CH3_PKT_CANCEL_SEND_REQ,
    MPIDI_CH3_PKT_CANCEL_SEND_RESP,
Xin Zhao's avatar
Xin Zhao committed
77 78
    /* RMA Packets begin here */
    MPIDI_CH3_PKT_PUT,
79
    MPIDI_CH3_PKT_PUT_IMMED,
80 81
    MPIDI_CH3_PKT_GET,
    MPIDI_CH3_PKT_ACCUMULATE,
82
    MPIDI_CH3_PKT_ACCUMULATE_IMMED,
Xin Zhao's avatar
Xin Zhao committed
83
    MPIDI_CH3_PKT_GET_ACCUM,
84
    MPIDI_CH3_PKT_GET_ACCUM_IMMED,
Xin Zhao's avatar
Xin Zhao committed
85
    MPIDI_CH3_PKT_FOP,
86 87
    MPIDI_CH3_PKT_FOP_IMMED,
    MPIDI_CH3_PKT_CAS_IMMED,
Xin Zhao's avatar
Xin Zhao committed
88
    MPIDI_CH3_PKT_GET_RESP,
89
    MPIDI_CH3_PKT_GET_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
90
    MPIDI_CH3_PKT_GET_ACCUM_RESP,
91
    MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
92
    MPIDI_CH3_PKT_FOP_RESP,
93 94
    MPIDI_CH3_PKT_FOP_RESP_IMMED,
    MPIDI_CH3_PKT_CAS_RESP_IMMED,
95
    MPIDI_CH3_PKT_LOCK,
96 97
    MPIDI_CH3_PKT_LOCK_ACK,
    MPIDI_CH3_PKT_LOCK_OP_ACK,
98
    MPIDI_CH3_PKT_UNLOCK,
99
    MPIDI_CH3_PKT_FLUSH,
100
    MPIDI_CH3_PKT_FLUSH_ACK,
101
    MPIDI_CH3_PKT_DECR_AT_COUNTER,
102 103
    /* RMA Packets end here */
    MPIDI_CH3_PKT_FLOW_CNTL_UPDATE,     /* FIXME: Unused */
104
    MPIDI_CH3_PKT_CLOSE,
Wesley Bland's avatar
Wesley Bland committed
105
    MPIDI_CH3_PKT_REVOKE,
Pavan Balaji's avatar
Pavan Balaji committed
106
    MPIDI_CH3_PKT_END_CH3,
107
    /* The channel can define additional types by defining the value
108
     * MPIDI_CH3_PKT_ENUM */
109
#if defined(MPIDI_CH3_PKT_ENUM)
Pavan Balaji's avatar
Pavan Balaji committed
110
    MPIDI_CH3_PKT_ENUM,
111
#endif
Pavan Balaji's avatar
Pavan Balaji committed
112
    MPIDI_CH3_PKT_END_ALL,
113
    MPIDI_CH3_PKT_INVALID = -1  /* forces a signed enum to quash warnings */
Pavan Balaji's avatar
Pavan Balaji committed
114
} MPIDI_CH3_Pkt_type_t;
115

116
/* These flags can be "OR'ed" together */
117 118
typedef enum {
    MPIDI_CH3_PKT_FLAG_NONE = 0,
119 120 121 122 123 124 125 126 127 128 129 130 131 132
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8,
    MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16,
    MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32,
    MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096,
    MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192
133
} MPIDI_CH3_Pkt_flags_t;
134

135
typedef struct MPIDI_CH3_Pkt_send {
136 137
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
138
    MPI_Request sender_req_id;  /* needed for ssend and send cancel */
139 140 141
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
142 143
#endif
} MPIDI_CH3_Pkt_send_t;
144

145
/* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends,
146 147 148 149
   use the same structure but have a different type value. */
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t;
150 151

#if defined(USE_EAGER_SHORT)
152
typedef struct MPIDI_CH3_Pkt_eagershort_send {
153 154 155 156 157 158
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
#endif
159 160
    int data[MPIDI_EAGER_SHORT_INTS];   /* FIXME: Experimental for now */
} MPIDI_CH3_Pkt_eagershort_send_t;
161
#endif /* defined(USE_EAGER_SHORT) */
162

163
typedef struct MPIDI_CH3_Pkt_eager_sync_ack {
164 165
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
166
} MPIDI_CH3_Pkt_eager_sync_ack_t;
167 168 169

typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t;

170
typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send {
171 172 173
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    MPI_Request receiver_req_id;
174
} MPIDI_CH3_Pkt_rndv_clr_to_send_t;
175

176
typedef struct MPIDI_CH3_Pkt_rndv_send {
177 178
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request receiver_req_id;
179
} MPIDI_CH3_Pkt_rndv_send_t;
180

181
typedef struct MPIDI_CH3_Pkt_cancel_send_req {
182 183 184
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
185
} MPIDI_CH3_Pkt_cancel_send_req_t;
186

187
typedef struct MPIDI_CH3_Pkt_cancel_send_resp {
188 189 190
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    int ack;
191
} MPIDI_CH3_Pkt_cancel_send_resp_t;
192

193 194 195
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the field below */
196 197 198
#if defined(MPIDI_CH3_PKT_DEFS)
MPIDI_CH3_PKT_DEFS
#endif
199
/* *INDENT-ON* */
200

201 202
#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_)    \
    {                                                                   \
203 204
        /* This macro returns target_datatype in RMA operation          \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
205
        err_ = MPI_SUCCESS;                                             \
Xin Zhao's avatar
Xin Zhao committed
206
        switch((pkt_).type) {                                           \
207
        case (MPIDI_CH3_PKT_PUT):                                       \
208
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
209
            datatype_ = (pkt_).put.datatype;                            \
210 211
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
Xin Zhao's avatar
Xin Zhao committed
212
            datatype_ = (pkt_).get.datatype;                            \
213 214
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
215
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
216
            datatype_ = (pkt_).accum.datatype;                          \
217
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
218
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
219
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
220
            datatype_ = (pkt_).get_accum.datatype;                      \
Xin Zhao's avatar
Xin Zhao committed
221
            break;                                                      \
222
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
223
            datatype_ = (pkt_).cas.datatype;                            \
224 225
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
226
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
227
            datatype_ = (pkt_).fop.datatype;                            \
228 229
            break;                                                      \
        default:                                                        \
Xin Zhao's avatar
Xin Zhao committed
230
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
231 232 233
        }                                                               \
    }

234 235
#define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_)          \
    {                                                                   \
236 237
        /* This macro returns target_count in RMA operation             \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
238 239 240
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
241
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
242 243 244 245 246 247
            count_ = (pkt_).put.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            count_ = (pkt_).get.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
248
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
249 250 251
            count_ = (pkt_).accum.count;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
252
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
253 254
            count_ = (pkt_).get_accum.count;                            \
            break;                                                      \
255
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
256
        case (MPIDI_CH3_PKT_FOP):                                       \
257
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
258 259 260 261 262 263 264 265 266
            count_ = 1;                                                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_)   \
    {                                                                   \
267 268 269
        /* This macro returns pointer to immed data in RMA operation    \
           packets (PUT, ACC, GACC, FOP, CAS) and RMA response          \
           packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */        \
270 271
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
272 273
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
            immed_data_ = (pkt_).put.info.data;                         \
274
            break;                                                      \
275 276
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
            immed_data_ = (pkt_).accum.info.data;                       \
277
            break;                                                      \
278 279
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            immed_data_ = (pkt_).get_accum.info.data;                   \
280
            break;                                                      \
281 282
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            immed_data_ = (pkt_).fop.info.data;                         \
283
            break;                                                      \
284
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
285 286 287 288
            /* Note that here we return pointer of origin data, not     \
               pointer of compare data. */                              \
            immed_data_ = &((pkt_).cas.origin_data);                    \
            break;                                                      \
289 290
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            immed_data_ = (pkt_).get_resp.info.data;                    \
291
            break;                                                      \
292 293
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            immed_data_ = (pkt_).get_accum_resp.info.data;              \
294
            break;                                                      \
295 296
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            immed_data_ = (pkt_).fop_resp.info.data;                    \
297
            break;                                                      \
298 299
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            immed_data_ = &((pkt_).cas_resp.info.data);                 \
300 301 302 303 304 305
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

Xin Zhao's avatar
Xin Zhao committed
306 307
#define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_)                 \
    {                                                                   \
308 309 310 311 312
        /* This macro returns flags in RMA operation packets (PUT, GET, \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
Xin Zhao's avatar
Xin Zhao committed
313 314 315
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
316
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
317 318 319 320 321 322
            flags_ = (pkt_).put.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            flags_ = (pkt_).get.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
323
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
324 325 326
            flags_ = (pkt_).accum.flags;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
327
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
328 329
            flags_ = (pkt_).get_accum.flags;                            \
            break;                                                      \
330
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
331 332 333
            flags_ = (pkt_).cas.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
334
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
335 336
            flags_ = (pkt_).fop.flags;                                  \
            break;                                                      \
337
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
338
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
339 340 341
            flags_ = (pkt_).get_resp.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
342
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
343 344 345
            flags_ = (pkt_).get_accum_resp.flags;                       \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
346
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
347 348
            flags_ = (pkt_).fop_resp.flags;                             \
            break;                                                      \
349
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
350 351
            flags_ = (pkt_).cas_resp.flags;                             \
            break;                                                      \
352 353 354
        case (MPIDI_CH3_PKT_LOCK):                                      \
            flags_ = (pkt_).lock.flags;                                 \
            break;                                                      \
355 356 357 358 359 360 361 362 363
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            flags_ = (pkt_).unlock.flags;                               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            flags_ = (pkt_).lock_ack.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            flags_ = (pkt_).lock_op_ack.flags;                          \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
364 365 366 367 368
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

369 370
#define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_)                       \
    {                                                                   \
371 372 373 374 375
        /* This macro erases flags in RMA operation packets (PUT, GET,  \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
376 377 378
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
379
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
380 381 382 383 384 385
            (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
386
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
387 388 389
            (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
390
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
391 392
            (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE;           \
            break;                                                      \
393
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
394 395 396
            (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
397
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
398 399
            (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
400
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
401
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
402 403 404
            (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
405
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
406 407 408
            (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;      \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
409
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
410 411
            (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
412
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
413 414
            (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
415 416 417
        case (MPIDI_CH3_PKT_LOCK):                                      \
            (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE;                \
            break;                                                      \
418 419 420 421 422 423 424 425 426
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;         \
            break;                                                      \
427 428 429 430 431 432 433
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_)   \
    {                                                                   \
434 435 436 437 438
        /* This macro returns source_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation       \
           response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP),  \
           RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control   \
           response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */      \
439 440 441
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
442
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
443 444 445
            win_hdl_ = (pkt_).put.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
446
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
447 448 449 450 451
            win_hdl_ = (pkt_).accum.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.source_win_handle;                   \
            break;                                                      \
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.source_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            win_hdl_ = (pkt_).lock_ack.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            win_hdl_ = (pkt_).lock_op_ack.source_win_handle;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH_ACK):                                 \
            win_hdl_ = (pkt_).flush_ack.source_win_handle;              \
            break;                                                      \
467 468 469 470 471
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

472
#define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_)   \
Xin Zhao's avatar
Xin Zhao committed
473
    {                                                                   \
474 475 476
        /* This macro returns target_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control      \
           packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */                \
Xin Zhao's avatar
Xin Zhao committed
477 478 479
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
480
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
481
            win_hdl_ = (pkt_).put.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
482 483
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
484
            win_hdl_ = (pkt_).get.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
485 486
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
487
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
488
            win_hdl_ = (pkt_).accum.target_win_handle;                  \
Xin Zhao's avatar
Xin Zhao committed
489 490
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
491
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
492
            win_hdl_ = (pkt_).get_accum.target_win_handle;              \
Xin Zhao's avatar
Xin Zhao committed
493
            break;                                                      \
494
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
495
            win_hdl_ = (pkt_).cas.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
496 497
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
498
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
499
            win_hdl_ = (pkt_).fop.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
500
            break;                                                      \
501 502 503
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.target_win_handle;                   \
            break;                                                      \
504 505 506 507 508 509 510 511 512
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.target_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.target_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_DECR_AT_COUNTER):                           \
            win_hdl_ = (pkt_).decr_at_cnt.target_win_handle;            \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
513 514 515 516 517 518 519
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \
    {                                                                   \
520 521
        /* This macro sets dataloop_size in RMA operation packets       \
           (PUT, GET, ACC, GACC) */                                     \
Xin Zhao's avatar
Xin Zhao committed
522 523 524
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
525
            (pkt_).put.info.metadata.dataloop_size = (dataloop_size_);  \
Xin Zhao's avatar
Xin Zhao committed
526 527
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
528
            (pkt_).get.info.metadata.dataloop_size = (dataloop_size_);  \
Xin Zhao's avatar
Xin Zhao committed
529 530
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
531
            (pkt_).accum.info.metadata.dataloop_size = (dataloop_size_); \
Xin Zhao's avatar
Xin Zhao committed
532 533
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
            (pkt_).get_accum.info.metadata.dataloop_size = (dataloop_size_); \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_STREAM_OFFSET(pkt_, stream_offset_, err_) \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            (stream_offset_) = (pkt_).accum.info.metadata.stream_offset; \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            (stream_offset_) = (pkt_).get_accum.info.metadata.stream_offset; \
550 551 552 553 554 555
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
#define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_)  \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_GET):                                       \
            request_hdl_ = (pkt_).get.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            request_hdl_ = (pkt_).get_accum.request_handle;             \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
            request_hdl_ = (pkt_).cas.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            request_hdl_ = (pkt_).fop.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            request_hdl_ = (pkt_).get_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            request_hdl_ = (pkt_).get_accum_resp.request_handle;        \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            request_hdl_ = (pkt_).fop_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            request_hdl_ = (pkt_).cas_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            request_hdl_ = (pkt_).lock.request_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            request_hdl_ = (pkt_).lock_ack.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            request_hdl_ = (pkt_).lock_op_ack.request_handle;           \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

603
typedef struct MPIDI_CH3_Pkt_put {
604
    MPIDI_CH3_Pkt_type_t type;
605
    MPIDI_CH3_Pkt_flags_t flags;
606 607 608
    void *addr;
    int count;
    MPI_Datatype datatype;
609 610
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
611
    union {
612 613 614 615 616 617
        /* note that we use struct here in order
         * to consistently access dataloop_size
         * by "pkt->info.metadata.dataloop_size". */
        struct {
            int dataloop_size;
        } metadata;
618 619
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
620 621 622
} MPIDI_CH3_Pkt_put_t;

typedef struct MPIDI_CH3_Pkt_get {
623
    MPIDI_CH3_Pkt_type_t type;
624
    MPIDI_CH3_Pkt_flags_t flags;
625 626 627
    void *addr;
    int count;
    MPI_Datatype datatype;
628 629
    struct {
        /* note that we use struct here in order
630
         * to consistently access dataloop_size
631 632 633 634
         * by "pkt->info.metadata.dataloop_size". */
        struct {
            int dataloop_size;  /* for derived datatypes */
        } metadata;
635
    } info;
636
    MPI_Request request_handle;
637
    MPI_Win target_win_handle;
638 639
} MPIDI_CH3_Pkt_get_t;

640 641 642 643 644 645
typedef struct MPIDI_CH3_Pkt_get_resp {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
646
    /* Followings are to piggyback IMMED data */
647 648
    struct {
        /* note that we use struct here in order
649 650
         * to consistently access data
         * by "pkt->info.data". */
651 652
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
653 654
} MPIDI_CH3_Pkt_get_resp_t;

655
typedef struct MPIDI_CH3_Pkt_accum {
656
    MPIDI_CH3_Pkt_type_t type;
657
    MPIDI_CH3_Pkt_flags_t flags;
658 659 660 661
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
662 663
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
664
    union {
665 666 667 668
        struct {
            int dataloop_size;
            MPI_Aint stream_offset;
        } metadata;
669 670
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
671 672
} MPIDI_CH3_Pkt_accum_t;

Xin Zhao's avatar
Xin Zhao committed
673 674 675
typedef struct MPIDI_CH3_Pkt_get_accum {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
676
    MPI_Request request_handle; /* For get_accumulate response */
Xin Zhao's avatar
Xin Zhao committed
677 678 679 680
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
681
    MPI_Win target_win_handle;
682
    union {
683 684 685 686
        struct {
            int dataloop_size;
            MPI_Aint stream_offset;
        } metadata;
687 688
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
Xin Zhao's avatar
Xin Zhao committed
689 690
} MPIDI_CH3_Pkt_get_accum_t;

691
typedef struct MPIDI_CH3_Pkt_get_accum_resp {
692
    MPIDI_CH3_Pkt_type_t type;
693
    MPI_Request request_handle;
694 695 696
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
697
    /* Followings are to piggyback IMMED data */
698 699
    struct {
        /* note that we use struct here in order
700 701
         * to consistently access data
         * by "pkt->info.data". */
702 703
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
704
} MPIDI_CH3_Pkt_get_accum_resp_t;
705

706
typedef struct MPIDI_CH3_Pkt_cas {
707
    MPIDI_CH3_Pkt_type_t type;
708
    MPIDI_CH3_Pkt_flags_t flags;
709
    MPI_Datatype datatype;
710
    void *addr;
711
    MPI_Request request_handle;
712
    MPI_Win target_win_handle;
713 714
    MPIDI_CH3_CAS_Immed_u origin_data;
    MPIDI_CH3_CAS_Immed_u compare_data;
715
} MPIDI_CH3_Pkt_cas_t;
716

717
typedef struct MPIDI_CH3_Pkt_cas_resp {
718 719
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
720 721
    struct {
        /* note that we use struct here in order
722 723
         * to consistently access data
         * by "pkt->info.data". */
724 725
        MPIDI_CH3_CAS_Immed_u data;
    } info;
726
    /* followings are used to decrement ack_counter at orign */
727
    int target_rank;
728 729
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_cas_resp_t;
730

731
typedef struct MPIDI_CH3_Pkt_fop {
732
    MPIDI_CH3_Pkt_type_t type;
733
    MPIDI_CH3_Pkt_flags_t flags;
734 735 736
    MPI_Datatype datatype;
    void *addr;
    MPI_Op op;
737
    MPI_Request request_handle;
738
    MPI_Win target_win_handle;
739 740
    struct {
        /* note that we use struct here in order
741 742
         * to consistently access data
         * by "pkt->info.data". */
743 744
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
745
} MPIDI_CH3_Pkt_fop_t;
746

747
typedef struct MPIDI_CH3_Pkt_fop_resp {
748 749
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
750 751
    struct {
        /* note that we use struct here in order
752 753
         * to consistently access data
         * by "pkt->info.data". */
754 755
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
756
    /* followings are used to decrement ack_counter at orign */
757
    int target_rank;
758 759
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_fop_resp_t;
760

761
typedef struct MPIDI_CH3_Pkt_lock {
762
    MPIDI_CH3_Pkt_type_t type;
763
    MPIDI_CH3_Pkt_flags_t flags;
764
    MPI_Win target_win_handle;
765
    /* Note that either source_win_handle
766 767 768 769 770
     * or request_handle will be used. Here
     * we need both of them because PUT/GET
     * may be converted to LOCK packet,
     * PUT has source_win_handle area and
     * GET has request_handle area. */
771
    MPI_Win source_win_handle;
772
    MPI_Request request_handle;
773
} MPIDI_CH3_Pkt_lock_t;
774

Xin Zhao's avatar
Xin Zhao committed
775 776 777 778
typedef struct MPIDI_CH3_Pkt_unlock {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
779
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
780 781 782 783 784 785 786 787
} MPIDI_CH3_Pkt_unlock_t;

typedef struct MPIDI_CH3_Pkt_flush {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
} MPIDI_CH3_Pkt_flush_t;

788
typedef struct MPIDI_CH3_Pkt_lock_ack {
789 790
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
791
    /* note that either source_win_handle
792
     * or request_handle is used. */
793
    MPI_Win source_win_handle;
794
    MPI_Request request_handle;
795
    int target_rank;
796 797 798 799 800
} MPIDI_CH3_Pkt_lock_ack_t;

typedef struct MPIDI_CH3_Pkt_lock_op_ack {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
801
    /* note that either source_win_handle
802
     * or request_handle is used. */
803
    MPI_Win source_win_handle;
804
    MPI_Request request_handle;
805 806
    int target_rank;
} MPIDI_CH3_Pkt_lock_op_ack_t;
807

Xin Zhao's avatar
Xin Zhao committed
808 809
typedef struct MPIDI_CH3_Pkt_flush_ack {
    MPIDI_CH3_Pkt_type_t type;
810
    MPI_Win source_win_handle;
811
    int target_rank;
812
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
813
} MPIDI_CH3_Pkt_flush_ack_t;
814

815 816 817 818
typedef struct MPIDI_CH3_Pkt_decr_at_counter {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
} MPIDI_CH3_Pkt_decr_at_counter_t;
819

820
typedef struct MPIDI_CH3_Pkt_close {
821 822
    MPIDI_CH3_Pkt_type_t type;
    int ack;
823
} MPIDI_CH3_Pkt_close_t;
824

825
typedef struct MPIDI_CH3_Pkt_revoke {
Wesley Bland's avatar
Wesley Bland committed
826 827
    MPIDI_CH3_Pkt_type_t type;
    MPIR_Context_id_t revoked_comm;
828
} MPIDI_CH3_Pkt_revoke_t;
Wesley Bland's avatar
Wesley Bland committed
829

830
typedef union MPIDI_CH3_Pkt {
831 832
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_eager_send_t eager_send;
833
#if defined(USE_EAGER_SHORT)
834
    MPIDI_CH3_Pkt_eagershort_send_t eagershort_send;
835
#endif                          /* defined(USE_EAGER_SHORT) */
836 837 838 839 840 841 842 843 844 845
    MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send;
    MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack;
    MPIDI_CH3_Pkt_eager_send_t ready_send;
    MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send;
    MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send;
    MPIDI_CH3_Pkt_rndv_send_t rndv_send;
    MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req;
    MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp;
    MPIDI_CH3_Pkt_put_t put;
    MPIDI_CH3_Pkt_get_t get;
846
    MPIDI_CH3_Pkt_get_resp_t get_resp;
847
    MPIDI_CH3_Pkt_accum_t accum;
Xin Zhao's avatar
Xin Zhao committed
848
    MPIDI_CH3_Pkt_get_accum_t get_accum;
849
    MPIDI_CH3_Pkt_lock_t lock;
850 851
    MPIDI_CH3_Pkt_lock_ack_t lock_ack;
    MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack;
852
    MPIDI_CH3_Pkt_unlock_t unlock;
853
    MPIDI_CH3_Pkt_flush_t flush;
854
    MPIDI_CH3_Pkt_flush_ack_t flush_ack;
855
    MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt;
856
    MPIDI_CH3_Pkt_close_t close;
857 858 859 860 861
    MPIDI_CH3_Pkt_cas_t cas;
    MPIDI_CH3_Pkt_cas_resp_t cas_resp;
    MPIDI_CH3_Pkt_fop_t fop;
    MPIDI_CH3_Pkt_fop_resp_t fop_resp;
    MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp;
Wesley Bland's avatar
Wesley Bland committed
862
    MPIDI_CH3_Pkt_revoke_t revoke;
863
#if defined(MPIDI_CH3_PKT_DECL)
864