mpidpkt.h 41.6 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef HAVE_MPIDPKT_H
#define HAVE_MPIDPKT_H
9

10 11
#include "oputil.h"

12
#ifdef HAVE_STDINT_H
13
#include <stdint.h>
14 15
#endif
#ifdef HAVE_INTTYPES_H
16
#include <inttypes.h>
17 18
#endif

19 20 21
/* Enable the use of data within the message packet for small messages */
#define USE_EAGER_SHORT
#define MPIDI_EAGER_SHORT_INTS 4
22
/* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */
23 24
#define MPIDI_EAGER_SHORT_SIZE 16

25
/* This is the number of ints that can be carried within an RMA packet */
Xin Zhao's avatar
Xin Zhao committed
26
#define MPIDI_RMA_IMMED_BYTES 16
27

28 29 30
/* Union over all types (integer, logical, and multi-language types) that are
   allowed in a CAS operation.  This is used to allocate enough space in the
   packet header for immediate data.  */
31
typedef union {
32
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_;
33 34 35 36 37 38 39 40
    MPIR_OP_TYPE_GROUP(C_INTEGER)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER)
    MPIR_OP_TYPE_GROUP(LOGICAL)
    MPIR_OP_TYPE_GROUP(BYTE)
    MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA)
    MPIR_OP_TYPE_GROUP(BYTE_EXTRA)
41
#undef MPIR_OP_TYPE_MACRO
42 43
} MPIDI_CH3_CAS_Immed_u;

44 45 46 47
/* Union over all types (all predefined types) that are allowed in a
   Fetch-and-op operation.  This can be too large for the packet header, so we
   limit the immediate space in the header to FOP_IMMED_INTS. */

48 49 50
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the fields below */
51
typedef union {
52
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_;
53 54
    MPIR_OP_TYPE_GROUP_ALL_BASIC
    MPIR_OP_TYPE_GROUP_ALL_EXTRA
55
#undef MPIR_OP_TYPE_MACRO
56
} MPIDI_CH3_FOP_Immed_u;
57
/* *INDENT-ON* */
58

59
/*
60
 * Predefined packet types.  This simplifies some of the code.
61
 */
62 63
/* FIXME: Having predefined names makes it harder to add new message types,
   such as different RMA types. */
64
typedef enum {
Xin Zhao's avatar
Xin Zhao committed
65
    MPIDI_CH3_PKT_EAGER_SEND = 0,
66
#if defined(USE_EAGER_SHORT)
67
    MPIDI_CH3_PKT_EAGERSHORT_SEND,
68
#endif /* defined(USE_EAGER_SHORT) */
69
    MPIDI_CH3_PKT_EAGER_SYNC_SEND,      /* FIXME: no sync eager */
70 71 72 73
    MPIDI_CH3_PKT_EAGER_SYNC_ACK,
    MPIDI_CH3_PKT_READY_SEND,
    MPIDI_CH3_PKT_RNDV_REQ_TO_SEND,
    MPIDI_CH3_PKT_RNDV_CLR_TO_SEND,
74
    MPIDI_CH3_PKT_RNDV_SEND,    /* FIXME: should be stream put */
75 76
    MPIDI_CH3_PKT_CANCEL_SEND_REQ,
    MPIDI_CH3_PKT_CANCEL_SEND_RESP,
Xin Zhao's avatar
Xin Zhao committed
77 78
    /* RMA Packets begin here */
    MPIDI_CH3_PKT_PUT,
79
    MPIDI_CH3_PKT_PUT_IMMED,
80 81
    MPIDI_CH3_PKT_GET,
    MPIDI_CH3_PKT_ACCUMULATE,
82
    MPIDI_CH3_PKT_ACCUMULATE_IMMED,
Xin Zhao's avatar
Xin Zhao committed
83
    MPIDI_CH3_PKT_GET_ACCUM,
84
    MPIDI_CH3_PKT_GET_ACCUM_IMMED,
Xin Zhao's avatar
Xin Zhao committed
85
    MPIDI_CH3_PKT_FOP,
86 87
    MPIDI_CH3_PKT_FOP_IMMED,
    MPIDI_CH3_PKT_CAS_IMMED,
Xin Zhao's avatar
Xin Zhao committed
88
    MPIDI_CH3_PKT_GET_RESP,
89
    MPIDI_CH3_PKT_GET_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
90
    MPIDI_CH3_PKT_GET_ACCUM_RESP,
91
    MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED,
Xin Zhao's avatar
Xin Zhao committed
92
    MPIDI_CH3_PKT_FOP_RESP,
93 94
    MPIDI_CH3_PKT_FOP_RESP_IMMED,
    MPIDI_CH3_PKT_CAS_RESP_IMMED,
95
    MPIDI_CH3_PKT_LOCK,
96 97
    MPIDI_CH3_PKT_LOCK_ACK,
    MPIDI_CH3_PKT_LOCK_OP_ACK,
98
    MPIDI_CH3_PKT_UNLOCK,
99
    MPIDI_CH3_PKT_FLUSH,
100
    MPIDI_CH3_PKT_FLUSH_ACK,
101
    MPIDI_CH3_PKT_DECR_AT_COUNTER,
102 103
    /* RMA Packets end here */
    MPIDI_CH3_PKT_FLOW_CNTL_UPDATE,     /* FIXME: Unused */
104
    MPIDI_CH3_PKT_CLOSE,
Wesley Bland's avatar
Wesley Bland committed
105
    MPIDI_CH3_PKT_REVOKE,
Pavan Balaji's avatar
Pavan Balaji committed
106
    MPIDI_CH3_PKT_END_CH3,
107
    /* The channel can define additional types by defining the value
108
     * MPIDI_CH3_PKT_ENUM */
109
#if defined(MPIDI_CH3_PKT_ENUM)
Pavan Balaji's avatar
Pavan Balaji committed
110
    MPIDI_CH3_PKT_ENUM,
111
#endif
Pavan Balaji's avatar
Pavan Balaji committed
112
    MPIDI_CH3_PKT_END_ALL,
113
    MPIDI_CH3_PKT_INVALID = -1  /* forces a signed enum to quash warnings */
Pavan Balaji's avatar
Pavan Balaji committed
114
} MPIDI_CH3_Pkt_type_t;
115

116
/* These flags can be "OR'ed" together */
117 118
typedef enum {
    MPIDI_CH3_PKT_FLAG_NONE = 0,
119 120 121 122 123 124 125 126 127 128 129 130 131 132
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8,
    MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16,
    MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32,
    MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096,
    MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192
133
} MPIDI_CH3_Pkt_flags_t;
134

135
typedef struct MPIDI_CH3_Pkt_send {
136 137
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
138
    MPI_Request sender_req_id;  /* needed for ssend and send cancel */
139 140 141
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
142 143
#endif
} MPIDI_CH3_Pkt_send_t;
144

145
/* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends,
146 147 148 149
   use the same structure but have a different type value. */
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t;
150 151

#if defined(USE_EAGER_SHORT)
152
typedef struct MPIDI_CH3_Pkt_eagershort_send {
153 154 155 156 157 158
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
#endif
159 160
    int data[MPIDI_EAGER_SHORT_INTS];   /* FIXME: Experimental for now */
} MPIDI_CH3_Pkt_eagershort_send_t;
161
#endif /* defined(USE_EAGER_SHORT) */
162

163
typedef struct MPIDI_CH3_Pkt_eager_sync_ack {
164 165
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
166
} MPIDI_CH3_Pkt_eager_sync_ack_t;
167 168 169

typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t;

170
typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send {
171 172 173
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    MPI_Request receiver_req_id;
174
} MPIDI_CH3_Pkt_rndv_clr_to_send_t;
175

176
typedef struct MPIDI_CH3_Pkt_rndv_send {
177 178
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request receiver_req_id;
179
} MPIDI_CH3_Pkt_rndv_send_t;
180

181
typedef struct MPIDI_CH3_Pkt_cancel_send_req {
182 183 184
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
185
} MPIDI_CH3_Pkt_cancel_send_req_t;
186

187
typedef struct MPIDI_CH3_Pkt_cancel_send_resp {
188 189 190
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    int ack;
191
} MPIDI_CH3_Pkt_cancel_send_resp_t;
192

193 194 195
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the field below */
196 197 198
#if defined(MPIDI_CH3_PKT_DEFS)
MPIDI_CH3_PKT_DEFS
#endif
199
/* *INDENT-ON* */
200

201 202
#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_)    \
    {                                                                   \
203 204
        /* This macro returns target_datatype in RMA operation          \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
205
        err_ = MPI_SUCCESS;                                             \
Xin Zhao's avatar
Xin Zhao committed
206
        switch((pkt_).type) {                                           \
207
        case (MPIDI_CH3_PKT_PUT):                                       \
208
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
209
            datatype_ = (pkt_).put.datatype;                            \
210 211
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
Xin Zhao's avatar
Xin Zhao committed
212
            datatype_ = (pkt_).get.datatype;                            \
213 214
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
215
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
216
            datatype_ = (pkt_).accum.datatype;                          \
217
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
218
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
219
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
220
            datatype_ = (pkt_).get_accum.datatype;                      \
Xin Zhao's avatar
Xin Zhao committed
221
            break;                                                      \
222
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
223
            datatype_ = (pkt_).cas.datatype;                            \
224 225
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
226
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
227
            datatype_ = (pkt_).fop.datatype;                            \
228 229
            break;                                                      \
        default:                                                        \
Xin Zhao's avatar
Xin Zhao committed
230
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
231 232 233
        }                                                               \
    }

234 235
#define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_)          \
    {                                                                   \
236 237
        /* This macro returns target_count in RMA operation             \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
238 239 240
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
241
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
242 243 244 245 246 247
            count_ = (pkt_).put.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            count_ = (pkt_).get.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
248
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
249 250 251
            count_ = (pkt_).accum.count;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
252
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
253 254
            count_ = (pkt_).get_accum.count;                            \
            break;                                                      \
255
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
256
        case (MPIDI_CH3_PKT_FOP):                                       \
257
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
258 259 260 261 262 263 264 265 266
            count_ = 1;                                                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_)   \
    {                                                                   \
267 268 269
        /* This macro returns pointer to immed data in RMA operation    \
           packets (PUT, ACC, GACC, FOP, CAS) and RMA response          \
           packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */        \
270 271
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
272 273
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
            immed_data_ = (pkt_).put.info.data;                         \
274
            break;                                                      \
275 276
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
            immed_data_ = (pkt_).accum.info.data;                       \
277
            break;                                                      \
278 279
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            immed_data_ = (pkt_).get_accum.info.data;                   \
280
            break;                                                      \
281 282
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            immed_data_ = (pkt_).fop.info.data;                         \
283
            break;                                                      \
284
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
285 286 287 288
            /* Note that here we return pointer of origin data, not     \
               pointer of compare data. */                              \
            immed_data_ = &((pkt_).cas.origin_data);                    \
            break;                                                      \
289 290
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            immed_data_ = (pkt_).get_resp.info.data;                    \
291
            break;                                                      \
292 293
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            immed_data_ = (pkt_).get_accum_resp.info.data;              \
294
            break;                                                      \
295 296
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            immed_data_ = (pkt_).fop_resp.info.data;                    \
297
            break;                                                      \
298 299
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            immed_data_ = &((pkt_).cas_resp.info.data);                 \
300 301 302 303 304 305
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

Xin Zhao's avatar
Xin Zhao committed
306 307
#define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_)                 \
    {                                                                   \
308 309 310 311 312
        /* This macro returns flags in RMA operation packets (PUT, GET, \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
Xin Zhao's avatar
Xin Zhao committed
313 314 315
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
316
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
317 318 319 320 321 322
            flags_ = (pkt_).put.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            flags_ = (pkt_).get.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
323
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
Xin Zhao's avatar
Xin Zhao committed
324 325 326
            flags_ = (pkt_).accum.flags;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
327
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
Xin Zhao's avatar
Xin Zhao committed
328 329
            flags_ = (pkt_).get_accum.flags;                            \
            break;                                                      \
330
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
331 332 333
            flags_ = (pkt_).cas.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
334
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
Xin Zhao's avatar
Xin Zhao committed
335 336
            flags_ = (pkt_).fop.flags;                                  \
            break;                                                      \
337
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
338
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
339 340 341
            flags_ = (pkt_).get_resp.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
342
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
343 344 345
            flags_ = (pkt_).get_accum_resp.flags;                       \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
346
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
347 348
            flags_ = (pkt_).fop_resp.flags;                             \
            break;                                                      \
349
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
350 351
            flags_ = (pkt_).cas_resp.flags;                             \
            break;                                                      \
352 353 354
        case (MPIDI_CH3_PKT_LOCK):                                      \
            flags_ = (pkt_).lock.flags;                                 \
            break;                                                      \
355 356 357 358 359 360 361 362 363
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            flags_ = (pkt_).unlock.flags;                               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            flags_ = (pkt_).lock_ack.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            flags_ = (pkt_).lock_op_ack.flags;                          \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
364 365 366 367 368
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

369 370
#define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_)                       \
    {                                                                   \
371 372 373 374 375
        /* This macro erases flags in RMA operation packets (PUT, GET,  \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
376 377 378
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
379
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
380 381 382 383 384 385
            (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
386
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
387 388 389
            (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
390
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
391 392
            (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE;           \
            break;                                                      \
393
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
394 395 396
            (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
397
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
398 399
            (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
400
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
401
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
402 403 404
            (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
405
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
406 407 408
            (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;      \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
409
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
410 411
            (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
412
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
413 414
            (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
415 416 417
        case (MPIDI_CH3_PKT_LOCK):                                      \
            (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE;                \
            break;                                                      \
418 419 420 421 422 423 424 425 426
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;         \
            break;                                                      \
427 428 429 430 431 432 433
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_)   \
    {                                                                   \
434 435 436 437 438
        /* This macro returns source_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation       \
           response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP),  \
           RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control   \
           response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */      \
439 440 441
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
442
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
443 444 445
            win_hdl_ = (pkt_).put.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
446
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
447 448 449 450 451
            win_hdl_ = (pkt_).accum.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.source_win_handle;                   \
            break;                                                      \
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.source_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            win_hdl_ = (pkt_).lock_ack.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            win_hdl_ = (pkt_).lock_op_ack.source_win_handle;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH_ACK):                                 \
            win_hdl_ = (pkt_).flush_ack.source_win_handle;              \
            break;                                                      \
467 468 469 470 471
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

472
#define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_)   \
Xin Zhao's avatar
Xin Zhao committed
473
    {                                                                   \
474 475 476
        /* This macro returns target_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control      \
           packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */                \
Xin Zhao's avatar
Xin Zhao committed
477 478 479
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
480
        case (MPIDI_CH3_PKT_PUT_IMMED):                                 \
481
            win_hdl_ = (pkt_).put.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
482 483
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
484
            win_hdl_ = (pkt_).get.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
485 486
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
487
        case (MPIDI_CH3_PKT_ACCUMULATE_IMMED):                          \
488
            win_hdl_ = (pkt_).accum.target_win_handle;                  \
Xin Zhao's avatar
Xin Zhao committed
489 490
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
491
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
492
            win_hdl_ = (pkt_).get_accum.target_win_handle;              \
Xin Zhao's avatar
Xin Zhao committed
493
            break;                                                      \
494
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
495
            win_hdl_ = (pkt_).cas.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
496 497
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
498
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
499
            win_hdl_ = (pkt_).fop.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
500
            break;                                                      \
501 502 503
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.target_win_handle;                   \
            break;                                                      \
504 505 506 507 508 509 510 511 512
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.target_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.target_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_DECR_AT_COUNTER):                           \
            win_hdl_ = (pkt_).decr_at_cnt.target_win_handle;            \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
513 514 515 516 517 518 519
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \
    {                                                                   \
520 521
        /* This macro sets dataloop_size in RMA operation packets       \
           (PUT, GET, ACC, GACC) */                                     \
Xin Zhao's avatar
Xin Zhao committed
522 523 524
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
525
            (pkt_).put.info.metadata.dataloop_size = (dataloop_size_);  \
Xin Zhao's avatar
Xin Zhao committed
526 527
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
528
            (pkt_).get.info.metadata.dataloop_size = (dataloop_size_);  \
Xin Zhao's avatar
Xin Zhao committed
529 530
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
531
            (pkt_).accum.info.metadata.dataloop_size = (dataloop_size_); \
Xin Zhao's avatar
Xin Zhao committed
532 533
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
534
            (pkt_).get_accum.info.metadata.dataloop_size = (dataloop_size_); \
Xin Zhao's avatar
Xin Zhao committed
535 536 537 538 539 540
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
#define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_)  \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_GET):                                       \
            request_hdl_ = (pkt_).get.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
        case (MPIDI_CH3_PKT_GET_ACCUM_IMMED):                           \
            request_hdl_ = (pkt_).get_accum.request_handle;             \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_IMMED):                                 \
            request_hdl_ = (pkt_).cas.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
        case (MPIDI_CH3_PKT_FOP_IMMED):                                 \
            request_hdl_ = (pkt_).fop.request_handle;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
        case (MPIDI_CH3_PKT_GET_RESP_IMMED):                            \
            request_hdl_ = (pkt_).get_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED):                      \
            request_hdl_ = (pkt_).get_accum_resp.request_handle;        \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
        case (MPIDI_CH3_PKT_FOP_RESP_IMMED):                            \
            request_hdl_ = (pkt_).fop_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP_IMMED):                            \
            request_hdl_ = (pkt_).cas_resp.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            request_hdl_ = (pkt_).lock.request_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            request_hdl_ = (pkt_).lock_ack.request_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            request_hdl_ = (pkt_).lock_op_ack.request_handle;           \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

588
typedef struct MPIDI_CH3_Pkt_put {
589
    MPIDI_CH3_Pkt_type_t type;
590
    MPIDI_CH3_Pkt_flags_t flags;
591 592 593
    void *addr;
    int count;
    MPI_Datatype datatype;
594 595
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
596
    union {
597 598 599 600 601 602
        /* note that we use struct here in order
         * to consistently access dataloop_size
         * by "pkt->info.metadata.dataloop_size". */
        struct {
            int dataloop_size;
        } metadata;
603 604
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
605 606 607
} MPIDI_CH3_Pkt_put_t;

typedef struct MPIDI_CH3_Pkt_get {
608
    MPIDI_CH3_Pkt_type_t type;
609
    MPIDI_CH3_Pkt_flags_t flags;
610 611 612
    void *addr;
    int count;
    MPI_Datatype datatype;
613 614
    struct {
        /* note that we use struct here in order
615
         * to consistently access dataloop_size
616 617 618 619
         * by "pkt->info.metadata.dataloop_size". */
        struct {
            int dataloop_size;  /* for derived datatypes */
        } metadata;
620
    } info;
621
    MPI_Request request_handle;
622
    MPI_Win target_win_handle;
623 624
} MPIDI_CH3_Pkt_get_t;

625 626 627 628 629 630
typedef struct MPIDI_CH3_Pkt_get_resp {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
631
    /* Followings are to piggyback IMMED data */
632 633
    struct {
        /* note that we use struct here in order
634 635
         * to consistently access data
         * by "pkt->info.data". */
636 637
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
638 639
} MPIDI_CH3_Pkt_get_resp_t;

640
typedef struct MPIDI_CH3_Pkt_accum {
641
    MPIDI_CH3_Pkt_type_t type;
642
    MPIDI_CH3_Pkt_flags_t flags;
643 644 645 646
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
647 648
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
649
    union {
650 651 652 653
        struct {
            int dataloop_size;
            MPI_Aint stream_offset;
        } metadata;
654 655
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
656 657
} MPIDI_CH3_Pkt_accum_t;

Xin Zhao's avatar
Xin Zhao committed
658 659 660
typedef struct MPIDI_CH3_Pkt_get_accum {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
661
    MPI_Request request_handle; /* For get_accumulate response */
Xin Zhao's avatar
Xin Zhao committed
662 663 664 665
    void *addr;
    int count;
    MPI_Datatype datatype;
    MPI_Op op;
666
    MPI_Win target_win_handle;
667
    union {
668 669 670 671
        struct {
            int dataloop_size;
            MPI_Aint stream_offset;
        } metadata;
672 673
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
Xin Zhao's avatar
Xin Zhao committed
674 675
} MPIDI_CH3_Pkt_get_accum_t;

676
typedef struct MPIDI_CH3_Pkt_get_accum_resp {
677
    MPIDI_CH3_Pkt_type_t type;
678
    MPI_Request request_handle;
679 680 681
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPIDI_CH3_Pkt_flags_t flags;
682
    /* Followings are to piggyback IMMED data */
683 684
    struct {
        /* note that we use struct here in order
685 686
         * to consistently access data
         * by "pkt->info.data". */
687 688
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
689
} MPIDI_CH3_Pkt_get_accum_resp_t;
690

691
typedef struct MPIDI_CH3_Pkt_cas {
692
    MPIDI_CH3_Pkt_type_t type;
693
    MPIDI_CH3_Pkt_flags_t flags;
694
    MPI_Datatype datatype;
695
    void *addr;
696
    MPI_Request request_handle;
697
    MPI_Win target_win_handle;
698 699
    MPIDI_CH3_CAS_Immed_u origin_data;
    MPIDI_CH3_CAS_Immed_u compare_data;
700
} MPIDI_CH3_Pkt_cas_t;
701

702
typedef struct MPIDI_CH3_Pkt_cas_resp {
703 704
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
705 706
    struct {
        /* note that we use struct here in order
707 708
         * to consistently access data
         * by "pkt->info.data". */
709 710
        MPIDI_CH3_CAS_Immed_u data;
    } info;
711
    /* followings are used to decrement ack_counter at orign */
712
    int target_rank;
713 714
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_cas_resp_t;
715

716
typedef struct MPIDI_CH3_Pkt_fop {
717
    MPIDI_CH3_Pkt_type_t type;
718
    MPIDI_CH3_Pkt_flags_t flags;
719 720 721
    MPI_Datatype datatype;
    void *addr;
    MPI_Op op;
722
    MPI_Request request_handle;
723
    MPI_Win target_win_handle;
724 725
    struct {
        /* note that we use struct here in order
726 727
         * to consistently access data
         * by "pkt->info.data". */
728 729
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
730
} MPIDI_CH3_Pkt_fop_t;
731

732
typedef struct MPIDI_CH3_Pkt_fop_resp {
733 734
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
735 736
    struct {
        /* note that we use struct here in order
737 738
         * to consistently access data
         * by "pkt->info.data". */
739 740
        char data[MPIDI_RMA_IMMED_BYTES];
    } info;
741
    /* followings are used to decrement ack_counter at orign */
742
    int target_rank;
743 744
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_fop_resp_t;
745

746
typedef struct MPIDI_CH3_Pkt_lock {
747
    MPIDI_CH3_Pkt_type_t type;
748
    MPIDI_CH3_Pkt_flags_t flags;
749
    MPI_Win target_win_handle;
750
    /* Note that either source_win_handle
751 752 753 754 755
     * or request_handle will be used. Here
     * we need both of them because PUT/GET
     * may be converted to LOCK packet,
     * PUT has source_win_handle area and
     * GET has request_handle area. */
756
    MPI_Win source_win_handle;
757
    MPI_Request request_handle;
758
} MPIDI_CH3_Pkt_lock_t;
759

Xin Zhao's avatar
Xin Zhao committed
760 761 762 763
typedef struct MPIDI_CH3_Pkt_unlock {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
764
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
765 766 767 768 769 770 771 772
} MPIDI_CH3_Pkt_unlock_t;

typedef struct MPIDI_CH3_Pkt_flush {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
} MPIDI_CH3_Pkt_flush_t;

773
typedef struct MPIDI_CH3_Pkt_lock_ack {
774 775
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
776
    /* note that either source_win_handle
777
     * or request_handle is used. */
778
    MPI_Win source_win_handle;
779
    MPI_Request request_handle;
780
    int target_rank;
781 782 783 784 785
} MPIDI_CH3_Pkt_lock_ack_t;

typedef struct MPIDI_CH3_Pkt_lock_op_ack {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
786
    /* note that either source_win_handle
787
     * or request_handle is used. */
788
    MPI_Win source_win_handle;
789
    MPI_Request request_handle;
790 791
    int target_rank;
} MPIDI_CH3_Pkt_lock_op_ack_t;
792

Xin Zhao's avatar
Xin Zhao committed
793 794
typedef struct MPIDI_CH3_Pkt_flush_ack {
    MPIDI_CH3_Pkt_type_t type;
795
    MPI_Win source_win_handle;
796
    int target_rank;
797
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
798
} MPIDI_CH3_Pkt_flush_ack_t;
799

800 801 802 803
typedef struct MPIDI_CH3_Pkt_decr_at_counter {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
} MPIDI_CH3_Pkt_decr_at_counter_t;
804

805
typedef struct MPIDI_CH3_Pkt_close {
806 807
    MPIDI_CH3_Pkt_type_t type;
    int ack;
808
} MPIDI_CH3_Pkt_close_t;
809

810
typedef struct MPIDI_CH3_Pkt_revoke {
Wesley Bland's avatar
Wesley Bland committed
811 812
    MPIDI_CH3_Pkt_type_t type;
    MPIR_Context_id_t revoked_comm;
813
} MPIDI_CH3_Pkt_revoke_t;
Wesley Bland's avatar
Wesley Bland committed
814

815
typedef union MPIDI_CH3_Pkt {
816 817
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_eager_send_t eager_send;
818
#if defined(USE_EAGER_SHORT)
819
    MPIDI_CH3_Pkt_eagershort_send_t eagershort_send;
820
#endif                          /* defined(USE_EAGER_SHORT) */
821 822 823 824 825 826 827 828 829 830
    MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send;
    MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack;
    MPIDI_CH3_Pkt_eager_send_t ready_send;
    MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send;
    MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send;
    MPIDI_CH3_Pkt_rndv_send_t rndv_send;
    MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req;
    MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp;
    MPIDI_CH3_Pkt_put_t put;
    MPIDI_CH3_Pkt_get_t get;
831
    MPIDI_CH3_Pkt_get_resp_t get_resp;
832
    MPIDI_CH3_Pkt_accum_t accum;
Xin Zhao's avatar
Xin Zhao committed
833
    MPIDI_CH3_Pkt_get_accum_t get_accum;
834
    MPIDI_CH3_Pkt_lock_t lock;
835 836
    MPIDI_CH3_Pkt_lock_ack_t lock_ack;
    MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack;
837
    MPIDI_CH3_Pkt_unlock_t unlock;
838
    MPIDI_CH3_Pkt_flush_t flush;
839
    MPIDI_CH3_Pkt_flush_ack_t flush_ack;
840
    MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt;
841
    MPIDI_CH3_Pkt_close_t close;
842 843 844 845 846
    MPIDI_CH3_Pkt_cas_t cas;
    MPIDI_CH3_Pkt_cas_resp_t cas_resp;
    MPIDI_CH3_Pkt_fop_t fop;
    MPIDI_CH3_Pkt_fop_resp_t fop_resp;
    MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp;
Wesley Bland's avatar
Wesley Bland committed
847
    MPIDI_CH3_Pkt_revoke_t revoke;
848
#if defined(MPIDI_CH3_PKT_DECL)
849
     MPIDI_CH3_PKT_DECL
850
#endif
851
} MPIDI_CH3_Pkt_t;
852 853

#if defined(MPID_USE_SEQUENCE_NUMBERS)