mpidpkt.h 41.8 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef HAVE_MPIDPKT_H
#define HAVE_MPIDPKT_H
9

10 11
#include "oputil.h"

12 13 14 15 16 17 18
#ifdef HAVE_STDINT_H
#  include <stdint.h>
#endif
#ifdef HAVE_INTTYPES_H
#  include <inttypes.h>
#endif

19 20 21
/* Enable the use of data within the message packet for small messages */
#define USE_EAGER_SHORT
#define MPIDI_EAGER_SHORT_INTS 4
22
/* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */
23 24
#define MPIDI_EAGER_SHORT_SIZE 16

25
/* This is the number of ints that can be carried within an RMA packet */
Xin Zhao's avatar
Xin Zhao committed
26
#define MPIDI_RMA_IMMED_BYTES 16
27

28 29 30
/* Union over all types (integer, logical, and multi-language types) that are
   allowed in a CAS operation.  This is used to allocate enough space in the
   packet header for immediate data.  */
31
typedef union {
32
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_;
33 34 35 36 37 38 39 40
    MPIR_OP_TYPE_GROUP(C_INTEGER)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER)
    MPIR_OP_TYPE_GROUP(LOGICAL)
    MPIR_OP_TYPE_GROUP(BYTE)
    MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA)
    MPIR_OP_TYPE_GROUP(BYTE_EXTRA)
41
#undef MPIR_OP_TYPE_MACRO
42 43
} MPIDI_CH3_CAS_Immed_u;

44 45 46 47
/* Union over all types (all predefined types) that are allowed in a
   Fetch-and-op operation.  This can be too large for the packet header, so we
   limit the immediate space in the header to FOP_IMMED_INTS. */

48 49 50
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the fields below */
51
typedef union {
52
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_;
53 54
    MPIR_OP_TYPE_GROUP_ALL_BASIC
    MPIR_OP_TYPE_GROUP_ALL_EXTRA
55
#undef MPIR_OP_TYPE_MACRO
56
} MPIDI_CH3_FOP_Immed_u;
57
/* *INDENT-ON* */
58

59
/*
60
 * Predefined packet types.  This simplifies some of the code.
61
 */
62 63
/* FIXME: Having predefined names makes it harder to add new message types,
   such as different RMA types. */
64
typedef enum {
Xin Zhao's avatar
Xin Zhao committed
65
    MPIDI_CH3_PKT_EAGER_SEND = 0,
66
#if defined(USE_EAGER_SHORT)
67
    MPIDI_CH3_PKT_EAGERSHORT_SEND,
68
#endif /* defined(USE_EAGER_SHORT) */
69
    MPIDI_CH3_PKT_EAGER_SYNC_SEND,      /* FIXME: no sync eager */
70 71 72 73
    MPIDI_CH3_PKT_EAGER_SYNC_ACK,
    MPIDI_CH3_PKT_READY_SEND,
    MPIDI_CH3_PKT_RNDV_REQ_TO_SEND,
    MPIDI_CH3_PKT_RNDV_CLR_TO_SEND,
74
    MPIDI_CH3_PKT_RNDV_SEND,    /* FIXME: should be stream put */
75 76
    MPIDI_CH3_PKT_CANCEL_SEND_REQ,
    MPIDI_CH3_PKT_CANCEL_SEND_RESP,
Xin Zhao's avatar
Xin Zhao committed
77 78
    /* RMA Packets begin here */
    MPIDI_CH3_PKT_PUT,
79 80
    MPIDI_CH3_PKT_GET,
    MPIDI_CH3_PKT_ACCUMULATE,
Xin Zhao's avatar
Xin Zhao committed
81 82 83 84 85 86 87
    MPIDI_CH3_PKT_GET_ACCUM,
    MPIDI_CH3_PKT_FOP,
    MPIDI_CH3_PKT_CAS,
    MPIDI_CH3_PKT_GET_RESP,
    MPIDI_CH3_PKT_GET_ACCUM_RESP,
    MPIDI_CH3_PKT_FOP_RESP,
    MPIDI_CH3_PKT_CAS_RESP,
88
    MPIDI_CH3_PKT_LOCK,
89 90
    MPIDI_CH3_PKT_LOCK_ACK,
    MPIDI_CH3_PKT_LOCK_OP_ACK,
91
    MPIDI_CH3_PKT_UNLOCK,
92
    MPIDI_CH3_PKT_FLUSH,
93
    MPIDI_CH3_PKT_FLUSH_ACK,
94
    MPIDI_CH3_PKT_DECR_AT_COUNTER,
95 96
    /* RMA Packets end here */
    MPIDI_CH3_PKT_FLOW_CNTL_UPDATE,     /* FIXME: Unused */
97
    MPIDI_CH3_PKT_CLOSE,
Wesley Bland's avatar
Wesley Bland committed
98
    MPIDI_CH3_PKT_REVOKE,
Pavan Balaji's avatar
Pavan Balaji committed
99
    MPIDI_CH3_PKT_END_CH3,
100
    /* The channel can define additional types by defining the value
101
     * MPIDI_CH3_PKT_ENUM */
102
# if defined(MPIDI_CH3_PKT_ENUM)
Pavan Balaji's avatar
Pavan Balaji committed
103
    MPIDI_CH3_PKT_ENUM,
104
# endif
Pavan Balaji's avatar
Pavan Balaji committed
105
    MPIDI_CH3_PKT_END_ALL,
106
    MPIDI_CH3_PKT_INVALID = -1  /* forces a signed enum to quash warnings */
Pavan Balaji's avatar
Pavan Balaji committed
107
} MPIDI_CH3_Pkt_type_t;
108

109
/* These flags can be "OR'ed" together */
110 111
typedef enum {
    MPIDI_CH3_PKT_FLAG_NONE = 0,
112 113 114 115 116 117 118 119 120 121 122 123 124 125
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8,
    MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16,
    MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32,
    MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096,
    MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192
126
} MPIDI_CH3_Pkt_flags_t;
127

128
typedef struct MPIDI_CH3_Pkt_send {
129 130
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
131
    MPI_Request sender_req_id;  /* needed for ssend and send cancel */
132 133 134
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
135 136
#endif
} MPIDI_CH3_Pkt_send_t;
137

138
/* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends,
139 140 141 142
   use the same structure but have a different type value. */
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t;
143 144

#if defined(USE_EAGER_SHORT)
145
typedef struct MPIDI_CH3_Pkt_eagershort_send {
146 147 148 149 150 151
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
#endif
152 153
    int data[MPIDI_EAGER_SHORT_INTS];   /* FIXME: Experimental for now */
} MPIDI_CH3_Pkt_eagershort_send_t;
154
#endif /* defined(USE_EAGER_SHORT) */
155

156
typedef struct MPIDI_CH3_Pkt_eager_sync_ack {
157 158
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
159
} MPIDI_CH3_Pkt_eager_sync_ack_t;
160 161 162

typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t;

163
typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send {
164 165 166
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    MPI_Request receiver_req_id;
167
} MPIDI_CH3_Pkt_rndv_clr_to_send_t;
168

169
typedef struct MPIDI_CH3_Pkt_rndv_send {
170 171
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request receiver_req_id;
172
} MPIDI_CH3_Pkt_rndv_send_t;
173

174
typedef struct MPIDI_CH3_Pkt_cancel_send_req {
175 176 177
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
178
} MPIDI_CH3_Pkt_cancel_send_req_t;
179

180
typedef struct MPIDI_CH3_Pkt_cancel_send_resp {
181 182 183
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    int ack;
184
} MPIDI_CH3_Pkt_cancel_send_resp_t;
185

186 187 188
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the field below */
189 190 191
#if defined(MPIDI_CH3_PKT_DEFS)
MPIDI_CH3_PKT_DEFS
#endif
192
/* *INDENT-ON* */
193

194 195
#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_)    \
    {                                                                   \
196 197
        /* This macro returns target_datatype in RMA operation          \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
198
        err_ = MPI_SUCCESS;                                             \
Xin Zhao's avatar
Xin Zhao committed
199
        switch((pkt_).type) {                                           \
200
        case (MPIDI_CH3_PKT_PUT):                                       \
Xin Zhao's avatar
Xin Zhao committed
201
            datatype_ = (pkt_).put.datatype;                            \
202 203
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
Xin Zhao's avatar
Xin Zhao committed
204
            datatype_ = (pkt_).get.datatype;                            \
205 206
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
Xin Zhao's avatar
Xin Zhao committed
207
            datatype_ = (pkt_).accum.datatype;                          \
208
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
209
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
Xin Zhao's avatar
Xin Zhao committed
210
            datatype_ = (pkt_).get_accum.datatype;                      \
Xin Zhao's avatar
Xin Zhao committed
211
            break;                                                      \
212
        case (MPIDI_CH3_PKT_CAS):                                       \
Xin Zhao's avatar
Xin Zhao committed
213
            datatype_ = (pkt_).cas.datatype;                            \
214 215
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
Xin Zhao's avatar
Xin Zhao committed
216
            datatype_ = (pkt_).fop.datatype;                            \
217 218
            break;                                                      \
        default:                                                        \
Xin Zhao's avatar
Xin Zhao committed
219
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
220 221 222
        }                                                               \
    }

223 224
#define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_)          \
    {                                                                   \
225 226
        /* This macro returns target_count in RMA operation             \
           packets. (PUT, GET, ACC, GACC, CAS, FOP) */                  \
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            count_ = (pkt_).put.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            count_ = (pkt_).get.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            count_ = (pkt_).accum.count;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            count_ = (pkt_).get_accum.count;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
        case (MPIDI_CH3_PKT_FOP):                                       \
            count_ = 1;                                                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_)         \
    {                                                                   \
252 253 254
        /* This macro returns immed_len in RMA operation                \
           packets (PUT, ACC, GACC, FOP, CAS) and RMA response          \
           packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */        \
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            immed_len_ = (pkt_).put.immed_len;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            immed_len_ = (pkt_).accum.immed_len;                        \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            immed_len_ = (pkt_).get_accum.immed_len;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            immed_len_ = (pkt_).fop.immed_len;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
            /* Note that here we return size of origin data, not        \
               size of compare data. */                                 \
            immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u);                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
            immed_len_ = (pkt_).get_resp.immed_len;                     \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
            immed_len_ = (pkt_).get_accum_resp.immed_len;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
            immed_len_ = (pkt_).fop_resp.immed_len;                     \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP):                                  \
            immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u);                 \
285 286 287 288 289 290 291 292
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_)   \
    {                                                                   \
293 294 295
        /* This macro returns pointer to immed data in RMA operation    \
           packets (PUT, ACC, GACC, FOP, CAS) and RMA response          \
           packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */        \
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            immed_data_ = (pkt_).put.data;                              \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            immed_data_ = (pkt_).accum.data;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            immed_data_ = (pkt_).get_accum.data;                        \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            immed_data_ = (pkt_).fop.data;                              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
            /* Note that here we return pointer of origin data, not     \
               pointer of compare data. */                              \
            immed_data_ = &((pkt_).cas.origin_data);                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
            immed_data_ = (pkt_).get_resp.data;                         \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
            immed_data_ = (pkt_).get_accum_resp.data;                   \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
            immed_data_ = (pkt_).fop_resp.data;                         \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP):                                  \
            immed_data_ = &((pkt_).cas_resp.data);                      \
326 327 328 329 330 331
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

Xin Zhao's avatar
Xin Zhao committed
332 333
#define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_)                 \
    {                                                                   \
334 335 336 337 338
        /* This macro returns flags in RMA operation packets (PUT, GET, \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
Xin Zhao's avatar
Xin Zhao committed
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            flags_ = (pkt_).put.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            flags_ = (pkt_).get.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            flags_ = (pkt_).accum.flags;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            flags_ = (pkt_).get_accum.flags;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            flags_ = (pkt_).cas.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            flags_ = (pkt_).fop.flags;                                  \
            break;                                                      \
359 360 361 362 363 364 365 366 367 368 369 370
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
            flags_ = (pkt_).get_resp.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
            flags_ = (pkt_).get_accum_resp.flags;                       \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
            flags_ = (pkt_).fop_resp.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP):                                  \
            flags_ = (pkt_).cas_resp.flags;                             \
            break;                                                      \
371 372 373
        case (MPIDI_CH3_PKT_LOCK):                                      \
            flags_ = (pkt_).lock.flags;                                 \
            break;                                                      \
374 375 376 377 378 379 380 381 382
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            flags_ = (pkt_).unlock.flags;                               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            flags_ = (pkt_).lock_ack.flags;                             \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            flags_ = (pkt_).lock_op_ack.flags;                          \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
383 384 385 386 387
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

388 389
#define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_)                       \
    {                                                                   \
390 391 392 393 394
        /* This macro erases flags in RMA operation packets (PUT, GET,  \
           ACC, GACC, FOP, CAS), RMA operation response packets         \
           (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control  \
           packets (UNLOCK) and RMA control response packets (LOCK_ACK, \
           LOCK_OP_ACK) */                                              \
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE;           \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
415 416 417 418 419 420 421 422 423 424 425 426
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
            (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
            (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;      \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
            (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP):                                  \
            (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
427 428 429
        case (MPIDI_CH3_PKT_LOCK):                                      \
            (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE;                \
            break;                                                      \
430 431 432 433 434 435 436 437 438
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE;         \
            break;                                                      \
439 440 441 442 443 444 445
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_)   \
    {                                                                   \
446 447 448 449 450
        /* This macro returns source_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation       \
           response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP),  \
           RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control   \
           response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */      \
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            win_hdl_ = (pkt_).put.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            win_hdl_ = (pkt_).get.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            win_hdl_ = (pkt_).accum.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            win_hdl_ = (pkt_).get_accum.source_win_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            win_hdl_ = (pkt_).cas.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            win_hdl_ = (pkt_).fop.source_win_handle;                    \
            break;                                                      \
471 472 473 474 475 476 477 478 479 480 481 482
        case (MPIDI_CH3_PKT_GET_RESP):                                  \
            win_hdl_ = (pkt_).get_resp.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM_RESP):                            \
            win_hdl_ = (pkt_).get_accum_resp.source_win_handle;         \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP_RESP):                                  \
            win_hdl_ = (pkt_).fop_resp.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS_RESP):                                  \
            win_hdl_ = (pkt_).cas_resp.source_win_handle;               \
            break;                                                      \
483 484 485
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.source_win_handle;                   \
            break;                                                      \
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.source_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_ACK):                                  \
            win_hdl_ = (pkt_).lock_ack.source_win_handle;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK_OP_ACK):                               \
            win_hdl_ = (pkt_).lock_op_ack.source_win_handle;            \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH_ACK):                                 \
            win_hdl_ = (pkt_).flush_ack.source_win_handle;              \
            break;                                                      \
501 502 503 504 505
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

506
#define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_)   \
Xin Zhao's avatar
Xin Zhao committed
507
    {                                                                   \
508 509 510
        /* This macro returns target_win_handle in RMA operation        \
           packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control      \
           packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */                \
Xin Zhao's avatar
Xin Zhao committed
511 512 513
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
514
            win_hdl_ = (pkt_).put.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
515 516
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
517
            win_hdl_ = (pkt_).get.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
518 519
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
520
            win_hdl_ = (pkt_).accum.target_win_handle;                  \
Xin Zhao's avatar
Xin Zhao committed
521 522
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
523
            win_hdl_ = (pkt_).get_accum.target_win_handle;              \
Xin Zhao's avatar
Xin Zhao committed
524 525
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
526
            win_hdl_ = (pkt_).cas.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
527 528
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
529
            win_hdl_ = (pkt_).fop.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
530
            break;                                                      \
531 532 533
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.target_win_handle;                   \
            break;                                                      \
534 535 536 537 538 539 540 541 542
        case (MPIDI_CH3_PKT_UNLOCK):                                    \
            win_hdl_ = (pkt_).unlock.target_win_handle;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FLUSH):                                     \
            win_hdl_ = (pkt_).flush.target_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_DECR_AT_COUNTER):                           \
            win_hdl_ = (pkt_).decr_at_cnt.target_win_handle;            \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
543 544 545 546 547 548 549
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \
    {                                                                   \
550 551
        /* This macro sets dataloop_size in RMA operation packets       \
           (PUT, GET, ACC, GACC) */                                     \
Xin Zhao's avatar
Xin Zhao committed
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            (pkt_).put.dataloop_size = (dataloop_size_);                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.dataloop_size = (dataloop_size_);                \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            (pkt_).accum.dataloop_size = (dataloop_size_);              \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            (pkt_).get_accum.dataloop_size = (dataloop_size_);          \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }


572
typedef struct MPIDI_CH3_Pkt_put {
573
    MPIDI_CH3_Pkt_type_t type;
574
    MPIDI_CH3_Pkt_flags_t flags;
575 576 577
    void *addr;
    int count;
    MPI_Datatype datatype;
578 579 580 581 582 583 584 585
    int dataloop_size;          /* for derived datatypes */
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
586
    char data[MPIDI_RMA_IMMED_BYTES];
587
    int immed_len;
588 589 590
} MPIDI_CH3_Pkt_put_t;

typedef struct MPIDI_CH3_Pkt_get {
591
    MPIDI_CH3_Pkt_type_t type;
592
    MPIDI_CH3_Pkt_flags_t flags;
593 594 595
    void *addr;
    int count;
    MPI_Datatype datatype;
596
    int dataloop_size;          /* for derived datatypes */
597
    MPI_Request request_handle;
598 599 600 601 602 603 604
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
605 606
} MPIDI_CH3_Pkt_get_t;

607 608 609 610 611 612 613
typedef struct MPIDI_CH3_Pkt_get_resp {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
614 615 616
    /* Followings are to piggyback IMMED data */
    int immed_len;
    char data[MPIDI_RMA_IMMED_BYTES];
617 618
} MPIDI_CH3_Pkt_get_resp_t;

619
typedef struct MPIDI_CH3_Pkt_accum {
620
    MPIDI_CH3_Pkt_type_t type;
621
    MPIDI_CH3_Pkt_flags_t flags;
622 623 624
    void *addr;
    int count;
    MPI_Datatype datatype;
625
    int dataloop_size;          /* for derived datatypes */
626
    MPI_Op op;
627 628 629 630 631 632 633
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
634
    char data[MPIDI_RMA_IMMED_BYTES];
635
    int immed_len;
636 637
} MPIDI_CH3_Pkt_accum_t;

Xin Zhao's avatar
Xin Zhao committed
638 639 640
typedef struct MPIDI_CH3_Pkt_get_accum {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
641
    MPI_Request request_handle; /* For get_accumulate response */
Xin Zhao's avatar
Xin Zhao committed
642 643 644
    void *addr;
    int count;
    MPI_Datatype datatype;
645
    int dataloop_size;          /* for derived datatypes */
Xin Zhao's avatar
Xin Zhao committed
646
    MPI_Op op;
647 648 649 650 651 652 653
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
654
    char data[MPIDI_RMA_IMMED_BYTES];
655
    int immed_len;
Xin Zhao's avatar
Xin Zhao committed
656 657
} MPIDI_CH3_Pkt_get_accum_t;

658
typedef struct MPIDI_CH3_Pkt_get_accum_resp {
659
    MPIDI_CH3_Pkt_type_t type;
660
    MPI_Request request_handle;
661 662 663 664
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
665 666 667
    /* Followings are to piggyback IMMED data */
    int immed_len;
    char data[MPIDI_RMA_IMMED_BYTES];
668
} MPIDI_CH3_Pkt_get_accum_resp_t;
669

670
typedef struct MPIDI_CH3_Pkt_cas {
671
    MPIDI_CH3_Pkt_type_t type;
672
    MPIDI_CH3_Pkt_flags_t flags;
673
    MPI_Datatype datatype;
674
    void *addr;
675
    MPI_Request request_handle;
676 677 678 679 680
    MPI_Win source_win_handle;
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
681 682
    MPIDI_CH3_CAS_Immed_u origin_data;
    MPIDI_CH3_CAS_Immed_u compare_data;
683
} MPIDI_CH3_Pkt_cas_t;
684

685
typedef struct MPIDI_CH3_Pkt_cas_resp {
686 687
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
688 689
    MPIDI_CH3_CAS_Immed_u data;
    /* followings are used to decrement ack_counter at orign */
690
    int target_rank;
691
    MPI_Win source_win_handle;
692 693
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_cas_resp_t;
694

695
typedef struct MPIDI_CH3_Pkt_fop {
696
    MPIDI_CH3_Pkt_type_t type;
697
    MPIDI_CH3_Pkt_flags_t flags;
698 699 700
    MPI_Datatype datatype;
    void *addr;
    MPI_Op op;
701
    MPI_Request request_handle;
702
    MPI_Win source_win_handle;
703 704 705 706
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
707
    char data[MPIDI_RMA_IMMED_BYTES];
708 709
    int immed_len;
} MPIDI_CH3_Pkt_fop_t;
710

711
typedef struct MPIDI_CH3_Pkt_fop_resp {
712 713
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
714 715 716
    char data[MPIDI_RMA_IMMED_BYTES];
    int immed_len;
    /* followings are used to decrement ack_counter at orign */
717
    int target_rank;
718 719 720
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_fop_resp_t;
721

722
typedef struct MPIDI_CH3_Pkt_lock {
723
    MPIDI_CH3_Pkt_type_t type;
724
    MPIDI_CH3_Pkt_flags_t flags;
725 726
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
727
} MPIDI_CH3_Pkt_lock_t;
728

Xin Zhao's avatar
Xin Zhao committed
729 730 731 732
typedef struct MPIDI_CH3_Pkt_unlock {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
733
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
734 735 736 737 738 739 740 741
} MPIDI_CH3_Pkt_unlock_t;

typedef struct MPIDI_CH3_Pkt_flush {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
} MPIDI_CH3_Pkt_flush_t;

742
typedef struct MPIDI_CH3_Pkt_lock_ack {
743 744 745
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
    MPI_Win source_win_handle;
746 747
    int target_rank;            /* Used in flush_ack response to look up the
                                 * target state at the origin. */
748 749 750 751 752 753 754 755
} MPIDI_CH3_Pkt_lock_ack_t;

typedef struct MPIDI_CH3_Pkt_lock_op_ack {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
    MPI_Win source_win_handle;
    int target_rank;
} MPIDI_CH3_Pkt_lock_op_ack_t;
756

Xin Zhao's avatar
Xin Zhao committed
757 758
typedef struct MPIDI_CH3_Pkt_flush_ack {
    MPIDI_CH3_Pkt_type_t type;
759
    MPI_Win source_win_handle;
760 761 762
    int target_rank;            /* Used in flush_ack response to look up the
                                 * target state at the origin. */
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
763
} MPIDI_CH3_Pkt_flush_ack_t;
764

765 766 767 768
typedef struct MPIDI_CH3_Pkt_decr_at_counter {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
} MPIDI_CH3_Pkt_decr_at_counter_t;
769

770
typedef struct MPIDI_CH3_Pkt_close {
771 772
    MPIDI_CH3_Pkt_type_t type;
    int ack;
773
} MPIDI_CH3_Pkt_close_t;
774

775
typedef struct MPIDI_CH3_Pkt_revoke {
Wesley Bland's avatar
Wesley Bland committed
776 777
    MPIDI_CH3_Pkt_type_t type;
    MPIR_Context_id_t revoked_comm;
778
} MPIDI_CH3_Pkt_revoke_t;
Wesley Bland's avatar
Wesley Bland committed
779

780
typedef union MPIDI_CH3_Pkt {
781 782
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_eager_send_t eager_send;
783
#if defined(USE_EAGER_SHORT)
784
    MPIDI_CH3_Pkt_eagershort_send_t eagershort_send;
785
#endif                          /* defined(USE_EAGER_SHORT) */
786 787 788 789 790 791 792 793 794 795
    MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send;
    MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack;
    MPIDI_CH3_Pkt_eager_send_t ready_send;
    MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send;
    MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send;
    MPIDI_CH3_Pkt_rndv_send_t rndv_send;
    MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req;
    MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp;
    MPIDI_CH3_Pkt_put_t put;
    MPIDI_CH3_Pkt_get_t get;
796
    MPIDI_CH3_Pkt_get_resp_t get_resp;
797
    MPIDI_CH3_Pkt_accum_t accum;
Xin Zhao's avatar
Xin Zhao committed
798
    MPIDI_CH3_Pkt_get_accum_t get_accum;
799
    MPIDI_CH3_Pkt_lock_t lock;
800 801
    MPIDI_CH3_Pkt_lock_ack_t lock_ack;
    MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack;
802
    MPIDI_CH3_Pkt_unlock_t unlock;
803
    MPIDI_CH3_Pkt_flush_t flush;
804
    MPIDI_CH3_Pkt_flush_ack_t flush_ack;
805
    MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt;
806
    MPIDI_CH3_Pkt_close_t close;
807 808 809 810 811
    MPIDI_CH3_Pkt_cas_t cas;
    MPIDI_CH3_Pkt_cas_resp_t cas_resp;
    MPIDI_CH3_Pkt_fop_t fop;
    MPIDI_CH3_Pkt_fop_resp_t fop_resp;
    MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp;
Wesley Bland's avatar
Wesley Bland committed
812
    MPIDI_CH3_Pkt_revoke_t revoke;
813
# if defined(MPIDI_CH3_PKT_DECL)
814
     MPIDI_CH3_PKT_DECL
815
# endif
816
} MPIDI_CH3_Pkt_t;
817 818

#if defined(MPID_USE_SEQUENCE_NUMBERS)
819
typedef struct MPIDI_CH3_Pkt_send_container {
820
    MPIDI_CH3_Pkt_send_t pkt;
821 822
    struct MPIDI_CH3_Pkt_send_container_s *next;
} MPIDI_CH3_Pkt_send_container_t;
823 824 825
#endif

#endif